diff --git a/benchmarks/haskell/Benchmarks.hs b/benchmarks/haskell/Benchmarks.hs --- a/benchmarks/haskell/Benchmarks.hs +++ b/benchmarks/haskell/Benchmarks.hs @@ -10,6 +10,7 @@ import System.IO #ifdef mingw32_HOST_OS +import System.IO.Temp (emptySystemTempFile) import System.Directory (removeFile) #endif @@ -19,7 +20,9 @@ import qualified Benchmarks.EncodeUtf8 as EncodeUtf8 import qualified Benchmarks.Equality as Equality import qualified Benchmarks.FileRead as FileRead +import qualified Benchmarks.FileWrite as FileWrite import qualified Benchmarks.FoldLines as FoldLines +import qualified Benchmarks.Micro as Micro import qualified Benchmarks.Multilang as Multilang import qualified Benchmarks.Pure as Pure import qualified Benchmarks.ReadNumbers as ReadNumbers @@ -38,11 +41,11 @@ mkSink :: IO (FilePath, Handle) mkSink = do #ifdef mingw32_HOST_OS - (sinkFn, sink) <- openTempFile "." "dev.null" + sinkFn <- emptySystemTempFile "dev.null" #else let sinkFn = "/dev/null" - sink <- openFile sinkFn WriteMode #endif + sink <- openFile sinkFn WriteMode hSetEncoding sink utf8 pure (sinkFn, sink) @@ -58,9 +61,13 @@ let tf = ("benchmarks/text-test-data" ) -- Cannot use envWithCleanup, because there is no instance NFData Handle (sinkFn, sink) <- mkSink + (fileWriteBenchmarks, fileWriteCleanup) <- FileWrite.mkFileWriteBenchmarks $ do + (fp, h) <- mkSink + return (h, rmSink fp) defaultMain [ Builder.benchmark , Concat.benchmark + , Micro.benchmark , bgroup "DecodeUtf8" [ env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html") , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml") @@ -75,6 +82,7 @@ ] , env (Equality.initEnv (tf "japanese.txt")) Equality.benchmark , FileRead.benchmark (tf "russian.txt") + , fileWriteBenchmarks , FoldLines.benchmark (tf "russian.txt") , Multilang.benchmark , bgroup "Pure" @@ -100,3 +108,4 @@ ] ] rmSink sinkFn + fileWriteCleanup diff --git a/benchmarks/haskell/Benchmarks/FileWrite.hs b/benchmarks/haskell/Benchmarks/FileWrite.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/FileWrite.hs @@ -0,0 +1,135 @@ +-- | Benchmarks simple file writing +-- +-- Tested in this benchmark: +-- +-- * Writing a file to the disk +-- + +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE CPP #-} + +module Benchmarks.FileWrite + ( mkFileWriteBenchmarks + ) where + +import Control.DeepSeq (NFData, deepseq) +import Data.Bifunctor (first) +import Data.List (intercalate, intersperse) +import Data.String (fromString) +import Data.Text (StrictText) +import Data.Text.Internal.Lazy (LazyText, defaultChunkSize) +import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(..), hSetBuffering, hSetNewlineMode) +import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfAppIO) +import qualified Data.Text as T +import qualified Data.Text.IO as T +import qualified Data.Text.IO.Utf8 as Utf8 +import qualified Data.Text.Lazy as L +import qualified Data.Text.Lazy.IO as L + +mkFileWriteBenchmarks :: IO (Handle, IO ()) -> IO (Benchmark, IO ()) +mkFileWriteBenchmarks mkSinkNRemove = do + let writeData = L.cycle $ fromString [minBound..maxBound] + +#ifdef ExtendedBenchmarks + lengths = [0..5] <> [10,20..100] <> [1000,3000,10000,100000] +#else + lengths = [0,1,100,3000,10000,100000] +#endif + + testGroup :: NFData text => (Handle -> text -> IO ()) -> ((String, StrictText -> text)) -> Newline -> BufferMode -> IO (Benchmark, IO ()) + testGroup hPutStr (textCharacteristics, select) nl mode = do + (h, removeFile) <- mkSinkNRemove + hSetBuffering h mode + hSetNewlineMode h $ NewlineMode nl nl + pure + ( bgroup (intercalate " " [textCharacteristics, show nl, show mode]) $ + lengths <&> \n -> let + t = select $ L.toStrict $ L.take n writeData + in bench ("length " <> show n) + $ deepseq t + $ whnfAppIO (hPutStr h) t + , removeFile + ) + + sequenceGroup "FileWrite hPutStr" +#ifdef ExtendedBenchmarks + [ testGroup T.hPutStr strict LF NoBuffering + , testGroup L.hPutStr lazy LF NoBuffering + + , testGroup T.hPutStr strict LF LineBuffering + , testGroup T.hPutStr strict CRLF LineBuffering + , testGroup T.hPutStr strictNewlines LF LineBuffering + , testGroup T.hPutStr strictNewlines CRLF LineBuffering + + , testGroup L.hPutStr lazy LF LineBuffering + , testGroup L.hPutStr lazy CRLF LineBuffering + , testGroup L.hPutStr lazySmallChunks LF LineBuffering + , testGroup L.hPutStr lazySmallChunks CRLF LineBuffering + , testGroup L.hPutStr lazyNewlines LF LineBuffering + , testGroup L.hPutStr lazyNewlines CRLF LineBuffering + , testGroup L.hPutStr lazySmallChunksNewlines LF LineBuffering + , testGroup L.hPutStr lazySmallChunksNewlines CRLF LineBuffering + + , testGroup T.hPutStr strict LF (BlockBuffering Nothing) + , testGroup T.hPutStr strict CRLF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines LF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing) + + , testGroup L.hPutStr lazy LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazy CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunks LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunks CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunksNewlines LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazySmallChunksNewlines CRLF (BlockBuffering Nothing) + + , sequenceGroup "UTF-8" + [ testGroup Utf8.hPutStr strict LF NoBuffering + , testGroup Utf8.hPutStr strict LF LineBuffering + , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing) + ] + ] +#else + [ testGroup T.hPutStr strictNewlines LF LineBuffering + , testGroup T.hPutStr strictNewlines CRLF LineBuffering + + , testGroup T.hPutStr strict LF (BlockBuffering Nothing) + , testGroup T.hPutStr strictNewlines CRLF (BlockBuffering Nothing) + + , testGroup L.hPutStr lazyNewlines LF LineBuffering + , testGroup L.hPutStr lazyNewlines CRLF LineBuffering + + , testGroup L.hPutStr lazy LF (BlockBuffering Nothing) + , testGroup L.hPutStr lazyNewlines CRLF (BlockBuffering Nothing) + + , sequenceGroup "UTF-8" + [ testGroup Utf8.hPutStr strict LF LineBuffering + , testGroup Utf8.hPutStr strict LF (BlockBuffering Nothing) + ] + ] +#endif + + where + lazy, lazyNewlines :: (String, StrictText -> LazyText) + lazy = ("lazy", L.fromChunks . T.chunksOf defaultChunkSize) + lazyNewlines = ("lazy many newlines", snd lazy . snd strictNewlines) + +#ifdef ExtendedBenchmarks + lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText) + lazySmallChunks = ("lazy small chunks", L.fromChunks . T.chunksOf 10) + lazySmallChunksNewlines = ("lazy small chunks many newlines", snd lazySmallChunks . snd strictNewlines) +#endif + + strict, strictNewlines :: (String, StrictText -> StrictText) + strict = ("strict", id) + strictNewlines = ("strict many newlines", mconcat . intersperse "\n" . T.chunksOf 5) + + sequenceGroup groupName tgs + = first (bgroup groupName) + . foldr (\(b,r) (bs,rs) -> (b:bs,r>>rs)) ([], return ()) + <$> sequence tgs + +(<&>) :: Functor f => f a -> (a -> b) -> f b +(<&>) = flip fmap + diff --git a/benchmarks/haskell/Benchmarks/Micro.hs b/benchmarks/haskell/Benchmarks/Micro.hs new file mode 100644 --- /dev/null +++ b/benchmarks/haskell/Benchmarks/Micro.hs @@ -0,0 +1,33 @@ +-- | Benchmarks on artificial data. + +module Benchmarks.Micro (benchmark) where + +import qualified Data.List.NonEmpty as NE +import qualified Data.Text.Lazy as TL +import qualified Data.Text as T +import Test.Tasty.Bench (Benchmark, Benchmarkable, bgroup, bcompareWithin, bench, nf) + +benchmark :: Benchmark +benchmark = bgroup "Micro" + [ blinear "lazy-inits--last" 500000 2 0.1 $ \len -> + nf (NE.last . TL.initsNE) (chunks len) + , blinear "lazy-inits--map-take1" 500000 2 0.1 $ \len -> + nf (map (TL.take 1) . TL.inits) (chunks len) + ] + +chunks :: Int -> TL.Text +chunks n = TL.fromChunks (replicate n (T.pack "a")) + +-- Check that running an action with input length (m * baseLen) +-- runs m times slower than the same action with input length baseLen. +blinear :: String -- ^ Name (must be globally unique!) + -> Int -- ^ Base length + -> Int -- ^ Multiplier m + -> Double -- ^ Slack s + -> (Int -> Benchmarkable) -- ^ Action to measure, parameterized by input length + -> Benchmark +blinear name baseLen m s run = bgroup name + [ bench "baseline" $ run baseLen + , bcompareWithin (fromIntegral m * (1 - s)) (fromIntegral m * (1 + s)) (name ++ ".baseline") $ + bench ("x" ++ show m) $ run (m * baseLen) + ] diff --git a/benchmarks/haskell/Benchmarks/Programs/Fold.hs b/benchmarks/haskell/Benchmarks/Programs/Fold.hs --- a/benchmarks/haskell/Benchmarks/Programs/Fold.hs +++ b/benchmarks/haskell/Benchmarks/Programs/Fold.hs @@ -17,8 +17,9 @@ ( benchmark ) where -import Data.List (foldl') +import Data.Foldable (Foldable(..)) import Data.List (intersperse) +import Prelude hiding (Foldable(..)) import System.IO (Handle) import Test.Tasty.Bench (Benchmark, bench, whnfIO) import qualified Data.Text as T @@ -29,7 +30,7 @@ benchmark :: FilePath -> Handle -> Benchmark benchmark i o = - bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80 + bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . foldText 80 -- | We represent a paragraph by a word list -- @@ -37,8 +38,8 @@ -- | Fold a text -- -fold :: Int -> T.Text -> TL.Text -fold maxWidth = TLB.toLazyText . mconcat . +foldText :: Int -> T.Text -> TL.Text +foldText maxWidth = TLB.toLazyText . mconcat . intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs -- | Fold a paragraph diff --git a/benchmarks/haskell/Benchmarks/Pure.hs b/benchmarks/haskell/Benchmarks/Pure.hs --- a/benchmarks/haskell/Benchmarks/Pure.hs +++ b/benchmarks/haskell/Benchmarks/Pure.hs @@ -26,6 +26,8 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import qualified Data.Text.Lazy.Encoding as TL +import Data.Semigroup +import Data.List.NonEmpty (NonEmpty((:|))) data Env = Env { bsa :: !BS.ByteString @@ -83,6 +85,14 @@ [ benchT $ nf T.concat tl , benchTL $ nf TL.concat tll ] + , bgroup "sconcat" + [ benchT $ nf sconcat (T.empty :| tl) + , benchTL $ nf sconcat (TL.empty :| tll) + ] + , bgroup "stimes" + [ benchT $ nf (stimes (10 :: Int)) ta + , benchTL $ nf (stimes (10 :: Int)) tla + ] , bgroup "cons" [ benchT $ nf (T.cons c) ta , benchTL $ nf (TL.cons c) tla @@ -192,6 +202,10 @@ [ benchT $ nf T.toUpper ta , benchTL $ nf TL.toUpper tla ] + , bgroup "toTitle" + [ benchT $ nf T.toTitle ta + , benchTL $ nf TL.toTitle tla + ] , bgroup "uncons" [ benchT $ nf T.uncons ta , benchTL $ nf TL.uncons tla @@ -204,6 +218,14 @@ [ benchT $ nf (T.zipWith min tb) ta , benchTL $ nf (TL.zipWith min tlb) tla ] + , bgroup "length . unpack" -- length should fuse with unpack + [ benchT $ nf (L.length . T.unpack) ta + , benchTL $ nf (L.length . TL.unpack) tla + ] + , bgroup "length . drop 1 . unpack" -- no list fusion because of drop 1 + [ benchT $ nf (L.length . L.drop 1 . T.unpack) ta + , benchTL $ nf (L.length . L.drop 1 . TL.unpack) tla + ] , bgroup "length" [ bgroup "cons" [ benchT $ nf (T.length . T.cons c) ta @@ -268,6 +290,10 @@ , bgroup "toUpper" [ benchT $ nf (T.length . T.toUpper) ta , benchTL $ nf (TL.length . TL.toUpper) tla + ] + , bgroup "toTitle" + [ benchT $ nf (T.length . T.toTitle) ta + , benchTL $ nf (TL.length . TL.toTitle) tla ] , bgroup "words" [ benchT $ nf (L.length . T.words) ta diff --git a/benchmarks/haskell/Benchmarks/ReadNumbers.hs b/benchmarks/haskell/Benchmarks/ReadNumbers.hs --- a/benchmarks/haskell/Benchmarks/ReadNumbers.hs +++ b/benchmarks/haskell/Benchmarks/ReadNumbers.hs @@ -21,8 +21,9 @@ , benchmark ) where +import Data.Foldable (Foldable(..)) +import Prelude hiding (Foldable(..)) import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) -import Data.List (foldl') import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL diff --git a/benchmarks/haskell/Benchmarks/WordFrequencies.hs b/benchmarks/haskell/Benchmarks/WordFrequencies.hs --- a/benchmarks/haskell/Benchmarks/WordFrequencies.hs +++ b/benchmarks/haskell/Benchmarks/WordFrequencies.hs @@ -13,9 +13,10 @@ , benchmark ) where -import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) -import Data.List (foldl') +import Data.Foldable (Foldable(..)) import Data.Map (Map) +import Prelude hiding (Foldable(..)) +import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T diff --git a/cbits/aarch64/measure_off.c b/cbits/aarch64/measure_off.c new file mode 100644 --- /dev/null +++ b/cbits/aarch64/measure_off.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include + +/* + measure_off_naive / measure_off_neon + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +static inline const ssize_t measure_off_neon(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + uint8x16_t w128[4] = {vld1q_u8(src), vld1q_u8(src + 16), vld1q_u8(src + 32), vld1q_u8(src + 48)}; + // Which bytes are either < 128 or >= 192? + uint8x16_t mask0 = vcgtq_s8((int8x16_t)w128[0], vdupq_n_s8(0xBF)); + uint8x16_t mask1 = vcgtq_s8((int8x16_t)w128[1], vdupq_n_s8(0xBF)); + uint8x16_t mask2 = vcgtq_s8((int8x16_t)w128[2], vdupq_n_s8(0xBF)); + uint8x16_t mask3 = vcgtq_s8((int8x16_t)w128[3], vdupq_n_s8(0xBF)); + + uint8x16_t mask01 = vaddq_u8(mask0, mask1); + uint8x16_t mask23 = vaddq_u8(mask2, mask3); + uint8x16_t mask = vaddq_u8(mask01, mask23); + + size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask)); + + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + while (src < srcend - 15){ + uint8x16_t w128 = vld1q_u8(src); + // Which bytes are either < 128 or >= 192? + uint8x16_t mask = vcgtq_s8((int8x16_t)w128, vdupq_n_s8(0xBF)); + size_t leads = (size_t)(-vaddvq_s8((int8x16_t)mask)); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } + + return measure_off_naive(src, srcend, cnt); +} + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { + ssize_t ret = measure_off_neon(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +} diff --git a/cbits/measure_off.c b/cbits/measure_off.c --- a/cbits/measure_off.c +++ b/cbits/measure_off.c @@ -1,229 +1,231 @@ -/* - * Copyright (c) 2021 Andrew Lelechenko - */ - -#include -#include -#include -#ifdef __x86_64__ -#include -#include -#include -#include -#endif -#include - -// stdatomic.h has been introduces in gcc 4.9 -#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__)) -#define __STDC_NO_ATOMICS__ -#endif - -#ifndef __STDC_NO_ATOMICS__ -#include -#endif - -/* - Clang-6 does not enable proper -march flags for assembly modules - which leads to "error: instruction requires: AVX-512 ISA" - at the assembler phase. - - Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6 - https://en.wikipedia.org/wiki/Xcode#Toolchain_versions - and it's the latest available version on macOS 10.13. - - Disable AVX-512 instructions as they are most likely not supported - on the hardware running clang-6. -*/ -#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \ - || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__) -#define COMPILER_SUPPORTS_AVX512 -#endif - - -#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) -bool has_avx512_vl_bw() { -#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__) - uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; - __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); - // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features - const bool has_avx512_bw = ebx & (1 << 30); - const bool has_avx512_vl = ebx & (1 << 31); - // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); - return has_avx512_bw && has_avx512_vl; -#else - return false; -#endif -} -#endif - -/* - measure_off_naive / measure_off_avx / measure_off_sse - take a UTF-8 sequence between src and srcend, and a number of characters cnt. - If the sequence is long enough to contain cnt characters, then return how many bytes - remained unconsumed. Otherwise, if the sequence is shorter, return - negated count of lacking characters. Cf. _hs_text_measure_off below. -*/ - -static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) -{ - // Count leading bytes in 8 byte sequence - while (src < srcend - 7){ - uint64_t w64; - memcpy(&w64, src, sizeof(uint64_t)); - // find leading bytes by finding every byte that is not a continuation - // byte. The bit twiddle only results in a 0 if the original byte starts - // with 0b11... - w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; - // compute the popcount of w64 with two bit shifts and a multiplication - size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) - * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i - // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) - // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) - // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k - // and each i satisfies this equation for exactly one such j - // So each byte of the result contains the sum we want. - ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j - if (cnt < leads) break; - cnt-= leads; - src+= 8; - } - - // Skip until next leading byte - while (src < srcend){ - uint8_t w8 = *src; - if ((int8_t)w8 >= -0x40) break; - src++; - } - - // Finish up with tail - while (src < srcend && cnt > 0){ - uint8_t leadByte = *src++; - cnt--; - src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); - } - - return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); -} - -#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) -__attribute__((target("avx512vl,avx512bw"))) -static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) -{ - while (src < srcend - 63){ - __m512i w512 = _mm512_loadu_si512((__m512i *)src); - // Which bytes are either < 128 or >= 192? - uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); - size_t leads = __builtin_popcountll(mask); - if (cnt < leads) break; - cnt-= leads; - src+= 64; - } - - // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties - // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html - - if (src < srcend - 31){ - __m256i w256 = _mm256_loadu_si256((__m256i *)src); - uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); - size_t leads = __builtin_popcountl(mask); - if (cnt >= leads){ - cnt-= leads; - src+= 32; - } - } - - if (src < srcend - 15){ - __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 - uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 - size_t leads = __builtin_popcountl(mask); - if (cnt >= leads){ - cnt-= leads; - src+= 16; - } - } - - return measure_off_naive(src, srcend, cnt); -} -#endif - -/* Count the number of bits set in the argument - * - This is a temporary workaround for the fact that - the GHC RTS linker does not recognize the `__builtin_popcountll` - symbol. - - See https://gitlab.haskell.org/ghc/ghc/-/issues/21787 - https://gitlab.haskell.org/ghc/ghc/-/issues/19900 - https://github.com/haskell/text/issues/450 - - It can be removed and the usages of popcount64 replaced with - `__builtin_popcountll` once affected versions of the compiler - are no longer in widespread use. - - Once GHC learns to recognize the symbol, this can be gated - by CPP to call __builtin_popcountll when using the appropriate - version of GHC. -*/ -static inline const size_t popcount16(uint16_t x) { - - // Taken from https://en.wikipedia.org/wiki/Hamming_weight - const uint16_t m1 = 0x5555; //binary: 0101... - const uint16_t m2 = 0x3333; //binary: 00110011.. - const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ... - x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits - x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits - x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits - return (x >> 8) + (x & 0x00FF); -} - -static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) -{ -#ifdef __x86_64__ - while (src < srcend - 15){ - __m128i w128 = _mm_loadu_si128((__m128i *)src); - // Which bytes are either < 128 or >= 192? - uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); - size_t leads = popcount16(mask); - if (cnt < leads) break; - cnt-= leads; - src+= 16; - } -#endif - - return measure_off_naive(src, srcend, cnt); -} - -typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); - -/* - _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), - and a number of code points (aka characters) cnt. If the buffer is long enough - to contain cnt characters, then _hs_text_measure_off returns a non-negative number, - measuring their size in code units (aka bytes). If the buffer is shorter, - _hs_text_measure_off returns a non-positive number, which is a negated total count - of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 - as well. - - This scheme allows us to implement both take/drop and length with the same C function. - - The input buffer (src, off, len) must be a valid UTF-8 sequence, - this condition is not checked. -*/ -ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { -#ifndef __STDC_NO_ATOMICS__ - static _Atomic measure_off_t s_impl = (measure_off_t)NULL; - measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); - if (!impl) { -#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) - impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; -#else - impl = measure_off_sse; -#endif - atomic_store_explicit(&s_impl, impl, memory_order_relaxed); - } -#else - measure_off_t impl = measure_off_sse; -#endif - ssize_t ret = (*impl)(src + off, src + off + len, cnt); - return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); -} +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#ifdef __x86_64__ +#include +#include +#include +#include +#endif +#include + +// stdatomic.h has been introduces in gcc 4.9 +#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__)) +#ifndef __STDC_NO_ATOMICS__ +#define __STDC_NO_ATOMICS__ +#endif +#endif + +#ifndef __STDC_NO_ATOMICS__ +#include +#endif + +/* + Clang-6 does not enable proper -march flags for assembly modules + which leads to "error: instruction requires: AVX-512 ISA" + at the assembler phase. + + Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6 + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions + and it's the latest available version on macOS 10.13. + + Disable AVX-512 instructions as they are most likely not supported + on the hardware running clang-6. +*/ +#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \ + || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__) +#define COMPILER_SUPPORTS_AVX512 +#endif + + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +bool has_avx512_vl_bw() { +#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__) + uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; + __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); + // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features + const bool has_avx512_bw = ebx & (1 << 30); + const bool has_avx512_vl = ebx & (1 << 31); + // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); + return has_avx512_bw && has_avx512_vl; +#else + return false; +#endif +} +#endif + +/* + measure_off_naive / measure_off_avx / measure_off_sse + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +__attribute__((target("avx512vl,avx512bw"))) +static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + __m512i w512 = _mm512_loadu_si512((__m512i *)src); + // Which bytes are either < 128 or >= 192? + uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); + size_t leads = __builtin_popcountll(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties + // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html + + if (src < srcend - 31){ + __m256i w256 = _mm256_loadu_si256((__m256i *)src); + uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 32; + } + } + + if (src < srcend - 15){ + __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 + uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 16; + } + } + + return measure_off_naive(src, srcend, cnt); +} +#endif + +/* Count the number of bits set in the argument + * + This is a temporary workaround for the fact that + the GHC RTS linker does not recognize the `__builtin_popcountll` + symbol. + + See https://gitlab.haskell.org/ghc/ghc/-/issues/21787 + https://gitlab.haskell.org/ghc/ghc/-/issues/19900 + https://github.com/haskell/text/issues/450 + + It can be removed and the usages of popcount64 replaced with + `__builtin_popcountll` once affected versions of the compiler + are no longer in widespread use. + + Once GHC learns to recognize the symbol, this can be gated + by CPP to call __builtin_popcountll when using the appropriate + version of GHC. +*/ +static inline const size_t popcount16(uint16_t x) { + + // Taken from https://en.wikipedia.org/wiki/Hamming_weight + const uint16_t m1 = 0x5555; //binary: 0101... + const uint16_t m2 = 0x3333; //binary: 00110011.. + const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ... + x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits + x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits + return (x >> 8) + (x & 0x00FF); +} + +static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ +#ifdef __x86_64__ + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are either < 128 or >= 192? + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); + size_t leads = popcount16(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } +#endif + + return measure_off_naive(src, srcend, cnt); +} + +typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { +#ifndef __STDC_NO_ATOMICS__ + static _Atomic measure_off_t s_impl = (measure_off_t)NULL; + measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); + if (!impl) { +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) + impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; +#else + impl = measure_off_sse; +#endif + atomic_store_explicit(&s_impl, impl, memory_order_relaxed); + } +#else + measure_off_t impl = measure_off_sse; +#endif + ssize_t ret = (*impl)(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +} diff --git a/cbits/validate_utf8.cpp b/cbits/validate_utf8.cpp deleted file mode 100644 --- a/cbits/validate_utf8.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "simdutf.h" - -extern "C" -int _hs_text_is_valid_utf8(const char* str, size_t len){ - return simdutf::validate_utf8(str, len); -} - -extern "C" -int _hs_text_is_valid_utf8_offset(const char* str, size_t off, size_t len){ - return simdutf::validate_utf8(str + off, len); -} diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,346 +1,440 @@ -### 2.1.1 - -* Add pure Haskell implementations as an alternative to C-based ones, - suitable for JavaScript backend. - -* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547) - -* [Share empty `Text` values](https://github.com/haskell/text/pull/493) - -* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553) - -* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551) - -* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560) - -### 2.1 - -* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474) - -* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503) - -* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483) - -* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527) - -* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528) - -* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534) - -### 2.0.2 - -* [Add decoding functions in `Data.Text.Encoding` that allow - more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge! - * `decodeASCIIPrefix` - * `decodeUtf8Chunk` - * `decodeUtf8More` - * `Utf8ValidState` - * `startUtf8ValidState` - * `StrictBuilder` - * `strictBuilderToText` - * `textToStrictBuilder` - * `validateUtf8Chunk` - * `validateUtf8More` - -* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495) - -* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497) - -* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499) - -* Add internal module `Data.Text.Internal.StrictBuilder` - -* Add internal module `Data.Text.Internal.Encoding` - -* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module. - -* [Speed up case conversions](https://github.com/haskell/text/pull/460) - -* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468) - -* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485) - -### 2.0.1 - -* Improve portability of C and C++ code. -* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413) -* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402) -* [Add `fromPtr0`](https://github.com/haskell/text/pull/423) -* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436) -* [Add `withCString`](https://github.com/haskell/text/pull/431) -* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437) - -### 2.0 - -* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): - * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. - * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. - * Rename functions and types in `Data.Text.Foreign` to reflect switch - from `Word16` to `Word8`. - * Rename slicing functions in `Data.Text.Unsafe` to reflect switch - from `Word16` to `Word8`. - * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. - * Change semantics and order of arguments of `Data.Text.Array.copyI`: - pass length, not end offset. - * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. - * Extend interface of `Data.Text.Array` with more utility functions. - * Add `instance Show Data.Text.Unsafe.Iter`. - * Add `Data.Text.measureOff`. - * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. - * Export `Data.Text.Internal.Lazy.equal`. - * Export `Data.Text.Internal.append`. - * Add `Data.Text.Internal.Private.spanAscii_`. - * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. -* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) -* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) -* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) -* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) -* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) -* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) -* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) - -### 1.2.5.0 - -* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) -* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320) -* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274) -* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281) -* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325) -* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302) -* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334) -* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333) - -### 1.2.4.1 - -* Support `template-haskell-2.17.0.0` -* Support `bytestring-0.11` -* Add `take . drop` related RULE - -### 1.2.4.0 - -* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232) - -* Update Haddock documentation to better reflect fusion eligibility; improve fusion - rules for `takeWhileEnd` and `length` (gh-241, ghc-202) - -* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times, - call it only `O(log n)` times on chunks of increasing size. The total - asymptotic complexity remains `O(nm)`. (gh-209) - -* Support `base-4.13.0.0` - -### 1.2.3.1 - -* Make `decodeUtf8With` fail explicitly for unsupported non-BMP - replacement characters instead silent undefined behaviour (gh-213) - -* Fix termination condition for file reads via `Data.Text.IO` - operations (gh-223) - -* A serious correctness issue affecting uses of `take` and `drop` with - negative counts has been fixed (gh-227) - -* A bug in the case-mapping functions resulting in unreasonably large - allocations with large arguments has been fixed (gh-221) - -### 1.2.3.0 - -* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec - (updated from 8.0). - -* Bug fix: the lazy `takeWhileEnd` function violated the - [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51) - (gh-184). - -* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197). - -* New function: `unsnoc` (gh-173). - -* Reduce memory overhead in `encodeUTF8` (gh-194). - -* Improve UTF-8 decoder error-recovery (gh-182). - -* Minor documentation improvements (`@since` annotations, more - examples, clarifications). - -#### 1.2.2.2 - -* The `toTitle` function now correctly handles letters that - immediately follow punctuation. Before, `"there's"` would turn into - `"There'S"`. Now, it becomes `"There's"`. - -* The implementation of unstreaming is faster, resulting in operations - such as `map` and `intersperse` speeding up by up to 30%, with - smaller code generated. - -* The optimised length comparison function is now more likely to be - used after some rewrite rule tweaking. - -* Bug fix: an off-by-one bug in `takeEnd` is fixed. - -* Bug fix: a logic error in `takeWord16` is fixed. - -#### 1.2.2.1 - -* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. - The build flag has been renamed accordingly. Your army of diligent - maintainers apologizes for the churn. - -* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec - (updated from 7.0) - -* An STG lint error has been fixed - -### 1.2.2.0 - -* The `integer-simple` package, upon which this package optionally - depended, has been replaced with `integer-pure`. The build flag has - been renamed accordingly. - -* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a - `get`, the error is propagated via `fail` instead of an uncatchable - crash. - -* New function: `takeWhileEnd` - -* New instances for the `Text` types: - * if `base` >= 4.7: `PrintfArg` - * if `base` >= 4.9: `Semigroup` - -#### 1.2.1.3 - -* Bug fix: As it turns out, moving the literal rewrite rules to simplifier - phase 2 does not prevent competition with the `unpack` rule, which is - also active in this phase. Unfortunately this was hidden due to a silly - test environment mistake. Moving literal rules back to phase 1 finally - fixes GHC Trac #10528 correctly. - -#### 1.2.1.2 - -* Bug fix: Run literal rewrite rules in simplifier phase 2. - The behavior of the simplifier changed in GHC 7.10.2, - causing these rules to fail to fire, leading to poor code generation - and long compilation times. See - [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). - -#### 1.2.1.1 - -* Expose unpackCString#, which you should never use. - -### 1.2.1.0 - -* Added Binary instances for both Text types. (If you have previously - been using the text-binary package to get a Binary instance, it is - now obsolete.) - -#### 1.2.0.6 - -* Fixed a space leak in UTF-8 decoding - -#### 1.2.0.5 - -* Feature parity: repeat, cycle, iterate are now implemented for lazy - Text, and the Data instance is more complete - -* Build speed: an inliner space explosion has been fixed with toCaseFold - -* Bug fix: encoding Int to a Builder would infinite-loop if the - integer-simple package was used - -* Deprecation: OnEncodeError and EncodeError are deprecated, as they - are never used - -* Internals: some types that are used internally in fusion-related - functions have moved around, been renamed, or been deleted (we don't - bump the major version if .Internal modules change) - -* Spec compliance: toCaseFold now follows the Unicode 7.0 spec - (updated from 6.3) - -#### 1.2.0.4 - -* Fixed an incompatibility with base < 4.5 - -#### 1.2.0.3 - -* Update formatRealFloat to correspond to the definition in versions - of base newer than 4.5 (https://github.com/bos/text/issues/105) - -#### 1.2.0.2 - -* Bumped lower bound on deepseq to 1.4 for compatibility with the - upcoming GHC 7.10 - -#### 1.2.0.1 - -* Fixed a buffer overflow in rendering of large Integers - (https://github.com/bos/text/issues/99) - -## 1.2.0.0 - -* Fixed an integer overflow in the replace function - (https://github.com/bos/text/issues/81) - -* Fixed a hang in lazy decodeUtf8With - (https://github.com/bos/text/issues/87) - -* Reduced codegen bloat caused by use of empty and single-character - literals - -* Added an instance of IsList for GHC 7.8 and above - -### 1.1.1.0 - -* The Data.Data instance now allows gunfold to work, via a virtual - pack constructor - -* dropEnd, takeEnd: new functions - -* Comparing the length of a Text against a number can now - short-circuit in more cases - -#### 1.1.0.1 - -* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes - in single-byte chunks - -## 1.1.0.0 - -* encodeUtf8: Performance is improved by up to 4x. - -* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, - available only if bytestring >= 0.10.4.0 is installed, that allow - very fast and flexible encoding of a Text value to a bytestring - Builder. - - As an example of the performance gain to be had, the - encodeUtf8BuilderEscaped function helps to double the speed of JSON - encoding in the latest version of aeson! (Note: if all you need is a - plain ByteString, encodeUtf8 is still the faster way to go.) - -* All of the internal module hierarchy is now publicly exposed. If a - module is in the .Internal hierarchy, or is documented as internal, - use at your own risk - there are no API stability guarantees for - internal modules! - -#### 1.0.0.1 - -* decodeUtf8: Fixed a regression that caused us to incorrectly - identify truncated UTF-8 as valid (gh-61) - -# 1.0.0.0 - -* Added support for Unicode 6.3.0 to case conversion functions - -* New function toTitle converts words in a string to title case - -* New functions peekCStringLen and withCStringLen simplify - interoperability with C functions - -* Added support for decoding UTF-8 in stream-friendly fashion - -* Fixed a bug in mapAccumL - -* Added trusted Haskell support - -* Removed support for GHC 6.10 (released in 2008) and older +### 2.1.4 - 2026-01-27 + +* [Upgrade to Unicode 17.0](https://github.com/haskell/text/pull/658) + [Fix `CaseMapping` generation script to not depend on GHC's Unicode data](https://github.com/haskell/text/pull/687) + +* [simdutf: update to 8.0.0](https://github.com/haskell/text/pull/685) + +* [Add `decodeUtf8Lenient` for lazy `Text`](https://github.com/haskell/text/pull/690) + +* [`scanl`/`scanr` should replace invalid `Char` in the initial value](https://github.com/haskell/text/pull/669) + +* [Shave off redundant field of `Text.Internal.Buffer`](https://github.com/haskell/text/pull/659) + +* [Switch from template-haskell to template-haskell-lift](https://github.com/haskell/text/pull/661) + +#### Minor changes + +* [Avoid calling `length` on chunks in lazy `splitAt`](https://github.com/haskell/text/pull/676) + +* [Check for zero length in internal `isSingleton`](https://github.com/haskell/text/pull/675) + +* [Implement folds directly, without resorting to streaming framework](https://github.com/haskell/text/pull/667) + +* [Implement `cons`, `snoc`, `head`, `isSingleton`, `isPrefixOf` directly, without resorting to streaming framework](https://github.com/haskell/text/pull/666) + +* [Mark `caseConvert` (the underlying implementation of `toUpper` / `toLower` / `toTitle`) as `INLINABLE`, not `INLINE`](https://github.com/haskell/text/pull/664) + +* [Express `index` via `measureOff` instead of going through fusion framework](https://github.com/haskell/text/pull/663) + +* [Guard `#define __STDC_NO_ATOMICS__` by `#ifndef`](https://github.com/haskell/text/pull/657) + +* [Support QuickCheck-2.17](https://github.com/haskell/text/pull/662) + +* [Bump lower bound of binary to >= 0.8.3](https://github.com/haskell/text/pull/673) + +#### Documentation + +* [A bit more documentation for `Data.Text.Internal.Encoding.Utf8`](https://github.com/haskell/text/pull/691) + +* [Clarify documentation of `Data.Text.Foreign`](https://github.com/haskell/text/pull/681) + +* [Haddocks: Hyperlink some identifiers and modules](https://github.com/haskell/text/pull/677) + +* [`since` pragmas for type synonyms](https://github.com/haskell/text/pull/671) + +* [Improve documentation for `streamDecodeUtf8With`](https://github.com/haskell/text/pull/665) + +* [Add comprehensive documentation for `hGetChunk`](https://github.com/haskell/text/pull/655) + +### 2.1.3 - 2025-08-01 + +* [Fix CRLF handling in IO functions](https://github.com/haskell/text/pull/649) + +* [Change `utf8LengthByLeader` to a branching implementation](https://github.com/haskell/text/pull/635) + +* [Define `stimes 0` for lazy text](https://github.com/haskell/text/pull/641) + +* [Add implementation of `sconcat` and `stimes` for strict `Text`](https://github.com/haskell/text/pull/580) and [Fix `stimes` for strict text when size wraps around `Int`](https://github.com/haskell/text/pull/639) + +* [Allow list fusion for `unpack` over both strict and lazy `Text`](https://github.com/haskell/text/pull/629) + +### 2.1.2 + +* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618) + +* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581) + +* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558) + +* [Add `foldlM'`](https://github.com/haskell/text/pull/543) + +* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599) + +* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608) + +* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619) + +* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565) + +* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579) + +* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600) + +* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589) + +* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617) + +* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575) + +* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572) + +* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611) + +* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code. + +### 2.1.1 + +* Add pure Haskell implementations as an alternative to C-based ones, + suitable for JavaScript backend. + +* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547) + +* [Share empty `Text` values](https://github.com/haskell/text/pull/493) + +* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553) + +* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551) + +* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560) + +### 2.1 + +* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474) + +* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503) + +* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483) + +* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527) + +* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528) + +* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534) + +### 2.0.2 + +* [Add decoding functions in `Data.Text.Encoding` that allow + more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge! + * `decodeASCIIPrefix` + * `decodeUtf8Chunk` + * `decodeUtf8More` + * `Utf8ValidState` + * `startUtf8ValidState` + * `StrictBuilder` + * `strictBuilderToText` + * `textToStrictBuilder` + * `validateUtf8Chunk` + * `validateUtf8More` + +* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495) + +* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497) + +* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499) + +* Add internal module `Data.Text.Internal.StrictBuilder` + +* Add internal module `Data.Text.Internal.Encoding` + +* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module. + +* [Speed up case conversions](https://github.com/haskell/text/pull/460) + +* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468) + +* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485) + +### 2.0.1 + +* Improve portability of C and C++ code. +* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413) +* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402) +* [Add `fromPtr0`](https://github.com/haskell/text/pull/423) +* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436) +* [Add `withCString`](https://github.com/haskell/text/pull/431) +* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437) + +### 2.0 + +* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): + * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. + * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. + * Rename functions and types in `Data.Text.Foreign` to reflect switch + from `Word16` to `Word8`. + * Rename slicing functions in `Data.Text.Unsafe` to reflect switch + from `Word16` to `Word8`. + * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. + * Change semantics and order of arguments of `Data.Text.Array.copyI`: + pass length, not end offset. + * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. + * Extend interface of `Data.Text.Array` with more utility functions. + * Add `instance Show Data.Text.Unsafe.Iter`. + * Add `Data.Text.measureOff`. + * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. + * Export `Data.Text.Internal.Lazy.equal`. + * Export `Data.Text.Internal.append`. + * Add `Data.Text.Internal.Private.spanAscii_`. + * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. +* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) +* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) +* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) +* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) +* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) +* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) +* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) + +### 1.2.5.0 + +* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) +* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320) +* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274) +* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281) +* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325) +* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302) +* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334) +* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333) + +### 1.2.4.1 + +* Support `template-haskell-2.17.0.0` +* Support `bytestring-0.11` +* Add `take . drop` related RULE + +### 1.2.4.0 + +* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232) + +* Update Haddock documentation to better reflect fusion eligibility; improve fusion + rules for `takeWhileEnd` and `length` (gh-241, ghc-202) + +* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times, + call it only `O(log n)` times on chunks of increasing size. The total + asymptotic complexity remains `O(nm)`. (gh-209) + +* Support `base-4.13.0.0` + +### 1.2.3.1 + +* Make `decodeUtf8With` fail explicitly for unsupported non-BMP + replacement characters instead silent undefined behaviour (gh-213) + +* Fix termination condition for file reads via `Data.Text.IO` + operations (gh-223) + +* A serious correctness issue affecting uses of `take` and `drop` with + negative counts has been fixed (gh-227) + +* A bug in the case-mapping functions resulting in unreasonably large + allocations with large arguments has been fixed (gh-221) + +### 1.2.3.0 + +* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec + (updated from 8.0). + +* Bug fix: the lazy `takeWhileEnd` function violated the + [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51) + (gh-184). + +* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197). + +* New function: `unsnoc` (gh-173). + +* Reduce memory overhead in `encodeUTF8` (gh-194). + +* Improve UTF-8 decoder error-recovery (gh-182). + +* Minor documentation improvements (`@since` annotations, more + examples, clarifications). + +#### 1.2.2.2 + +* The `toTitle` function now correctly handles letters that + immediately follow punctuation. Before, `"there's"` would turn into + `"There'S"`. Now, it becomes `"There's"`. + +* The implementation of unstreaming is faster, resulting in operations + such as `map` and `intersperse` speeding up by up to 30%, with + smaller code generated. + +* The optimised length comparison function is now more likely to be + used after some rewrite rule tweaking. + +* Bug fix: an off-by-one bug in `takeEnd` is fixed. + +* Bug fix: a logic error in `takeWord16` is fixed. + +#### 1.2.2.1 + +* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. + The build flag has been renamed accordingly. Your army of diligent + maintainers apologizes for the churn. + +* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec + (updated from 7.0) + +* An STG lint error has been fixed + +### 1.2.2.0 + +* The `integer-simple` package, upon which this package optionally + depended, has been replaced with `integer-pure`. The build flag has + been renamed accordingly. + +* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a + `get`, the error is propagated via `fail` instead of an uncatchable + crash. + +* New function: `takeWhileEnd` + +* New instances for the `Text` types: + * if `base` >= 4.7: `PrintfArg` + * if `base` >= 4.9: `Semigroup` + +#### 1.2.1.3 + +* Bug fix: As it turns out, moving the literal rewrite rules to simplifier + phase 2 does not prevent competition with the `unpack` rule, which is + also active in this phase. Unfortunately this was hidden due to a silly + test environment mistake. Moving literal rules back to phase 1 finally + fixes GHC Trac #10528 correctly. + +#### 1.2.1.2 + +* Bug fix: Run literal rewrite rules in simplifier phase 2. + The behavior of the simplifier changed in GHC 7.10.2, + causing these rules to fail to fire, leading to poor code generation + and long compilation times. See + [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). + +#### 1.2.1.1 + +* Expose unpackCString#, which you should never use. + +### 1.2.1.0 + +* Added Binary instances for both Text types. (If you have previously + been using the text-binary package to get a Binary instance, it is + now obsolete.) + +#### 1.2.0.6 + +* Fixed a space leak in UTF-8 decoding + +#### 1.2.0.5 + +* Feature parity: repeat, cycle, iterate are now implemented for lazy + Text, and the Data instance is more complete + +* Build speed: an inliner space explosion has been fixed with toCaseFold + +* Bug fix: encoding Int to a Builder would infinite-loop if the + integer-simple package was used + +* Deprecation: OnEncodeError and EncodeError are deprecated, as they + are never used + +* Internals: some types that are used internally in fusion-related + functions have moved around, been renamed, or been deleted (we don't + bump the major version if .Internal modules change) + +* Spec compliance: toCaseFold now follows the Unicode 7.0 spec + (updated from 6.3) + +#### 1.2.0.4 + +* Fixed an incompatibility with base < 4.5 + +#### 1.2.0.3 + +* Update formatRealFloat to correspond to the definition in versions + of base newer than 4.5 (https://github.com/bos/text/issues/105) + +#### 1.2.0.2 + +* Bumped lower bound on deepseq to 1.4 for compatibility with the + upcoming GHC 7.10 + +#### 1.2.0.1 + +* Fixed a buffer overflow in rendering of large Integers + (https://github.com/bos/text/issues/99) + +## 1.2.0.0 + +* Fixed an integer overflow in the replace function + (https://github.com/bos/text/issues/81) + +* Fixed a hang in lazy decodeUtf8With + (https://github.com/bos/text/issues/87) + +* Reduced codegen bloat caused by use of empty and single-character + literals + +* Added an instance of IsList for GHC 7.8 and above + +### 1.1.1.0 + +* The Data.Data instance now allows gunfold to work, via a virtual + pack constructor + +* dropEnd, takeEnd: new functions + +* Comparing the length of a Text against a number can now + short-circuit in more cases + +#### 1.1.0.1 + +* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes + in single-byte chunks + +## 1.1.0.0 + +* encodeUtf8: Performance is improved by up to 4x. + +* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, + available only if bytestring >= 0.10.4.0 is installed, that allow + very fast and flexible encoding of a Text value to a bytestring + Builder. + + As an example of the performance gain to be had, the + encodeUtf8BuilderEscaped function helps to double the speed of JSON + encoding in the latest version of aeson! (Note: if all you need is a + plain ByteString, encodeUtf8 is still the faster way to go.) + +* All of the internal module hierarchy is now publicly exposed. If a + module is in the .Internal hierarchy, or is documented as internal, + use at your own risk - there are no API stability guarantees for + internal modules! + +#### 1.0.0.1 + +* decodeUtf8: Fixed a regression that caused us to incorrectly + identify truncated UTF-8 as valid (gh-61) + +# 1.0.0.0 + +* Added support for Unicode 6.3.0 to case conversion functions + +* New function toTitle converts words in a string to title case + +* New functions peekCStringLen and withCStringLen simplify + interoperability with C functions + +* Added support for decoding UTF-8 in stream-friendly fashion + +* Fixed a bug in mapAccumL + +* Added trusted Haskell support + +* Removed support for GHC 6.10 (released in 2008) and older diff --git a/scripts/Arsec.hs b/scripts/Arsec.hs --- a/scripts/Arsec.hs +++ b/scripts/Arsec.hs @@ -1,49 +1,48 @@ -module Arsec - ( - Comment - , comment - , semi - , showC - , unichar - , unichars - , module Control.Applicative - , module Control.Monad - , module Data.Char - , module Text.ParserCombinators.Parsec.Char - , module Text.ParserCombinators.Parsec.Combinator - , module Text.ParserCombinators.Parsec.Error - , module Text.ParserCombinators.Parsec.Prim - ) where - -import Prelude hiding (head, tail) -import Control.Monad -import Control.Applicative -import Data.Char -import Numeric (readHex, showHex) -import Text.ParserCombinators.Parsec.Char hiding (lower, upper) -import Text.ParserCombinators.Parsec.Combinator hiding (optional) -import Text.ParserCombinators.Parsec.Error -import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) - -type Comment = String - -unichar :: Parser Char -unichar = do - digits <- many1 hexDigit - case readHex digits of - [] -> error "unichar: cannot parse hex digits" - (hd, _) : _ -> pure $ chr hd - -unichars :: Parser [Char] -unichars = manyTill (unichar <* spaces) semi - -semi :: Parser () -semi = char ';' *> spaces *> pure () - -comment :: Parser Comment -comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" - -showC :: Char -> String -showC c = "'\\x" ++ d ++ "'" - where h = showHex (ord c) "" - d = replicate (4 - length h) '0' ++ h +module Arsec + ( + Comment + , comment + , semi + , showC + , unichar + , unichars + , module Control.Applicative + , module Control.Monad + , module Text.ParserCombinators.Parsec.Char + , module Text.ParserCombinators.Parsec.Combinator + , module Text.ParserCombinators.Parsec.Error + , module Text.ParserCombinators.Parsec.Prim + ) where + +import Prelude hiding (head, tail) +import Control.Monad +import Control.Applicative +import Data.Char +import Numeric (readHex, showHex) +import Text.ParserCombinators.Parsec.Char hiding (lower, upper) +import Text.ParserCombinators.Parsec.Combinator hiding (optional) +import Text.ParserCombinators.Parsec.Error +import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) + +type Comment = String + +unichar :: Parser Char +unichar = do + digits <- many1 hexDigit + case readHex digits of + [] -> error "unichar: cannot parse hex digits" + (hd, _) : _ -> pure $ chr hd + +unichars :: Parser [Char] +unichars = manyTill (unichar <* spaces) semi + +semi :: Parser () +semi = char ';' *> spaces *> pure () + +comment :: Parser Comment +comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" + +showC :: Char -> String +showC c = "'\\x" ++ d ++ "'" + where h = showHex (ord c) "" + d = replicate (4 - length h) '0' ++ h diff --git a/scripts/CaseFolding.hs b/scripts/CaseFolding.hs --- a/scripts/CaseFolding.hs +++ b/scripts/CaseFolding.hs @@ -1,46 +1,47 @@ --- This script processes the following source file: --- --- http://unicode.org/Public/UNIDATA/CaseFolding.txt - -module CaseFolding - ( - CaseFolding(..) - , Fold(..) - , parseCF - , mapCF - ) where - -import Arsec -import Data.Bits - -data Fold = Fold { - code :: Char - , status :: Char - , mapping :: [Char] - , name :: String - } deriving (Eq, Ord, Show) - -data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } - deriving (Show) - -entries :: Parser CaseFolding -entries = CF <$> many comment <*> many (entry <* many comment) - where - entry = Fold <$> unichar <* semi - <*> oneOf "CFST" <* semi - <*> unichars - <*> (string "# " *> manyTill anyToken (char '\n')) - -parseCF :: FilePath -> IO (Either ParseError CaseFolding) -parseCF name = parse entries name <$> readFile name - -mapCF :: CaseFolding -> [String] -mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last] - where - typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" - ,"{-# NOINLINE foldMapping #-}" - ,"foldMapping = \\case"] - last = " _ -> unI64 0" - printUnusual c = " -- " ++ name c ++ "\n" ++ - " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) - where x:y:z:_ = mapping c ++ repeat '\0' +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/CaseFolding.txt + +module CaseFolding + ( + CaseFolding(..) + , Fold(..) + , parseCF + , mapCF + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data Fold = Fold { + code :: Char + , status :: Char + , mapping :: [Char] + , name :: String + } deriving (Eq, Ord, Show) + +data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } + deriving (Show) + +entries :: Parser CaseFolding +entries = CF <$> many comment <*> many (entry <* many comment) + where + entry = Fold <$> unichar <* semi + <*> oneOf "CFST" <* semi + <*> unichars + <*> (string "# " *> manyTill anyToken (char '\n')) + +parseCF :: FilePath -> IO (Either ParseError CaseFolding) +parseCF name = parse entries name <$> readFile name + +mapCF :: CaseFolding -> [String] +mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last] + where + typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE foldMapping #-}" + ,"foldMapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = mapping c ++ repeat '\0' diff --git a/scripts/CaseMapping.hs b/scripts/CaseMapping.hs --- a/scripts/CaseMapping.hs +++ b/scripts/CaseMapping.hs @@ -1,41 +1,66 @@ -import System.Environment -import System.IO - -import Arsec -import CaseFolding -import SpecialCasing - -main = do - args <- getArgs - let oname = case args of - [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs" - [o] -> o - psc <- parseSC "SpecialCasing.txt" - pcf <- parseCF "CaseFolding.txt" - scs <- case psc of - Left err -> print err >> return undefined - Right ms -> return ms - cfs <- case pcf of - Left err -> print err >> return undefined - Right ms -> return ms - h <- openFile oname WriteMode - let comments = map ("--" ++) $ - take 2 (cfComments cfs) ++ take 2 (scComments scs) - mapM_ (hPutStrLn h) $ - ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" - ,"-- Generated by scripts/CaseMapping.hs"] ++ - comments ++ - ["" - ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" - ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" - ,"module Data.Text.Internal.Fusion.CaseMapping where" - ,"import GHC.Int" - ,"import GHC.Exts" - ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" - ,"unI64 (I64# n) = n" - ,""] - mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs) - mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs) - mapM_ (hPutStrLn h) (mapSC "title" title toTitle scs) - mapM_ (hPutStrLn h) (mapCF cfs) - hClose h +import Data.Char (isDigit) +import Data.Foldable (toList) +import Data.List (stripPrefix) +import Data.Maybe (fromJust) +import System.Environment +import System.IO + +import Arsec +import CaseFolding +import SpecialCasing +import UnicodeData + +main = do + args <- getArgs + let oname = case args of + [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs" + [o] -> o + psc <- parseSC "SpecialCasing.txt" + pcf <- parseCF "CaseFolding.txt" + ud <- parseUD "UnicodeData.txt" + scs <- case psc of + Left err -> print err >> return undefined + Right ms -> return ms + cfs <- case pcf of + Left err -> print err >> return undefined + Right ms -> return ms + ud <- case ud of + Left err -> print err >> return undefined + Right ms -> return ms + h <- openFile oname WriteMode + let comments = map ("--" ++) $ + take 2 (cfComments cfs) ++ take 2 (scComments scs) + version = parseVersion (cfComments cfs) + mapM_ (hPutStrLn h) $ + ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ,"-- Generated by scripts/CaseMapping.hs"] ++ + comments ++ + ["" + ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" + ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" + ,"module Data.Text.Internal.Fusion.CaseMapping where" + ,"import GHC.Int" + ,"import GHC.Exts" + ,"import Data.Version (Version, makeVersion)" + ,"unicodeVersion :: Version" + ,"unicodeVersion = makeVersion " ++ version + ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" + ,"unI64 (I64# n) = n" + ,""] + let get f = [(k, d) | c <- toList ud, Just d <- [f c], let k = charUD c, k /= d] + mapM_ (hPutStrLn h) (mapSC "upper" upper (get toUpperUD) scs) + mapM_ (hPutStrLn h) (mapSC "lower" lower (get toLowerUD) scs) + mapM_ (hPutStrLn h) (mapSC "title" title (get toTitleUD) scs) + mapM_ (hPutStrLn h) (mapCF cfs) + hClose h + +-- Parse version from CaseFolding comments +-- and render it as a list (an argument of makeVersion) +parseVersion :: [String] -> String +parseVersion comments = fromJust $ do + line' : _ <- pure comments + line'' <- stripPrefix " CaseFolding-" line' + let (v1, line1) = span isDigit line'' + (v2, line2) = span isDigit (drop 1 line1) + (v3, _) = span isDigit (drop 1 line2) + pure $ "[" ++ v1 ++ ", " ++ v2 ++ ", " ++ v3 ++ "]" diff --git a/scripts/SpecialCasing.hs b/scripts/SpecialCasing.hs --- a/scripts/SpecialCasing.hs +++ b/scripts/SpecialCasing.hs @@ -1,63 +1,61 @@ --- This script processes the following source file: --- --- http://unicode.org/Public/UNIDATA/SpecialCasing.txt - -module SpecialCasing - ( - SpecialCasing(..) - , Case(..) - , parseSC - , mapSC - ) where - -import Arsec -import Data.Bits - -data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } - deriving (Show) - -data Case = Case { - code :: Char - , lower :: [Char] - , title :: [Char] - , upper :: [Char] - , conditions :: String - , name :: String - } deriving (Eq, Ord, Show) - -entries :: Parser SpecialCasing -entries = SC <$> many comment <*> many (entry <* many comment) - where - entry = Case <$> unichar <* semi - <*> unichars - <*> unichars - <*> unichars - <*> manyTill anyToken (string "# ") - <*> manyTill anyToken (char '\n') - -parseSC :: FilePath -> IO (Either ParseError SpecialCasing) -parseSC name = parse entries name <$> readFile name - -mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing - -> [String] -mapSC which access twiddle (SC _ ms) = - typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] - where - ms' = filter p ms - p c = [k] /= a && a /= [twiddle k] && null (conditions c) - where a = access c - k = code c - unusual = map code ms' - usual = filter (\c -> twiddle c /= c && c `notElem` unusual) [minBound..maxBound] - - typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" - ,"{-# NOINLINE " ++ which ++ "Mapping #-}" - ,which ++ "Mapping = \\case"] - last = " _ -> unI64 0" - printUnusual c = " -- " ++ name c ++ "\n" ++ - " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) - where x:y:z:_ = access c ++ repeat '\0' - printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (twiddle c)) - -ucFirst (c:cs) = toUpper c : cs -ucFirst [] = [] +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/SpecialCasing.txt + +module SpecialCasing + ( + SpecialCasing(..) + , Case(..) + , parseSC + , mapSC + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } + deriving (Show) + +data Case = Case { + code :: Char + , lower :: [Char] + , title :: [Char] + , upper :: [Char] + , conditions :: String + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser SpecialCasing +entries = SC <$> many comment <*> many (entry <* many comment) + where + entry = Case <$> unichar <* semi + <*> unichars + <*> unichars + <*> unichars + <*> manyTill anyToken (string "# ") + <*> manyTill anyToken (char '\n') + +parseSC :: FilePath -> IO (Either ParseError SpecialCasing) +parseSC name = parse entries name <$> readFile name + +mapSC :: String -> (Case -> String) -> [(Char, Char)] -> SpecialCasing + -> [String] +mapSC which access twiddle (SC _ ms) = + typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] + where + ms' = filter p ms + p c = [k] /= a && null (conditions c) + where a = access c + k = code c + unusual = map code ms' + usual = filter (\(c, _) -> c `notElem` unusual) twiddle + + typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE " ++ which ++ "Mapping #-}" + ,which ++ "Mapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = access c ++ repeat '\0' + printUsual (c, c') = " " ++ showC c ++ "# -> unI64 " ++ show (ord c') diff --git a/scripts/UnicodeData.hs b/scripts/UnicodeData.hs new file mode 100644 --- /dev/null +++ b/scripts/UnicodeData.hs @@ -0,0 +1,51 @@ +-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/UnicodeData.txt +-- +-- Format description: https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt + +module UnicodeData + ( UnicodeData + , Data(..) + , toTitleUD + , parseUD + ) where + +import Debug.Trace +import Arsec hiding (semi) +import Data.Array +import Data.Functor (void) +import Data.List (sort) +import Data.Maybe (fromMaybe) + +type UnicodeData = Array Int Data + +-- "Simple_Titlecase_Mapping: If this field is null, then the Simple_Titlecase_Mapping +-- is the same as the Simple_Uppercase_Mapping for this character." +-- -- https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt +toTitleUD :: Data -> Maybe Char +toTitleUD d = toTitleUD_ d <|> toUpperUD d + +data Data = Data { + charUD :: {-# UNPACK #-} !Char + , toUpperUD :: {-# UNPACK #-} !(Maybe Char) + , toLowerUD :: {-# UNPACK #-} !(Maybe Char) + , toTitleUD_ :: {-# UNPACK #-} !(Maybe Char) + } deriving (Eq, Ord, Show) + +-- I'm pretty sure UnicodeData.txt is sorted but still sort it to be 100% certain. +entries :: Parser UnicodeData +entries = (\xs -> listArray (0, length xs - 1) xs) <$> many entry <* eof + where + entry = Data <$> unichar <* semi + <* replicateM_ 11 (ignoreField <* semi) + <*> optional unichar <* semi + <*> optional unichar <* semi + <*> optional unichar <* char '\n' + semi = char ';' + +ignoreField :: Parser () +ignoreField = void (many (satisfy (\c -> c /= ';'))) + +parseUD :: FilePath -> IO (Either ParseError UnicodeData) +parseUD name = parse entries name <$> readFile name diff --git a/simdutf/hs_simdutf.c b/simdutf/hs_simdutf.c new file mode 100644 --- /dev/null +++ b/simdutf/hs_simdutf.c @@ -0,0 +1,9 @@ +#include "simdutf_c.h" + +int _hs_text_is_valid_utf8(const char *buf, size_t len) { + return simdutf_validate_utf8(buf, len); +} + +int _hs_text_is_valid_utf8_offset(const char *buf, size_t off, size_t len) { + return simdutf_validate_utf8(buf + off, len); +} diff --git a/simdutf/simdutf.cpp b/simdutf/simdutf.cpp # file too large to diff: simdutf/simdutf.cpp diff --git a/simdutf/simdutf.h b/simdutf/simdutf.h --- a/simdutf/simdutf.h +++ b/simdutf/simdutf.h @@ -1,1142 +1,13573 @@ -/* auto-generated on 2022-03-21 23:28:26 -0400. Do not edit! */ -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf.h -/* begin file include/simdutf.h */ -#ifndef SIMDUTF_H -#define SIMDUTF_H -#include -#include -#include - -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/compiler_check.h -/* begin file include/simdutf/compiler_check.h */ -#ifndef SIMDUTF_COMPILER_CHECK_H -#define SIMDUTF_COMPILER_CHECK_H - -#ifndef __cplusplus -#error simdutf requires a C++ compiler -#endif - -#ifndef SIMDUTF_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define SIMDUTF_CPLUSPLUS __cplusplus -#endif -#endif - -// C++ 17 -#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) -#define SIMDUTF_CPLUSPLUS17 1 -#endif - -// C++ 14 -#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) -#define SIMDUTF_CPLUSPLUS14 1 -#endif - -// C++ 11 -#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) -#define SIMDUTF_CPLUSPLUS11 1 -#endif - -#ifndef SIMDUTF_CPLUSPLUS11 -#error simdutf requires a compiler compliant with the C++11 standard -#endif - -#endif // SIMDUTF_COMPILER_CHECK_H -/* end file include/simdutf/compiler_check.h */ -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/common_defs.h -/* begin file include/simdutf/common_defs.h */ -#ifndef SIMDUTF_COMMON_DEFS_H -#define SIMDUTF_COMMON_DEFS_H - -#include -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/portability.h -/* begin file include/simdutf/portability.h */ -#ifndef SIMDUTF_PORTABILITY_H -#define SIMDUTF_PORTABILITY_H - -#include -#include -#include -#include -#include -#ifndef _WIN32 -// strcasecmp, strncasecmp -#include -#endif - -#ifdef _MSC_VER -#define SIMDUTF_VISUAL_STUDIO 1 -/** - * We want to differentiate carefully between - * clang under visual studio and regular visual - * studio. - * - * Under clang for Windows, we enable: - * * target pragmas so that part and only part of the - * code gets compiled for advanced instructions. - * - */ -#ifdef __clang__ -// clang under visual studio -#define SIMDUTF_CLANG_VISUAL_STUDIO 1 -#else -// just regular visual studio (best guess) -#define SIMDUTF_REGULAR_VISUAL_STUDIO 1 -#endif // __clang__ -#endif // _MSC_VER - -#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO -// https://en.wikipedia.org/wiki/C_alternative_tokens -// This header should have no effect, except maybe -// under Visual Studio. -#include -#endif - -#if defined(__x86_64__) || defined(_M_AMD64) -#define SIMDUTF_IS_X86_64 1 -#elif defined(__aarch64__) || defined(_M_ARM64) -#define SIMDUTF_IS_ARM64 1 -#elif defined(__PPC64__) || defined(_M_PPC64) -//#define SIMDUTF_IS_PPC64 1 -#pragma message("The simdutf library does yet support SIMD acceleration under\ -POWER processors. Please see https://github.com/lemire/simdutf/issues/51") -#else -// The simdutf library is designed -// for 64-bit processors and it seems that you are not -// compiling for a known 64-bit platform. Please -// use a 64-bit target such as x64 or 64-bit ARM for best performance. -#define SIMDUTF_IS_32BITS 1 - -// We do not support 32-bit platforms, but it can be -// handy to identify them. -#if defined(_M_IX86) || defined(__i386__) -#define SIMDUTF_IS_X86_32BITS 1 -#elif defined(__arm__) || defined(_M_ARM) -#define SIMDUTF_IS_ARM_32BITS 1 -#elif defined(__PPC__) || defined(_M_PPC) -#define SIMDUTF_IS_PPC_32BITS 1 -#endif - -#endif // defined(__x86_64__) || defined(_M_AMD64) - -#ifdef SIMDUTF_IS_32BITS -#ifndef SIMDUTF_NO_PORTABILITY_WARNING -#pragma message("The simdutf library is designed \ -for 64-bit processors and it seems that you are not \ -compiling for a known 64-bit platform. All fast kernels \ -will be disabled and performance may be poor. Please \ -use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.") -#endif // SIMDUTF_NO_PORTABILITY_WARNING -#endif // SIMDUTF_IS_32BITS - -// this is almost standard? -#undef STRINGIFY_IMPLEMENTATION_ -#undef STRINGIFY -#define STRINGIFY_IMPLEMENTATION_(a) #a -#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) - -// Our fast kernels require 64-bit systems. -// -// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. -// Furthermore, the number of SIMD registers is reduced. -// -// On 32-bit ARM, we would have smaller registers. -// -// The simdutf users should still have the fallback kernel. It is -// slower, but it should run everywhere. - -// -// Enable valid runtime implementations, and select SIMDUTF_BUILTIN_IMPLEMENTATION -// - -// We are going to use runtime dispatch. -#ifdef SIMDUTF_IS_X86_64 -#ifdef __clang__ -// clang does not have GCC push pop -// warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be *outside* of a -// namespace. -#define SIMDUTF_TARGET_REGION(T) \ - _Pragma(STRINGIFY( \ - clang attribute push(__attribute__((target(T))), apply_to = function))) -#define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") -#elif defined(__GNUC__) -// GCC is easier -#define SIMDUTF_TARGET_REGION(T) \ - _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) -#define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") -#endif // clang then gcc - -#endif // x86 - -// Default target region macros don't do anything. -#ifndef SIMDUTF_TARGET_REGION -#define SIMDUTF_TARGET_REGION(T) -#define SIMDUTF_UNTARGET_REGION -#endif - -// Is threading enabled? -#if defined(_REENTRANT) || defined(_MT) -#ifndef SIMDUTF_THREADS_ENABLED -#define SIMDUTF_THREADS_ENABLED -#endif -#endif - -// workaround for large stack sizes under -O0. -// https://github.com/simdutf/simdutf/issues/691 -#ifdef __APPLE__ -#ifndef __OPTIMIZE__ -// Apple systems have small stack sizes in secondary threads. -// Lack of compiler optimization may generate high stack usage. -// Users may want to disable threads for safety, but only when -// in debug mode which we detect by the fact that the __OPTIMIZE__ -// macro is not defined. -#undef SIMDUTF_THREADS_ENABLED -#endif -#endif - - -#if defined(__clang__) -#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) -#elif defined(__GNUC__) -#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) -#else -#define NO_SANITIZE_UNDEFINED -#endif - -#ifdef SIMDUTF_VISUAL_STUDIO -// This is one case where we do not distinguish between -// regular visual studio and clang under visual studio. -// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has) -#define simdutf_strcasecmp _stricmp -#define simdutf_strncasecmp _strnicmp -#else -// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8). -// So they are only useful for ASCII in our context. -// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings -#define simdutf_strcasecmp strcasecmp -#define simdutf_strncasecmp strncasecmp -#endif - -#ifdef NDEBUG - -#ifdef SIMDUTF_VISUAL_STUDIO -#define SIMDUTF_UNREACHABLE() __assume(0) -#define SIMDUTF_ASSUME(COND) __assume(COND) -#else -#define SIMDUTF_UNREACHABLE() __builtin_unreachable(); -#define SIMDUTF_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) -#endif - -#else // NDEBUG - -#define SIMDUTF_UNREACHABLE() assert(0); -#define SIMDUTF_ASSUME(COND) assert(COND) - -#endif - -#endif // SIMDUTF_PORTABILITY_H -/* end file include/simdutf/portability.h */ - - -#if defined(__GNUC__) - // Marks a block with a name so that MCA analysis can see it. - #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); - #define SIMDUTF_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); - #define SIMDUTF_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); -#else - #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) - #define SIMDUTF_END_DEBUG_BLOCK(name) - #define SIMDUTF_DEBUG_BLOCK(name, block) -#endif - -// Align to N-byte boundary -#define SIMDUTF_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) -#define SIMDUTF_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) - -#define SIMDUTF_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) - -#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) - - #define simdutf_really_inline __forceinline - #define simdutf_never_inline __declspec(noinline) - - #define simdutf_unused - #define simdutf_warn_unused - - #ifndef simdutf_likely - #define simdutf_likely(x) x - #endif - #ifndef simdutf_unlikely - #define simdutf_unlikely(x) x - #endif - - #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning( push )) - #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 )) - #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER )) - // Get rid of Intellisense-only warnings (Code Analysis) - // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910). - #ifdef __has_include - #if __has_include() - #include - #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) - #endif - #endif - - #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS - #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS - #endif - - #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) - #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING - #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning( pop )) - -#else // SIMDUTF_REGULAR_VISUAL_STUDIO - - #define simdutf_really_inline inline __attribute__((always_inline)) - #define simdutf_never_inline inline __attribute__((noinline)) - - #define simdutf_unused __attribute__((unused)) - #define simdutf_warn_unused __attribute__((warn_unused_result)) - - #ifndef simdutf_likely - #define simdutf_likely(x) __builtin_expect(!!(x), 1) - #endif - #ifndef simdutf_unlikely - #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) - #endif - - #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") - // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary - #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS SIMDUTF_PUSH_DISABLE_WARNINGS \ - SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ - SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) - #define SIMDUTF_PRAGMA(P) _Pragma(#P) - #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) - #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) - #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) - #else - #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS - #endif - #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) - #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) - #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") - - - -#endif // MSC_VER - -#if defined(SIMDUTF_VISUAL_STUDIO) - /** - * It does not matter here whether you are using - * the regular visual studio or clang under visual - * studio. - */ - #if SIMDUTF_USING_LIBRARY - #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) - #else - #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) - #endif -#else - #define SIMDUTF_DLLIMPORTEXPORT -#endif - -/// If EXPR is an error, returns it. -#define SIMDUTF_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } } - - -#endif // SIMDUTF_COMMON_DEFS_H -/* end file include/simdutf/common_defs.h */ -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/encoding_types.h -/* begin file include/simdutf/encoding_types.h */ -#include - -namespace simdutf { - -enum encoding_type { - UTF16_LE, // BOM 0xff 0xfe - UTF16_BE, // BOM 0xfe 0xff - UTF32_LE, // BOM 0xff 0xfe 0x00 0x00 - UTF32_BE, // BOM 0x00 0x00 0xfe 0xff - UTF8, // BOM 0xef 0xbb 0xbf - unspecified -}; - -std::string to_string(encoding_type bom); - -// Note that BOM for UTF8 is discouraged. -namespace BOM { - -/** - * Checks for a BOM. If not, returns unspecified - * @param input the string to process - * @param length the length of the string in words - * @return the corresponding encoding - */ - -encoding_type check_bom(const uint8_t* byte, size_t length); -encoding_type check_bom(const char* byte, size_t length); -/** - * Returns the size, in bytes, of the BOM for a given encoding type. - * Note that UTF8 BOM are discouraged. - * @param bom the encoding type - * @return the size in bytes of the corresponding BOM - */ -size_t bom_byte_size(encoding_type bom); - -} // BOM namespace -} // simdutf namespace -/* end file include/simdutf/encoding_types.h */ - -SIMDUTF_PUSH_DISABLE_WARNINGS -SIMDUTF_DISABLE_UNDESIRED_WARNINGS - -// Public API -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/simdutf_version.h -/* begin file include/simdutf/simdutf_version.h */ -// /include/simdutf/simdutf_version.h automatically generated by release.py, -// do not change by hand -#ifndef SIMDUTF_SIMDUTF_VERSION_H -#define SIMDUTF_SIMDUTF_VERSION_H - -/** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION 1.0.1 - -namespace simdutf { -enum { - /** - * The major version (MAJOR.minor.revision) of simdutf being used. - */ - SIMDUTF_VERSION_MAJOR = 1, - /** - * The minor version (major.MINOR.revision) of simdutf being used. - */ - SIMDUTF_VERSION_MINOR = 0, - /** - * The revision (major.minor.REVISION) of simdutf being used. - */ - SIMDUTF_VERSION_REVISION = 0 -}; -} // namespace simdutf - -#endif // SIMDUTF_SIMDUTF_VERSION_H -/* end file include/simdutf/simdutf_version.h */ -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/implementation.h -/* begin file include/simdutf/implementation.h */ -#ifndef SIMDUTF_IMPLEMENTATION_H -#define SIMDUTF_IMPLEMENTATION_H -#include -#if !defined(SIMDUTF_NO_THREADS) -#include -#endif -#include -// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/internal/isadetection.h -/* begin file include/simdutf/internal/isadetection.h */ -/* From -https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h -Highly modified. - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef SIMDutf_INTERNAL_ISADETECTION_H -#define SIMDutf_INTERNAL_ISADETECTION_H - -#include -#include -#if defined(_MSC_VER) -#include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) -#include -#endif - -namespace simdutf { -namespace internal { - - -enum instruction_set { - DEFAULT = 0x0, - NEON = 0x1, - AVX2 = 0x4, - SSE42 = 0x8, - PCLMULQDQ = 0x10, - BMI1 = 0x20, - BMI2 = 0x40, - ALTIVEC = 0x80, - AVX512F = 0x100, - AVX512BW = 0x200, - AVX512DQ = 0x400 -}; - -#if defined(__PPC64__) - -static inline uint32_t detect_supported_architectures() { - return instruction_set::ALTIVEC; -} - -#elif defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 - -#if defined(__ARM_NEON) - -static inline uint32_t detect_supported_architectures() { - return instruction_set::NEON; -} - -#else // ARM without NEON - -static inline uint32_t detect_supported_architectures() { - return instruction_set::DEFAULT; -} - -#endif - -#elif defined(__x86_64__) || defined(_M_AMD64) // x64 - - -namespace { -namespace cpuid_bit { - // Can be found on Intel ISA Reference for CPUID - - // EAX = 0x01 - constexpr uint32_t pclmulqdq = uint32_t(1) << 1; ///< @private bit 1 of ECX for EAX=0x1 - constexpr uint32_t sse42 = uint32_t(1) << 20; ///< @private bit 20 of ECX for EAX=0x1 - - // EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf) - // See: "Table 3-8. Information Returned by CPUID Instruction" - namespace ebx { - constexpr uint32_t bmi1 = uint32_t(1) << 3; - constexpr uint32_t avx2 = uint32_t(1) << 5; - constexpr uint32_t bmi2 = uint32_t(1) << 8; - constexpr uint32_t avx512f = uint32_t(1) << 16; - constexpr uint32_t avx512dq = uint32_t(1) << 17; - constexpr uint32_t avx512ifma = uint32_t(1) << 21; - constexpr uint32_t avx512cd = uint32_t(1) << 28; - constexpr uint32_t avx512bw = uint32_t(1) << 30; - constexpr uint32_t avx512vl = uint32_t(1) << 31; - } - - namespace ecx { - constexpr uint32_t avx512vbmi = uint32_t(1) << 1; - constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6; - constexpr uint32_t avx512vnni = uint32_t(1) << 11; - constexpr uint32_t avx512bitalg = uint32_t(1) << 12; - constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14; - } - namespace edx { - constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8; - } - } -} - - - -static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, - uint32_t *edx) { -#if defined(_MSC_VER) - int cpu_info[4]; - __cpuid(cpu_info, *eax); - *eax = cpu_info[0]; - *ebx = cpu_info[1]; - *ecx = cpu_info[2]; - *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) - uint32_t level = *eax; - __get_cpuid(level, eax, ebx, ecx, edx); -#else - uint32_t a = *eax, b, c = *ecx, d; - asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); - *eax = a; - *ebx = b; - *ecx = c; - *edx = d; -#endif -} - -static inline uint32_t detect_supported_architectures() { - uint32_t eax; - uint32_t ebx = 0; - uint32_t ecx = 0; - uint32_t edx = 0; - uint32_t host_isa = 0x0; - - // EBX for EAX=0x1 - eax = 0x1; - cpuid(&eax, &ebx, &ecx, &edx); - - if (ecx & cpuid_bit::sse42) { - host_isa |= instruction_set::SSE42; - } - - if (ecx & cpuid_bit::pclmulqdq) { - host_isa |= instruction_set::PCLMULQDQ; - } - - // ECX for EAX=0x7 - eax = 0x7; - ecx = 0x0; // Sub-leaf = 0 - cpuid(&eax, &ebx, &ecx, &edx); - if (ebx & cpuid_bit::ebx::avx2) { - host_isa |= instruction_set::AVX2; - } - if (ebx & cpuid_bit::ebx::bmi1) { - host_isa |= instruction_set::BMI1; - } - if (ebx & cpuid_bit::ebx::bmi2) { - host_isa |= instruction_set::BMI2; - } - if (ebx & cpuid_bit::ebx::avx512f) { - host_isa |= instruction_set::AVX512F; - } - if (ebx & cpuid_bit::ebx::avx512bw) { - host_isa |= instruction_set::AVX512BW; - } - if (ebx & cpuid_bit::ebx::avx512dq) { - host_isa |= instruction_set::AVX512DQ; - } - - return host_isa; -} -#else // fallback - - -static inline uint32_t detect_supported_architectures() { - return instruction_set::DEFAULT; -} - - -#endif // end SIMD extension detection code - -} // namespace internal -} // namespace simdutf - -#endif // SIMDutf_INTERNAL_ISADETECTION_H -/* end file include/simdutf/internal/isadetection.h */ - - -namespace simdutf { - -/** - * Autodetect the encoding of the input. - * - * @param input the string to analyze. - * @param length the length of the string in bytes. - * @return the detected encoding type - */ -simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * input, size_t length) noexcept; -simdutf_really_inline simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const uint8_t * input, size_t length) noexcept { - return autodetect_encoding(reinterpret_cast(input), length); -} - - -/** - * Validate the UTF-8 string. - * - * Overridden by each implementation. - * - * @param buf the UTF-8 string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid UTF-8. - */ -simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; - -/** - * Validate the UTF-16LE string. - * - * Overridden by each implementation. - * - * This function is not BOM-aware. - * - * @param buf the UTF-16LE string to validate. - * @param len the length of the string in number of 2-byte words (char16_t). - * @return true if and only if the string is valid UTF-16LE. - */ -simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept; - -/** - * Convert possibly broken UTF-8 string into UTF-16LE string. - * - * During the conversion also validation of the input string is done. - * This function is suitable to work with inputs from untrusted sources. - * - * @param input the UTF-8 string to convert - * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return the number of written char16_t; 0 if the input was not valid UTF-8 string - */ -simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) noexcept; - -/** - * Convert valid UTF-8 string into UTF-16LE string. - * - * This function assumes that the input string is valid UTF-8. - * - * @param input the UTF-8 string to convert - * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return the number of written char16_t - */ -simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept; - -/** - * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format. - * - * This function does not validate the input. - * - * This function is not BOM-aware. - * - * @param input the UTF-8 string to process - * @param length the length of the string in bytes - * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE - */ -simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept; - -/** - * Convert possibly broken UTF-16LE string into UTF-8 string. - * - * During the conversion also validation of the input string is done. - * This function is suitable to work with inputs from untrusted sources. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @param utf8_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-16LE string - */ -simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; - -/** - * Convert valid UTF-16LE string into UTF-8 string. - * - * This function assumes that the input string is valid UTF-16LE. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @param utf8_buffer the pointer to buffer that can hold the conversion result - * @return number of written words; 0 if conversion is not possible - */ -simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; - -/** - * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format. - * - * This function does not validate the input. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16LE string as UTF-8 - */ -simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept; - -/** - * Count the number of code points (characters) in the string assuming that - * it is valid. - * - * This function assumes that the input string is valid UTF-16LE. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to process - * @param length the length of the string in 2-byte words (char16_t) - * @return number of code points - */ -simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept; - -/** - * Count the number of code points (characters) in the string assuming that - * it is valid. - * - * This function assumes that the input string is valid UTF-8. - * - * @param input the UTF-8 string to process - * @param length the length of the string in bytes - * @return number of code points - */ -simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept; - -/** - * An implementation of simdutf for a particular CPU architecture. - * - * Also used to maintain the currently active implementation. The active implementation is - * automatically initialized on first use to the most advanced implementation supported by the host. - */ -class implementation { -public: - - /** - * The name of this implementation. - * - * const implementation *impl = simdutf::active_implementation; - * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" - */ - virtual const std::string &name() const { return _name; } - - /** - * The description of this implementation. - * - * const implementation *impl = simdutf::active_implementation; - * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" - */ - virtual const std::string &description() const { return _description; } - - /** - * The instruction sets this implementation is compiled against - * and the current CPU match. This function may poll the current CPU/system - * and should therefore not be called too often if performance is a concern. - * - * - * @return true if the implementation can be safely used on the current system (determined at runtime) - */ - bool supported_by_runtime_system() const; - - /** - * This function will try to detect the encoding - * @param input the string to identify - * @param length the length of the string in bytes. - * @return the encoding type detected - */ - virtual encoding_type autodetect_encoding(const char * input, size_t length) const noexcept; - - /** - * @private For internal implementation use - * - * The instruction sets this implementation is compiled against. - * - * @return a mask of all required `internal::instruction_set::` values - */ - virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; - - - /** - * Validate the UTF-8 string. - * - * Overridden by each implementation. - * - * @param buf the UTF-8 string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid UTF-8. - */ - simdutf_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; - - /** - * Validate the UTF-16LE string. - * - * Overridden by each implementation. - * - * This function is not BOM-aware. - * - * @param buf the UTF-16LE string to validate. - * @param len the length of the string in number of 2-byte words (char16_t). - * @return true if and only if the string is valid UTF-16LE. - */ - simdutf_warn_unused virtual bool validate_utf16(const char16_t *buf, size_t len) const noexcept = 0; - - /** - * Convert possibly broken UTF-8 string into UTF-16LE string. - * - * During the conversion also validation of the input string is done. - * This function is suitable to work with inputs from untrusted sources. - * - * @param input the UTF-8 string to convert - * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return the number of written char16_t; 0 if the input was not valid UTF-8 string - */ - simdutf_warn_unused virtual size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) const noexcept = 0; - - /** - * Convert valid UTF-8 string into UTF-16LE string. - * - * This function assumes that the input string is valid UTF-8. - * - * @param input the UTF-8 string to convert - * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return the number of written char16_t - */ - simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; - - /** - * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format. - * - * This function does not validate the input. - * - * @param input the UTF-8 string to process - * @param length the length of the string in bytes - * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE - */ - simdutf_warn_unused virtual size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept = 0; - - /** - * Convert possibly broken UTF-16LE string into UTF-8 string. - * - * During the conversion also validation of the input string is done. - * This function is suitable to work with inputs from untrusted sources. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @param utf8_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-16LE string - */ - simdutf_warn_unused virtual size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0; - - /** - * Convert valid UTF-16LE string into UTF-8 string. - * - * This function assumes that the input string is valid UTF-16LE. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @param utf8_buffer the pointer to buffer that can hold the conversion result - * @return number of written words; 0 if conversion is not possible - */ - simdutf_warn_unused virtual size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0; - - /** - * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format. - * - * This function does not validate the input. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to convert - * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16LE string as UTF-8 - */ - simdutf_warn_unused virtual size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept = 0; - - /** - * Count the number of code points (characters) in the string assuming that - * it is valid. - * - * This function assumes that the input string is valid UTF-16LE. - * - * This function is not BOM-aware. - * - * @param input the UTF-16LE string to process - * @param length the length of the string in 2-byte words (char16_t) - * @return number of code points - */ - simdutf_warn_unused virtual size_t count_utf16(const char16_t * input, size_t length) const noexcept = 0; - - /** - * Count the number of code points (characters) in the string assuming that - * it is valid. - * - * This function assumes that the input string is valid UTF-8. - * - * @param input the UTF-8 string to process - * @param length the length of the string in bytes - * @return number of code points - */ - simdutf_warn_unused virtual size_t count_utf8(const char * input, size_t length) const noexcept = 0; - - - -protected: - /** @private Construct an implementation with the given name and description. For subclasses. */ - simdutf_really_inline implementation( - std::string name, - std::string description, - uint32_t required_instruction_sets - ) : - _name(name), - _description(description), - _required_instruction_sets(required_instruction_sets) - { - } - virtual ~implementation()=default; - -private: - /** - * The name of this implementation. - */ - const std::string _name; - - /** - * The description of this implementation. - */ - const std::string _description; - - /** - * Instruction sets required for this implementation. - */ - const uint32_t _required_instruction_sets; -}; - -/** @private */ -namespace internal { - -/** - * The list of available implementations compiled into simdutf. - */ -class available_implementation_list { -public: - /** Get the list of available implementations compiled into simdutf */ - simdutf_really_inline available_implementation_list() {} - /** Number of implementations */ - size_t size() const noexcept; - /** STL const begin() iterator */ - const implementation * const *begin() const noexcept; - /** STL const end() iterator */ - const implementation * const *end() const noexcept; - - /** - * Get the implementation with the given name. - * - * Case sensitive. - * - * const implementation *impl = simdutf::available_implementations["westmere"]; - * if (!impl) { exit(1); } - * if (!imp->supported_by_runtime_system()) { exit(1); } - * simdutf::active_implementation = impl; - * - * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" - * @return the implementation, or nullptr if the parse failed. - */ - const implementation * operator[](const std::string &name) const noexcept { - for (const implementation * impl : *this) { - if (impl->name() == name) { return impl; } - } - return nullptr; - } - - /** - * Detect the most advanced implementation supported by the current host. - * - * This is used to initialize the implementation on startup. - * - * const implementation *impl = simdutf::available_implementation::detect_best_supported(); - * simdutf::active_implementation = impl; - * - * @return the most advanced supported implementation for the current host, or an - * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported - * implementation. Will never return nullptr. - */ - const implementation *detect_best_supported() const noexcept; -}; - -template -class atomic_ptr { -public: - atomic_ptr(T *_ptr) : ptr{_ptr} {} - -#if defined(SIMDUTF_NO_THREADS) - operator const T*() const { return ptr; } - const T& operator*() const { return *ptr; } - const T* operator->() const { return ptr; } - - operator T*() { return ptr; } - T& operator*() { return *ptr; } - T* operator->() { return ptr; } - atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; } - -#else - operator const T*() const { return ptr.load(); } - const T& operator*() const { return *ptr; } - const T* operator->() const { return ptr.load(); } - - operator T*() { return ptr.load(); } - T& operator*() { return *ptr; } - T* operator->() { return ptr.load(); } - atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; } - -#endif - -private: -#if defined(SIMDUTF_NO_THREADS) - T* ptr; -#else - std::atomic ptr; -#endif -}; - -} // namespace internal - -/** - * The list of available implementations compiled into simdutf. - */ -extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations; - -/** - * The active implementation. - * - * Automatically initialized on first use to the most advanced implementation supported by this hardware. - */ -extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr active_implementation; - -} // namespace simdutf - -#endif // SIMDUTF_IMPLEMENTATION_H -/* end file include/simdutf/implementation.h */ - - -// Implementation-internal files (must be included before the implementations themselves, to keep -// amalgamation working--otherwise, the first time a file is included, it might be put inside the -// #ifdef SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other implementations can't -// compile unless that implementation is turned on). - - -SIMDUTF_POP_DISABLE_WARNINGS - -#endif // SIMDUTF_H -/* end file include/simdutf.h */ +/* auto-generated on 2026-01-13 09:03:21 +0100. Do not edit! */ +/* begin file include/simdutf.h */ +#ifndef SIMDUTF_H +#define SIMDUTF_H +#include + +/* begin file include/simdutf/compiler_check.h */ +#ifndef SIMDUTF_COMPILER_CHECK_H +#define SIMDUTF_COMPILER_CHECK_H + +#ifndef __cplusplus + #error simdutf requires a C++ compiler +#endif + +#ifndef SIMDUTF_CPLUSPLUS + #if defined(_MSVC_LANG) && !defined(__clang__) + #define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) + #else + #define SIMDUTF_CPLUSPLUS __cplusplus + #endif +#endif + +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + +// C++ 23 +#if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) + #define SIMDUTF_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDUTF_CPLUSPLUS20) && (SIMDUTF_CPLUSPLUS >= 202002L) + #define SIMDUTF_CPLUSPLUS20 1 +#endif + +// C++ 17 +#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) + #define SIMDUTF_CPLUSPLUS17 1 +#endif + +// C++ 14 +#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) + #define SIMDUTF_CPLUSPLUS14 1 +#endif + +// C++ 11 +#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) + #define SIMDUTF_CPLUSPLUS11 1 +#endif + +#ifndef SIMDUTF_CPLUSPLUS11 + #error simdutf requires a compiler compliant with the C++11 standard +#endif + +#endif // SIMDUTF_COMPILER_CHECK_H +/* end file include/simdutf/compiler_check.h */ +/* begin file include/simdutf/common_defs.h */ +#ifndef SIMDUTF_COMMON_DEFS_H +#define SIMDUTF_COMMON_DEFS_H + +/* begin file include/simdutf/portability.h */ +#ifndef SIMDUTF_PORTABILITY_H +#define SIMDUTF_PORTABILITY_H + + +#include +#include +#include +#include +#ifndef _WIN32 + // strcasecmp, strncasecmp + #include +#endif + +#if defined(__apple_build_version__) + #if __apple_build_version__ < 14000000 + #define SIMDUTF_SPAN_DISABLED \ + 1 // apple-clang/13 doesn't support std::convertible_to + #endif +#endif + +#if SIMDUTF_CPLUSPLUS20 + #include + #if __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L && \ + !defined(SIMDUTF_SPAN_DISABLED) + #define SIMDUTF_SPAN 1 + #endif // __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L + #if __cpp_lib_atomic_ref >= 201806L + #define SIMDUTF_ATOMIC_REF 1 + #endif // __cpp_lib_atomic_ref + #if __has_cpp_attribute(maybe_unused) >= 201603L + #define SIMDUTF_MAYBE_UNUSED_AVAILABLE 1 + #endif // __has_cpp_attribute(maybe_unused) >= 201603L +#endif + +/** + * We want to check that it is actually a little endian system at + * compile-time. + */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) + #define SIMDUTF_IS_BIG_ENDIAN 0 +#else + #if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ + #include + #elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) + #include + #else // defined(__APPLE__) || defined(__FreeBSD__) + + #ifdef __has_include + #if __has_include() + #include + #endif //__has_include() + #endif //__has_include + + #endif // defined(__APPLE__) || defined(__FreeBSD__) + + #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN 0 + #endif + + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 0 + #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 1 + #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ + +/** + * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined. + */ + +#ifdef _MSC_VER + #define SIMDUTF_VISUAL_STUDIO 1 + /** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + * + * Under clang for Windows, we enable: + * * target pragmas so that part and only part of the + * code gets compiled for advanced instructions. + * + */ + #ifdef __clang__ + // clang under visual studio + #define SIMDUTF_CLANG_VISUAL_STUDIO 1 + #else + // just regular visual studio (best guess) + #define SIMDUTF_REGULAR_VISUAL_STUDIO 1 + #endif // __clang__ +#endif // _MSC_VER + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + // https://en.wikipedia.org/wiki/C_alternative_tokens + // This header should have no effect, except maybe + // under Visual Studio. + #include +#endif + +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) + #define SIMDUTF_IS_X86_64 1 +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + #define SIMDUTF_IS_ARM64 1 +#elif defined(__PPC64__) || defined(_M_PPC64) + #if defined(__VEC__) && defined(__ALTIVEC__) + #define SIMDUTF_IS_PPC64 1 + #endif +#elif defined(__s390__) +// s390 IBM system. Big endian. +#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64 + // RISC-V 64-bit + #define SIMDUTF_IS_RISCV64 1 + + // #if __riscv_v_intrinsic >= 1000000 + // #define SIMDUTF_HAS_RVV_INTRINSICS 1 + // #define SIMDUTF_HAS_RVV_TARGET_REGION 1 + // #elif ... + // Check for special compiler versions that implement pre v1.0 intrinsics + #if __riscv_v_intrinsic >= 11000 + #define SIMDUTF_HAS_RVV_INTRINSICS 1 + #endif + + #define SIMDUTF_HAS_ZVBB_INTRINSICS \ + 0 // there is currently no way to detect this + + #if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector && \ + __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 + // RISC-V V extension + #define SIMDUTF_IS_RVV 1 + #if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000 + // RISC-V Vector Basic Bit-manipulation + #define SIMDUTF_IS_ZVBB 1 + #endif + #endif + +#elif defined(__loongarch_lp64) + #if defined(__loongarch_sx) && defined(__loongarch_asx) + #define SIMDUTF_IS_LSX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both + #elif defined(__loongarch_sx) + #define SIMDUTF_IS_LSX 1 + #endif +#else + // The simdutf library is designed + // for 64-bit processors and it seems that you are not + // compiling for a known 64-bit platform. Please + // use a 64-bit target such as x64 or 64-bit ARM for best performance. + #define SIMDUTF_IS_32BITS 1 + + // We do not support 32-bit platforms, but it can be + // handy to identify them. + #if defined(_M_IX86) || defined(__i386__) + #define SIMDUTF_IS_X86_32BITS 1 + #elif defined(__arm__) || defined(_M_ARM) + #define SIMDUTF_IS_ARM_32BITS 1 + #elif defined(__PPC__) || defined(_M_PPC) + #define SIMDUTF_IS_PPC_32BITS 1 + #endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) + +#ifdef SIMDUTF_IS_32BITS + #ifndef SIMDUTF_NO_PORTABILITY_WARNING + // In the future, we may want to warn users of 32-bit systems that + // the simdutf does not support accelerated kernels for such systems. + #endif // SIMDUTF_NO_PORTABILITY_WARNING +#endif // SIMDUTF_IS_32BITS + +// this is almost standard? +#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a +#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The simdutf users should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select +// SIMDUTF_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + #ifdef __clang__ + // clang does not have GCC push pop + // warning: clang attribute push can't be used within a namespace in clang + // up til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be + // *outside* of a namespace. + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma(SIMDUTF_STRINGIFY(clang attribute push( \ + __attribute__((target(T))), apply_to = function))) + #define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") + #elif defined(__GNUC__) + // GCC is easier + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T))) + #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") + #endif // clang then gcc + +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + +// Default target region macros don't do anything. +#ifndef SIMDUTF_TARGET_REGION + #define SIMDUTF_TARGET_REGION(T) + #define SIMDUTF_UNTARGET_REGION +#endif + +// Is threading enabled? +#if defined(_REENTRANT) || defined(_MT) + #ifndef SIMDUTF_THREADS_ENABLED + #define SIMDUTF_THREADS_ENABLED + #endif +#endif + +// workaround for large stack sizes under -O0. +// https://github.com/simdutf/simdutf/issues/691 +#ifdef __APPLE__ + #ifndef __OPTIMIZE__ + // Apple systems have small stack sizes in secondary threads. + // Lack of compiler optimization may generate high stack usage. + // Users may want to disable threads for safety, but only when + // in debug mode which we detect by the fact that the __OPTIMIZE__ + // macro is not defined. + #undef SIMDUTF_THREADS_ENABLED + #endif +#endif + +#ifdef SIMDUTF_VISUAL_STUDIO + // This is one case where we do not distinguish between + // regular visual studio and clang under visual studio. + // clang under Windows has _stricmp (like visual studio) but not strcasecmp + // (as clang normally has) + #define simdutf_strcasecmp _stricmp + #define simdutf_strncasecmp _strnicmp +#else + // The strcasecmp, strncasecmp, and strcasestr functions do not work with + // multibyte strings (e.g. UTF-8). So they are only useful for ASCII in our + // context. + // https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings + #define simdutf_strcasecmp strcasecmp + #define simdutf_strncasecmp strncasecmp +#endif + +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ >= 11 + #define SIMDUTF_GCC11ORMORE 1 + #endif // __GNUC__ >= 11 + #if __GNUC__ == 10 + #define SIMDUTF_GCC10 1 + #endif // __GNUC__ == 10 + #if __GNUC__ < 10 + #define SIMDUTF_GCC9OROLDER 1 + #endif // __GNUC__ == 10 +#endif // defined(__GNUC__) && !defined(__clang__) + +#endif // SIMDUTF_PORTABILITY_H +/* end file include/simdutf/portability.h */ +/* begin file include/simdutf/avx512.h */ +#ifndef SIMDUTF_AVX512_H_ +#define SIMDUTF_AVX512_H_ + +/* + It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS. + + All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`, + where a feature is a code name for extensions. + + Please see the listing below to find which are supported. +*/ + +#ifndef SIMDUTF_HAS_AVX512F + #if defined(__AVX512F__) && __AVX512F__ == 1 + #define SIMDUTF_HAS_AVX512F 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512DQ + #if defined(__AVX512DQ__) && __AVX512DQ__ == 1 + #define SIMDUTF_HAS_AVX512DQ 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512IFMA + #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1 + #define SIMDUTF_HAS_AVX512IFMA 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512CD + #if defined(__AVX512CD__) && __AVX512CD__ == 1 + #define SIMDUTF_HAS_AVX512CD 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BW + #if defined(__AVX512BW__) && __AVX512BW__ == 1 + #define SIMDUTF_HAS_AVX512BW 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VL + #if defined(__AVX512VL__) && __AVX512VL__ == 1 + #define SIMDUTF_HAS_AVX512VL 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI + #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1 + #define SIMDUTF_HAS_AVX512VBMI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI2 + #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1 + #define SIMDUTF_HAS_AVX512VBMI2 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VNNI + #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1 + #define SIMDUTF_HAS_AVX512VNNI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BITALG + #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1 + #define SIMDUTF_HAS_AVX512BITALG 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ + #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1 + #define SIMDUTF_HAS_AVX512VPOPCNTDQ 1 + #endif +#endif + +#endif // SIMDUTF_AVX512_H_ +/* end file include/simdutf/avx512.h */ + +// Sometimes logging is useful, but we want it disabled by default +// and free of any logging code in release builds. +#ifdef SIMDUTF_LOGGING + #include + #define simdutf_log(msg) \ + std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; + #define simdutf_log_assert(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cerr << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; \ + std::abort(); \ + } \ + } while (0) +#else + #define simdutf_log(msg) + #define simdutf_log_assert(cond, msg) +#endif + +#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) + #define SIMDUTF_DEPRECATED __declspec(deprecated) + + #define simdutf_really_inline __forceinline // really inline in release mode + #define simdutf_always_inline __forceinline // always inline, no matter what + #define simdutf_never_inline __declspec(noinline) + + #define simdutf_unused + #define simdutf_warn_unused + + #ifndef simdutf_likely + #define simdutf_likely(x) x + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) x + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning(push)) + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0)) + #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) \ + __pragma(warning(disable : WARNING_NUMBER)) + // Get rid of Intellisense-only warnings (Code Analysis) + // Though __has_include is C++17, it is supported in Visual Studio 2017 or + // better (_MSC_VER>=1910). + #ifdef __has_include + #if __has_include() + #include + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) + #endif + #endif + + #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING + #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning(pop)) + #define SIMDUTF_DISABLE_UNUSED_WARNING +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + #if defined(__OPTIMIZE__) || defined(NDEBUG) + #define simdutf_really_inline inline __attribute__((always_inline)) + #else + #define simdutf_really_inline inline + #endif + #define simdutf_always_inline \ + inline __attribute__((always_inline)) // always inline, no matter what + #define SIMDUTF_DEPRECATED __attribute__((deprecated)) + #define simdutf_never_inline inline __attribute__((noinline)) + + #define simdutf_unused __attribute__((unused)) + #define simdutf_warn_unused __attribute__((warn_unused_result)) + + #ifndef simdutf_likely + #define simdutf_likely(x) __builtin_expect(!!(x), 1) + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) + #endif + // clang-format off + #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") + // gcc doesn't seem to disable all warnings with all and extra, add warnings + // here as necessary + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) + #define SIMDUTF_PRAGMA(P) _Pragma(#P) + #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) \ + SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) + #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) + #else + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + #define SIMDUTF_DISABLE_DEPRECATED_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) + #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") + #define SIMDUTF_DISABLE_UNUSED_WARNING \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-function) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-const-variable) + // clang-format on + +#endif // MSC_VER + +// Conditional constexpr macro: expands to constexpr for C++17+, empty otherwise +#if SIMDUTF_CPLUSPLUS17 + #define simdutf_constexpr constexpr +#else + #define simdutf_constexpr +#endif + +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + +#ifndef SIMDUTF_DLLIMPORTEXPORT + #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio + /** + * Windows users need to do some extra work when building + * or using a dynamic library (DLL). When building, we need + * to set SIMDUTF_DLLIMPORTEXPORT to __declspec(dllexport). + * When *using* the DLL, the user needs to set + * SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport). + * + * Static libraries not need require such work. + * + * It does not matter here whether you are using + * the regular visual studio or clang under visual + * studio, you still need to handle these issues. + * + * Non-Windows systems do not have this complexity. + */ + #if SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY + + // We set SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL + // under Windows. It should never happen that both + // SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY and + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY are set. + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) + #elif SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY + // Windows user who call a dynamic library should set + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY to 1. + + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) + #else + // We assume by default static linkage + #define SIMDUTF_DLLIMPORTEXPORT + #endif + #else // defined(SIMDUTF_VISUAL_STUDIO) + // Non-Windows systems do not have this complexity. + #define SIMDUTF_DLLIMPORTEXPORT + #endif // defined(SIMDUTF_VISUAL_STUDIO) +#endif + +#if SIMDUTF_MAYBE_UNUSED_AVAILABLE + #define simdutf_maybe_unused [[maybe_unused]] +#else + #define simdutf_maybe_unused +#endif + +#endif // SIMDUTF_COMMON_DEFS_H +/* end file include/simdutf/common_defs.h */ +/* begin file include/simdutf/encoding_types.h */ +#ifndef SIMDUTF_ENCODING_TYPES_H +#define SIMDUTF_ENCODING_TYPES_H +#include + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include +#endif + +namespace simdutf { + +enum encoding_type { + UTF8 = 1, // BOM 0xef 0xbb 0xbf + UTF16_LE = 2, // BOM 0xff 0xfe + UTF16_BE = 4, // BOM 0xfe 0xff + UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00 + UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff + Latin1 = 32, + + unspecified = 0 +}; + +#ifndef SIMDUTF_IS_BIG_ENDIAN + #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." +#endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = +#if SIMDUTF_IS_BIG_ENDIAN + BIG +#else + LITTLE +#endif +}; + +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; +} + +simdutf_warn_unused std::string to_string(encoding_type bom); + +// Note that BOM for UTF8 is discouraged. +namespace BOM { + +/** + * Checks for a BOM. If not, returns unspecified + * @param input the string to process + * @param length the length of the string in code units + * @return the corresponding encoding + */ + +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); +/** + * Returns the size, in bytes, of the BOM for a given encoding type. + * Note that UTF8 BOM are discouraged. + * @param bom the encoding type + * @return the size in bytes of the corresponding BOM + */ +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); + +} // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + +} // namespace simdutf +#endif +/* end file include/simdutf/encoding_types.h */ +/* begin file include/simdutf/error.h */ +#ifndef SIMDUTF_ERROR_H +#define SIMDUTF_ERROR_H +namespace simdutf { + +enum error_code { + SUCCESS = 0, + HEADER_BITS, // Any byte must have fewer than 5 header bits. + TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, + // where N is the UTF-8 character length This is also the error + // when the input is truncated. + TOO_LONG, // We either have too many consecutive continuation bytes or the + // string starts with a continuation byte. + OVERLONG, // The decoded character must be above U+7F for two-byte characters, + // U+7FF for three-byte characters, and U+FFFF for four-byte + // characters. + TOO_LARGE, // The decoded character must be less than or equal to + // U+10FFFF,less than or equal than U+7F for ASCII OR less than + // equal than U+FF for Latin1 + SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or + // UTF-32) + // OR + // a high surrogate must be followed by a low surrogate + // and a low surrogate must be preceded by a high surrogate + // (UTF-16) + // OR + // there must be no surrogate at all and one is + // found (Latin1 functions) + // OR + // *specifically* for the function + // utf8_length_from_utf16_with_replacement, a surrogate (whether + // in error or not) has been found (I.e., whether we are in the + // Basic Multilingual Plane or not). + INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid + // base64 string. This may include a misplaced + // padding character ('='). + BASE64_INPUT_REMAINDER, // The base64 input terminates with a single + // character, excluding padding (=). It is also used + // in strict mode when padding is not adequate. + BASE64_EXTRA_BITS, // The base64 input terminates with non-zero + // padding bits. + OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. + OTHER // Not related to validation/transcoding. +}; +#if SIMDUTF_CPLUSPLUS17 +inline std::string_view error_to_string(error_code code) noexcept { + switch (code) { + case SUCCESS: + return "SUCCESS"; + case HEADER_BITS: + return "HEADER_BITS"; + case TOO_SHORT: + return "TOO_SHORT"; + case TOO_LONG: + return "TOO_LONG"; + case OVERLONG: + return "OVERLONG"; + case TOO_LARGE: + return "TOO_LARGE"; + case SURROGATE: + return "SURROGATE"; + case INVALID_BASE64_CHARACTER: + return "INVALID_BASE64_CHARACTER"; + case BASE64_INPUT_REMAINDER: + return "BASE64_INPUT_REMAINDER"; + case BASE64_EXTRA_BITS: + return "BASE64_EXTRA_BITS"; + case OUTPUT_BUFFER_TOO_SMALL: + return "OUTPUT_BUFFER_TOO_SMALL"; + default: + return "OTHER"; + } +} +#endif + +struct result { + error_code error; + size_t count; // In case of error, indicates the position of the error. In + // case of success, indicates the number of code units + // validated/written. + + simdutf_really_inline simdutf_constexpr23 result() noexcept + : error{error_code::SUCCESS}, count{0} {} + + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept + : error{err}, count{pos} {} + + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { + return error == error_code::SUCCESS; + } + + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { + return error != error_code::SUCCESS; + } +}; + +struct full_result { + error_code error; + size_t input_count; + size_t output_count; + bool padding_error = false; // true if the error is due to padding, only + // meaningful when error is not SUCCESS + + simdutf_really_inline simdutf_constexpr23 full_result() noexcept + : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} + + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out} {} + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out}, + padding_error{padding_err} {} + + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { + if (error == error_code::SUCCESS) { + return result{error, output_count}; + } else { + return result{error, input_count}; + } + } +}; + +} // namespace simdutf +#endif +/* end file include/simdutf/error.h */ + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + +// Public API +/* begin file include/simdutf/simdutf_version.h */ +// /include/simdutf/simdutf_version.h automatically generated by release.py, +// do not change by hand +#ifndef SIMDUTF_SIMDUTF_VERSION_H +#define SIMDUTF_SIMDUTF_VERSION_H + +/** The version of simdutf being used (major.minor.revision) */ +#define SIMDUTF_VERSION "8.0.0" + +namespace simdutf { +enum { + /** + * The major version (MAJOR.minor.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MAJOR = 8, + /** + * The minor version (major.MINOR.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MINOR = 0, + /** + * The revision (major.minor.REVISION) of simdutf being used. + */ + SIMDUTF_VERSION_REVISION = 0 +}; +} // namespace simdutf + +#endif // SIMDUTF_SIMDUTF_VERSION_H +/* end file include/simdutf/simdutf_version.h */ +/* begin file include/simdutf/implementation.h */ +#ifndef SIMDUTF_IMPLEMENTATION_H +#define SIMDUTF_IMPLEMENTATION_H +#if !defined(SIMDUTF_NO_THREADS) + #include +#endif +#include +#ifdef SIMDUTF_INTERNAL_TESTS + #include +#endif +/* begin file include/simdutf/internal/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDutf_INTERNAL_ISADETECTION_H +#define SIMDutf_INTERNAL_ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) + #include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + #include +#endif + + +// RISC-V ISA detection utilities +#if SIMDUTF_IS_RISCV64 && defined(__linux__) + #include // for syscall +// We define these ourselves, for backwards compatibility +struct simdutf_riscv_hwprobe { + int64_t key; + uint64_t value; +}; + #define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__) + #define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4 + #define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2) + #define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17) +#endif // SIMDUTF_IS_RISCV64 && defined(__linux__) + +#if defined(__loongarch__) && defined(__linux__) + #include +// bits/hwcap.h +// #define HWCAP_LOONGARCH_LSX (1 << 4) +// #define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +namespace simdutf { +namespace internal { + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40, + ALTIVEC = 0x80, + AVX512F = 0x100, + AVX512DQ = 0x200, + AVX512IFMA = 0x400, + AVX512PF = 0x800, + AVX512ER = 0x1000, + AVX512CD = 0x2000, + AVX512BW = 0x4000, + AVX512VL = 0x8000, + AVX512VBMI2 = 0x10000, + AVX512VPOPCNTDQ = 0x2000, + RVV = 0x4000, + ZVBB = 0x8000, + LSX = 0x40000, + LASX = 0x80000, +}; + +#if defined(__PPC64__) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::ALTIVEC; +} + +#elif SIMDUTF_IS_RISCV64 + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if SIMDUTF_IS_RVV + host_isa |= instruction_set::RVV; + #endif + #if SIMDUTF_IS_ZVBB + host_isa |= instruction_set::ZVBB; + #endif + #if defined(__linux__) + simdutf_riscv_hwprobe probes[] = {{SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0}}; + long ret = simdutf_riscv_hwprobe(&probes, sizeof probes / sizeof *probes, 0, + nullptr, 0); + if (ret == 0) { + uint64_t extensions = probes[0].value; + if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V) + host_isa |= instruction_set::RVV; + if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB) + host_isa |= instruction_set::ZVBB; + } + #endif + #if defined(RUN_IN_SPIKE_SIMULATOR) + // Proxy Kernel does not implement yet hwprobe syscall + host_isa |= instruction_set::RVV; + #endif + return host_isa; +} + +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#elif defined(__x86_64__) || defined(_M_AMD64) // x64 + +namespace { +namespace cpuid_bit { +// Can be found on Intel ISA Reference for CPUID + +// EAX = 0x01 +constexpr uint32_t pclmulqdq = uint32_t(1) + << 1; ///< @private bit 1 of ECX for EAX=0x1 +constexpr uint32_t sse42 = uint32_t(1) + << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t osxsave = + (uint32_t(1) << 26) | + (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + +// EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf) +// See: "Table 3-8. Information Returned by CPUID Instruction" +namespace ebx { +constexpr uint32_t bmi1 = uint32_t(1) << 3; +constexpr uint32_t avx2 = uint32_t(1) << 5; +constexpr uint32_t bmi2 = uint32_t(1) << 8; +constexpr uint32_t avx512f = uint32_t(1) << 16; +constexpr uint32_t avx512dq = uint32_t(1) << 17; +constexpr uint32_t avx512ifma = uint32_t(1) << 21; +constexpr uint32_t avx512cd = uint32_t(1) << 28; +constexpr uint32_t avx512bw = uint32_t(1) << 30; +constexpr uint32_t avx512vl = uint32_t(1) << 31; +} // namespace ebx + +namespace ecx { +constexpr uint32_t avx512vbmi = uint32_t(1) << 1; +constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6; +constexpr uint32_t avx512vnni = uint32_t(1) << 11; +constexpr uint32_t avx512bitalg = uint32_t(1) << 12; +constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14; +} // namespace ecx +namespace edx { +constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8; +} +namespace xcr0_bit { +constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX +constexpr uint64_t avx512_saved = + uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM +} // namespace xcr0_bit +} // namespace cpuid_bit +} // namespace + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { + #if defined(_MSC_VER) + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; + #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); + #else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; + #endif +} + +static inline uint64_t xgetbv() { + #if defined(_MSC_VER) + return _xgetbv(0); + #else + uint32_t xcr0_lo, xcr0_hi; + asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); + #endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax; + uint32_t ebx = 0; + uint32_t ecx = 0; + uint32_t edx = 0; + uint32_t host_isa = 0x0; + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_bit::sse42) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_bit::pclmulqdq) { + host_isa |= instruction_set::PCLMULQDQ; + } + + if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) { + return host_isa; + } + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; // Sub-leaf = 0 + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_bit::ebx::avx2) { + host_isa |= instruction_set::AVX2; + } + if (ebx & cpuid_bit::ebx::bmi1) { + host_isa |= instruction_set::BMI1; + } + if (ebx & cpuid_bit::ebx::bmi2) { + host_isa |= instruction_set::BMI2; + } + if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == + cpuid_bit::xcr0_bit::avx512_saved)) { + return host_isa; + } + if (ebx & cpuid_bit::ebx::avx512f) { + host_isa |= instruction_set::AVX512F; + } + if (ebx & cpuid_bit::ebx::avx512bw) { + host_isa |= instruction_set::AVX512BW; + } + if (ebx & cpuid_bit::ebx::avx512cd) { + host_isa |= instruction_set::AVX512CD; + } + if (ebx & cpuid_bit::ebx::avx512dq) { + host_isa |= instruction_set::AVX512DQ; + } + if (ebx & cpuid_bit::ebx::avx512vl) { + host_isa |= instruction_set::AVX512VL; + } + if (ecx & cpuid_bit::ecx::avx512vbmi2) { + host_isa |= instruction_set::AVX512VBMI2; + } + if (ecx & cpuid_bit::ecx::avx512vpopcnt) { + host_isa |= instruction_set::AVX512VPOPCNTDQ; + } + return host_isa; +} +#elif defined(__loongarch__) + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if defined(__linux__) + uint64_t hwcap = 0; + hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_LOONGARCH_LSX) { + host_isa |= instruction_set::LSX; + } + if (hwcap & HWCAP_LOONGARCH_LASX) { + host_isa |= instruction_set::LASX; + } + #endif + return host_isa; +} +#else // fallback + +// includes 32-bit ARM. +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif // end SIMD extension detection code + +} // namespace internal +} // namespace simdutf + +#endif // SIMDutf_INTERNAL_ISADETECTION_H +/* end file include/simdutf/internal/isadetection.h */ + +#if SIMDUTF_SPAN + #include + #include + #include + #include +#endif +#if SIMDUTF_CPLUSPLUS17 + #include +#endif +// The following defines are conditionally enabled/disabled during amalgamation. +// By default all features are enabled, regular code shouldn't check them. Only +// when user code really relies of a selected subset, it's good to verify these +// flags, like: +// +// #if !SIMDUTF_FEATURE_UTF16 +// # error("Please amalgamate simdutf with UTF-16 support") +// #endif +// +#define SIMDUTF_FEATURE_DETECT_ENCODING 1 +#define SIMDUTF_FEATURE_ASCII 1 +#define SIMDUTF_FEATURE_LATIN1 1 +#define SIMDUTF_FEATURE_UTF8 1 +#define SIMDUTF_FEATURE_UTF16 1 +#define SIMDUTF_FEATURE_UTF32 1 +#define SIMDUTF_FEATURE_BASE64 1 + +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include + +namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; + + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template +constexpr constexpr_ptr constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy operator*() const { + return constexpr_write_ptr_proxy{p}; + } + + constexpr constexpr_write_ptr_proxy + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif + +#if SIMDUTF_SPAN +/// helpers placed in namespace detail are not a part of the public API +namespace simdutf { +namespace detail { +/** + * matches a byte, in the many ways C++ allows. note that these + * are all distinct types. + */ +template +concept byte_like = std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v; + +template +concept is_byte_like = byte_like>; + +template +concept is_pointer = std::is_pointer_v; + +/** + * matches anything that behaves like std::span and points to character-like + * data such as: std::byte, char, unsigned char, signed char, std::int8_t, + * std::uint8_t + */ +template +concept input_span_of_byte_like = requires(const T &t) { + { t.size() } noexcept -> std::convertible_to; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; +}; + +template +concept is_mutable = !std::is_const_v>; + +/** + * like span_of_byte_like, but for an output span (intended to be written to) + */ +template +concept output_span_of_byte_like = requires(T &t) { + { t.size() } noexcept -> std::convertible_to; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; + { *t.data() } noexcept -> is_mutable; +}; + +/** + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array + * invalid examples: int*, std::array + */ +template +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t{} } -> simdutf::detail::byte_like; +}; +template +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +template +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; + +template +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; + +/** + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* + */ +template +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN + +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); +} + +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 +} + +namespace utf32 { +template constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; +} +} // namespace utf32 + +namespace utf16 { +template constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; +} +} // namespace utf16 + +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(src) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast(const_cast(src)); + const auto dst_value = + std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(dst) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter++; // ASCII + counter += static_cast( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template +simdutf_constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template +simdutf_constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +simdutf_really_inline constexpr bool low_surrogate(char16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +template +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate(c); + low_surrogate = is_low_surrogate(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if simdutf_constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl(reinterpret_cast(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16 +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors(buf, len, utf8_output, 0); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast(p[i] > 0x7F); // two-byte + counter += static_cast(p[i] > 0x7FF); // three-byte + counter += static_cast(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; + return res; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::indexes_into_byte_like) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + +#if SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const char *input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const uint8_t *input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. can be a anything span-like that has a + * data() and size() that points to character data: std::string, + * std::string_view, std::vector, std::span etc. + * @return the detected encoding type + */ +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding( + const detail::input_span_of_byte_like auto &input) noexcept { + return autodetect_encoding(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN + +/** + * Autodetect the possible encodings of the input in one pass. + * E.g., if the input might be UTF-16LE or UTF-8, this function returns + * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). + * + * Overridden by each implementation. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused int detect_encodings(const char *input, + size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const uint8_t *input, size_t length) noexcept { + return detect_encodings(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { + return detect_encodings(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-8 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf8_with_errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool +validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8(reinterpret_cast(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 +/** + * Validate the UTF-8 string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf8_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +validate_utf8_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8_with_errors( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii(reinterpret_cast(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string and stop on error. It might be faster than + * validate_utf8 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_ascii_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_ascii_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii_with_errors( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string as a UTF-16 sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16le_as_ascii(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Validate the UTF-16 string. + * This function may be best when you expect the input to be almost always + * valid. Otherwise, consider using validate_utf16_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-16LE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ +simdutf_warn_unused bool validate_utf16le(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool +validate_utf16le(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 +/** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ +simdutf_warn_unused bool validate_utf16be(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), input.size()); + } else + #endif + { + return validate_utf16be(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Validate the UTF-16 string and stop on error. + * It might be faster than validate_utf16 when an error is expected to occur + * early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16le_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16le_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16be_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16le(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16le(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16be(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16be(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. If input and output points to different + * memory areas, the procedure copies string, and it's expected that output + * memory is at least as big as the input. It's also possible to set input equal + * output, that makes replacements an in-place operation. + * + * @param input the UTF-16 string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16(std::span input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-32 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf32_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ +simdutf_warn_unused bool validate_utf32(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf32(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 +/** + * Validate the UTF-32 string and stop on error. It might be faster than + * validate_utf32 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf32_with_errors(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input, + size_t length, + char *utf8_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8( + const detail::input_span_of_byte_like auto &latin1_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert( + detail::constexpr_cast_ptr(latin1_input.data()), + latin1_input.size(), + detail::constexpr_cast_writeptr(utf8_output.data())); + } else + #endif + { + return convert_latin1_to_utf8( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-8 string with output limit. + * + * This function is suitable to work with inputs from untrusted sources. + * + * We write as many characters as possible. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t +convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert_safe_constexpr( + input.data(), input.size(), utf8_output.data(), utf8_output.size()); + } else + #endif + { + return convert_latin1_to_utf8_safe( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16le( + const detail::input_span_of_byte_like auto &latin1_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + latin1_input.data(), latin1_input.size(), utf16_output.data()); + } else + #endif + { + return convert_latin1_to_utf16le( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16be( + reinterpret_cast(input.data()), input.size(), + output.data()); + } +} + #endif // SIMDUTF_SPAN +/** + * Compute the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in Latin1 code units (char) required to + * encode the UTF-16 string as Latin1 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf16(size_t length) noexcept { + return length; +} + +/** + * Compute the number of code units that this Latin1 string would require in + * UTF-16 format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 2-byte code units (char16_t) required to + * encode the Latin1 string as UTF-16 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf32( + const detail::input_span_of_byte_like auto &latin1_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf32::convert( + latin1_input.data(), latin1_input.size(), utf32_output.data()); + } else + #endif + { + return convert_latin1_to_utf32( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input, + size_t length, + char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_latin1( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert(input.data(), input.size(), + output.data()); + } else + #endif + { + return convert_utf8_to_latin1(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_utf8_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format even when the UTF-16LE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +utf8_length_from_utf16le_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format even when the UTF-16BE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16be_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert a Latin1 string into a UTF-16 string. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t. + */ +simdutf_warn_unused size_t convert_latin1_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_latin1_with_errors( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_latin1_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_with_errors( + utf8_input.data(), utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf8_to_latin1_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16le_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16be_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf32_with_errors( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf32_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_valid_utf8_to_latin1( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_latin1( + const detail::input_span_of_byte_like auto &valid_utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_valid_utf8_to_latin1( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), latin1_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert valid UTF-8 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16le( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16le( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16be( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16be( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf32( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf32( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_latin1(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_latin1( + const detail::input_span_of_byte_like auto &latin1_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(), + latin1_input.size()); + } else + #endif + { + return utf8_length_from_latin1( + reinterpret_cast(latin1_input.data()), + latin1_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ +simdutf_warn_unused size_t latin1_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return latin1_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Compute the number of 2-byte code units that this UTF-8 string would require + * in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 string + * as UTF-16LE + */ +simdutf_warn_unused size_t utf16_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf16_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of 4-byte code units that this UTF-8 string would require + * in UTF-32 format. + * + * This function is equivalent to count_utf8 + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 string + * as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf32_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string with output limit. + * + * We write as many characters as possible into the output buffer, + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 16-bit code units (char16_t) + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input, + size_t length, + char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_safe( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + const full_result r = + scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data(), + utf8_output.size()); + if (r.error != error_code::SUCCESS && + r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + return 0; + } + return r.output_count; + } else + #endif + { + return convert_utf16_to_utf8_safe( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_latin1( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_latin1_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf8_with_errors( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert UTF-16 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16LE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16le_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16le_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16BE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16be_to_latin1( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16be_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf8( + std::span valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-32 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf32(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into + * UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf32_with_errors(std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf32_with_errors( + std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf32_with_errors( + std::span utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf32(std::span valid_utf16_input, + std::span utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; compute the number of bytes that this UTF-16 + * string would require in UTF-8 format even when the UTF-16LE content contains + * mismatched surrogates that have to be replaced by the replacement character + * (0xFFFD). + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf8( + std::span utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf8_with_errors( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf8_with_errors( + std::span utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf8( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf8( + std::span valid_utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf8( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-32 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16le(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_latin1( + std::span utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_latin1_with_errors( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_latin1_with_errors( + std::span utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_with_errors( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf32_to_latin1( + std::span valid_utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_valid( + detail::constexpr_cast_ptr(valid_utf32_input.data()), + valid_utf32_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } + #endif + { + return convert_valid_utf32_to_latin1( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +latin1_length_from_utf32(size_t length) noexcept { + return length; +} + +/** + * Compute the number of bytes that this Latin1 string would require in UTF-32 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 4-byte code units (char32_t) required to + * encode the Latin1 string as UTF-32 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +utf32_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16be(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-32 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16_with_errors(std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16le_with_errors( + std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16be_with_errors( + std::span utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-32 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16(std::span valid_utf32_input, + std::span utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16le(std::span valid_utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16le(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16be(std::span valid_utf32_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16be(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ +void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(std::span utf16_input, + std::span utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::change_endianness_utf16( + utf16_input.data(), utf16_input.size(), utf16_output.data()); + } else + #endif + { + return change_endianness_utf16(utf16_input.data(), utf16_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf32(std::span valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ +simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(std::span valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-32 format. + * + * This function is equivalent to count_utf16. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16le( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16be( + std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). It is acceptable to pass invalid UTF-16 strings but in such + * cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ +simdutf_warn_unused size_t count_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return count_utf8(reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-8 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so + * that the short UTF-8 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-8, but possibly + * truncated. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes + */ +simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return trim_partial_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Given a valid UTF-16BE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16BE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16BE, but possibly + * truncated. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16be(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16LE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16LE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16LE, but possibly + * truncated. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16le(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16, but possibly + * truncated. We use the native endianness. + * + * @param input the UTF-16 string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || \ + SIMDUTF_FEATURE_DETECT_ENCODING + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif +#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_BASE64 +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include +#include +#include +#include +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template simdutf_constexpr23 bool is_eight_byte(char_type c) { + if simdutf_constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast(x & 0xFF); + *dst++ = static_cast((x >> 8) & 0xFF); + *dst++ = static_cast((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + *dst++ = static_cast(triple & 0xFF); + } +} + +template +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if simdutf_constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if simdutf_constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src, + size_t srclen, + base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +template +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen +} + +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(base64_options options) { + switch (options) { + case base64_default: + return "base64_default"; + case base64_url: + return "base64_url"; + case base64_reverse_padding: + return "base64_reverse_padding"; + case base64_url_with_padding: + return "base64_url_with_padding"; + case base64_default_accept_garbage: + return "base64_default_accept_garbage"; + case base64_url_accept_garbage: + return "base64_url_accept_garbage"; + case base64_default_or_url: + return "base64_default_or_url"; + case base64_default_or_url_accept_garbage: + return "base64_default_or_url_accept_garbage"; + } + return ""; +} + #endif // SIMDUTF_CPLUSPLUS17 + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(last_chunk_handling_options options) { + switch (options) { + case loose: + return "loose"; + case strict: + return "strict"; + case stop_before_partial: + return "stop_before_partial"; + case only_full_chunks: + return "only_full_chunks"; + } + return ""; +} + #endif + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximum number of binary bytes + */ +simdutf_warn_unused size_t +maximal_binary_length_from_base64(const char *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ +simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, usually base64_default or + * base64_url, and base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if successful. + */ +simdutf_warn_unused result base64_to_binary( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} + +/** + * Provide the base64 length in bytes given the length of a binary input, + * taking into account line breaks. + * + * @param length the length of the input in bytes + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( + size_t length, base64_options options = base64_default, + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} + +/** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a binary input to a base64 output with line breaks. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options) + */ +size_t +binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), line_length, options); + } +} + #endif // SIMDUTF_SPAN + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a binary input to a base64 output, using atomic accesses. + * This function comes with a potentially significant performance + * penalty, but it may be useful in some cases where the input + * buffers are shared between threads, to avoid undefined + * behavior in case of data races. + * + * The function is for advanced users. Its main use case is when + * to silence sanitizer warnings. We have no documented use case + * where this function is actually necessary in terms of practical correctness. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @brief atomic_binary_to_base64 + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t +atomic_binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused size_t +atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + return atomic_binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail + * to provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of bytes written if successful. + */ +simdutf_warn_unused result +base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Check if a character is an ignorable base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is an ignorable base64 character, false + * otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} + +/** + * Check if a character is a valid base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * Note that padding characters are not considered valid base64 characters in + * this context, nor are spaces. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} + +/** + * Check if a character is a valid base64 character or the padding character + * ('='). Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are three possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer + * is too small (OUTPUT_BUFFER_TOO_SMALL). + * + * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written + * and the number of units processed, see description of the parameters and + * returned value. + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected + * to discard the output unless the parameter decode_up_to_bad_char is set to + * true. In that case, the function will decode up to the first invalid + * character. Extra padding characters ('=') are considered invalid characters. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 8-bit + * or 16-bit units + * @param length the length of the string in 8-bit or 16-bit units. + * @param output the pointer to a buffer that can hold the conversion + * result. + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of units processed if successful. + */ +simdutf_warn_unused result +base64_to_binary_safe(const char *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +// the span overload has moved to the bottom of the file + +simdutf_warn_unused result +base64_to_binary_safe(const char16_t *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; + // span overload moved to bottom of file + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a base64 input to a binary output with a size limit and using atomic + * operations. + * + * Like `base64_to_binary_safe` but using atomic operations, this function is + * thread-safe for concurrent memory access, allowing the output + * buffers to be shared between threads without undefined behavior in case of + * data races. + * + * This function comes with a potentially significant performance penalty, but + * is useful when thread safety is needed during base64 decoding. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @param input the base64 input to decode + * @param length the length of the input in bytes + * @param output the pointer to buffer that can hold the conversion + * result + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use (default, url, etc.) + * @param last_chunk_options the last chunk handling options (loose, strict, + * stop_before_partial) + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result struct with an error code and count indicating error + * position or success + */ +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char16_t *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept; + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline simdutf_warn_unused std::tuple +atomic_base64_to_binary_safe( + const detail::input_span_of_byte_like auto &binary_input, + detail::output_span_of_byte_like auto &&output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = output.size(); + auto ret = atomic_base64_to_binary_safe( + reinterpret_cast(binary_input.data()), binary_input.size(), + reinterpret_cast(output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_warn_unused std::tuple +atomic_base64_to_binary_safe( + std::span base64_input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + auto ret = atomic_base64_to_binary_safe( + base64_input.data(), base64_input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +#endif // SIMDUTF_FEATURE_BASE64 + +/** + * An implementation of simdutf for a particular CPU architecture. + * + * Also used to maintain the currently active implementation. The active + * implementation is automatically initialized on first use to the most advanced + * implementation supported by the host. + */ +class implementation { +public: + /** + * The name of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string name() const { return std::string(_name); } + + /** + * The description of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string description() const { return std::string(_description); } + + /** + * The instruction sets this implementation is compiled against + * and the current CPU match. This function may poll the current CPU/system + * and should therefore not be called too often if performance is a concern. + * + * + * @return true if the implementation can be safely used on the current system + * (determined at runtime) + */ + bool supported_by_runtime_system() const; + +#if SIMDUTF_FEATURE_DETECT_ENCODING + /** + * This function will try to detect the encoding + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual encoding_type autodetect_encoding(const char *input, + size_t length) const noexcept; + + /** + * This function will try to detect the possible encodings in one pass + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual int detect_encodings(const char *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + + /** + * @private For internal implementation use + * + * The instruction sets this implementation is compiled against. + * + * @return a mask of all required `internal::instruction_set::` values + */ + virtual uint32_t required_instruction_sets() const { + return _required_instruction_sets; + } + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ + simdutf_warn_unused virtual bool validate_utf8(const char *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 + /** + * Validate the UTF-8 string and stop on errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_ascii(const char *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-16LE string.This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ + simdutf_warn_unused virtual bool + validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 + /** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ + simdutf_warn_unused virtual bool + validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16le_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + + /** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16be_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + /** + * Copies the UTF-16LE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; + /** + * Copies the UTF-16BE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-32 string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ + simdutf_warn_unused virtual bool + validate_utf32(const char32_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 + /** + * Validate the UTF-32 string and stop on error. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf32_with_errors(const char32_t *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf8(const char *input, size_t length, + char *utf8_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_latin1_with_errors(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format even when the UTF-16LE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format even when the UTF-16BE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_utf32_with_errors(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of 2-byte code units that this UTF-8 string would + * require in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 + * string as UTF-16LE + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of 4-byte code units that this UTF-8 string would + * require in UTF-32 format. + * + * This function is equivalent to count_utf8. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function does not validate the input. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 + * string as UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-L16LE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF16-BE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32BE string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_latin1_with_errors(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_utf8_with_errors(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ + virtual void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_latin1(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf32(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-16LE/BE string would require in + * Latin1 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf16(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16le(const char16_t *input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16be(const char16_t *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_BASE64 + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char *input, size_t length) const noexcept; + + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) const noexcept; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if + * successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you + * fail to provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process, in ASCII stored as + * 16-bit units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the + * number of bytes written if successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of base64 bytes + */ + simdutf_warn_unused size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) const noexcept; + + /** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ + virtual size_t + binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) const noexcept = 0; + + /** + * Convert a binary input to a base64 output with lines of given length. + * Lines are separated by a single linefeed character. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of each line, values smaller than 4 are + * interpreted as 4 + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options, line_length) + */ + virtual size_t binary_to_base64_with_lines( + const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) const noexcept = 0; + + /** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ + virtual const char *find(const char *start, const char *end, + char character) const noexcept = 0; + virtual const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept = 0; +#endif // SIMDUTF_FEATURE_BASE64 + +#ifdef SIMDUTF_INTERNAL_TESTS + // This method is exported only in developer mode, its purpose + // is to expose some internal test procedures from the given + // implementation and then use them through our standard test + // framework. + // + // Regular users should not use it, the tests of the public + // API are enough. + + struct TestProcedure { + // display name + std::string name; + + // procedure should return whether given test pass or not + void (*procedure)(const implementation &); + }; + + virtual std::vector internal_tests() const; +#endif + +protected: + /** @private Construct an implementation with the given name and description. + * For subclasses. */ + simdutf_really_inline implementation(const char *name, + const char *description, + uint32_t required_instruction_sets) + : _name(name), _description(description), + _required_instruction_sets(required_instruction_sets) {} + +protected: + ~implementation() = default; + +private: + /** + * The name of this implementation. + */ + const char *_name; + + /** + * The description of this implementation. + */ + const char *_description; + + /** + * Instruction sets required for this implementation. + */ + const uint32_t _required_instruction_sets; +}; + +/** @private */ +namespace internal { + +/** + * The list of available implementations compiled into simdutf. + */ +class available_implementation_list { +public: + /** Get the list of available implementations compiled into simdutf */ + simdutf_really_inline available_implementation_list() {} + /** Number of implementations */ + size_t size() const noexcept; + /** STL const begin() iterator */ + const implementation *const *begin() const noexcept; + /** STL const end() iterator */ + const implementation *const *end() const noexcept; + + /** + * Get the implementation with the given name. + * + * Case sensitive. + * + * const implementation *impl = + * simdutf::available_implementations["westmere"]; if (!impl) { exit(1); } if + * (!imp->supported_by_runtime_system()) { exit(1); } + * simdutf::active_implementation = impl; + * + * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" + * @return the implementation, or nullptr if the parse failed. + */ + const implementation *operator[](const std::string &name) const noexcept { + for (const implementation *impl : *this) { + if (impl->name() == name) { + return impl; + } + } + return nullptr; + } + + /** + * Detect the most advanced implementation supported by the current host. + * + * This is used to initialize the implementation on startup. + * + * const implementation *impl = + * simdutf::available_implementation::detect_best_supported(); + * simdutf::active_implementation = impl; + * + * @return the most advanced supported implementation for the current host, or + * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no + * supported implementation. Will never return nullptr. + */ + const implementation *detect_best_supported() const noexcept; +}; + +template class atomic_ptr { +public: + atomic_ptr(T *_ptr) : ptr{_ptr} {} + +#if defined(SIMDUTF_NO_THREADS) + operator const T *() const { return ptr; } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr; } + + operator T *() { return ptr; } + T &operator*() { return *ptr; } + T *operator->() { return ptr; } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#else + operator const T *() const { return ptr.load(); } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr.load(); } + + operator T *() { return ptr.load(); } + T &operator*() { return *ptr; } + T *operator->() { return ptr.load(); } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#endif + +private: +#if defined(SIMDUTF_NO_THREADS) + T *ptr; +#else + std::atomic ptr; +#endif +}; + +class detect_best_supported_implementation_on_first_use; + +} // namespace internal + +/** + * The list of available implementations compiled into simdutf. + */ +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & +get_available_implementations(); + +/** + * The active implementation. + * + * Automatically initialized on first use to the most advanced implementation + * supported by this hardware. + */ +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr & +get_active_implementation(); + +} // namespace simdutf + +#if SIMDUTF_FEATURE_BASE64 + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +namespace simdutf { + +template +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same::value || + std::is_same::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = (std::min)( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple + base64_to_binary_safe( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#endif // SIMDUTF_FEATURE_BASE64 + +#endif // SIMDUTF_IMPLEMENTATION_H +/* end file include/simdutf/implementation.h */ + +// Implementation-internal files (must be included before the implementations +// themselves, to keep amalgamation working--otherwise, the first time a file is +// included, it might be put inside the #ifdef +// SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other +// implementations can't compile unless that implementation is turned on). + +SIMDUTF_POP_DISABLE_WARNINGS + +#endif // SIMDUTF_H +/* end file include/simdutf.h */ diff --git a/simdutf/simdutf_c.h b/simdutf/simdutf_c.h new file mode 100644 --- /dev/null +++ b/simdutf/simdutf_c.h @@ -0,0 +1,339 @@ +/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include() +#else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { +#endif + +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */ diff --git a/src/Data/Text.hs b/src/Data/Text.hs --- a/src/Data/Text.hs +++ b/src/Data/Text.hs @@ -1,2065 +1,2276 @@ -{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-} -{-# LANGUAGE TemplateHaskellQuotes #-} -{-# LANGUAGE Trustworthy #-} -{-# LANGUAGE UnliftedFFITypes #-} -{-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE PartialTypeSignatures #-} - -{-# OPTIONS_GHC -fno-warn-orphans #-} -{-# OPTIONS_GHC -Wno-partial-type-signatures #-} - --- | --- Module : Data.Text --- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts, --- (c) 2008, 2009 Tom Harper --- (c) 2021 Andrew Lelechenko --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : GHC --- --- A time and space-efficient implementation of Unicode text. --- Suitable for performance critical use, both in terms of large data --- quantities and high speed. --- --- /Note/: Read below the synopsis for important notes on the use of --- this module. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions, e.g. --- --- > import qualified Data.Text as T --- --- To use an extended and very rich family of functions for working --- with Unicode text (including normalization, regular expressions, --- non-standard encodings, text breaking, and locales), see the --- . --- - -module Data.Text - ( - -- * Strict vs lazy types - -- $strict - - -- * Acceptable data - -- $replacement - - -- * Definition of character - -- $character_definition - - -- * Fusion - -- $fusion - - -- * Types - Text - , StrictText - - -- * Creation and elimination - , pack - , unpack - , singleton - , empty - - -- * Basic interface - , cons - , snoc - , append - , uncons - , unsnoc - , head - , last - , tail - , init - , null - , length - , compareLength - - -- * Transformations - , map - , intercalate - , intersperse - , transpose - , reverse - , replace - - -- ** Case conversion - -- $case - , toCaseFold - , toLower - , toUpper - , toTitle - - -- ** Justification - , justifyLeft - , justifyRight - , center - - -- * Folds - , foldl - , foldl' - , foldl1 - , foldl1' - , foldr - , foldr' - , foldr1 - - -- ** Special folds - , concat - , concatMap - , any - , all - , maximum - , minimum - , isAscii - - -- * Construction - - -- ** Scans - , scanl - , scanl1 - , scanr - , scanr1 - - -- ** Accumulating maps - , mapAccumL - , mapAccumR - - -- ** Generation and unfolding - , replicate - , unfoldr - , unfoldrN - - -- * Substrings - - -- ** Breaking strings - , take - , takeEnd - , drop - , dropEnd - , takeWhile - , takeWhileEnd - , dropWhile - , dropWhileEnd - , dropAround - , strip - , stripStart - , stripEnd - , splitAt - , breakOn - , breakOnEnd - , break - , span - , spanM - , spanEndM - , group - , groupBy - , inits - , tails - - -- ** Breaking into many substrings - -- $split - , splitOn - , split - , chunksOf - - -- ** Breaking into lines and words - , lines - --, lines' - , words - , unlines - , unwords - - -- * Predicates - , isPrefixOf - , isSuffixOf - , isInfixOf - - -- ** View patterns - , stripPrefix - , stripSuffix - , commonPrefixes - - -- * Searching - , filter - , breakOnAll - , find - , elem - , partition - - -- , findSubstring - - -- * Indexing - -- $index - , index - , findIndex - , count - - -- * Zipping - , zip - , zipWith - - -- -* Ordered text - -- , sort - - -- * Low level operations - , copy - , unpackCString# - , unpackCStringAscii# - - , measureOff - ) where - -import Prelude (Char, Bool(..), Int, Maybe(..), String, - Eq, (==), (/=), Ord(..), Ordering(..), (++), - Monad(..), pure, Read(..), - (&&), (||), (+), (-), (.), ($), ($!), (>>), - not, return, otherwise, quot) -import Control.DeepSeq (NFData(rnf)) -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Data.Bits ((.&.)) -import qualified Data.Char as Char -import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, - Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) -import Control.Monad (foldM) -import Control.Monad.ST (ST, runST) -import qualified Data.Text.Array as A -import qualified Data.List as L hiding (head, tail) -import Data.Binary (Binary(get, put)) -import Data.Monoid (Monoid(..)) -import Data.Semigroup (Semigroup(..)) -import Data.String (IsString(..)) -import Data.Text.Internal.ArrayUtils (memchr) -import Data.Text.Internal.IsAscii (isAscii) -import Data.Text.Internal.Reverse (reverse) -import Data.Text.Internal.Measure (measure_off) -import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4) -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import Data.Text.Encoding (decodeUtf8', encodeUtf8) -import Data.Text.Internal.Fusion (stream, reverseStream, unstream) -import Data.Text.Internal.Private (span_) -import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack) -import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) -import qualified Prelude as P -import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, - reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray) -import Data.Text.Internal.Search (indices) -import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) -#if defined(__HADDOCK__) -import Data.ByteString (ByteString) -import qualified Data.Text.Lazy as L -#endif -import Data.Word (Word8) -import Foreign.C.Types -import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) -import qualified GHC.Exts as Exts -import GHC.Int (Int8) -import GHC.Stack (HasCallStack) -import qualified Language.Haskell.TH.Lib as TH -import qualified Language.Haskell.TH.Syntax as TH -import Text.Printf (PrintfArg, formatArg, formatString) -import System.Posix.Types (CSsize(..)) - -#if MIN_VERSION_template_haskell(2,16,0) -import Data.Text.Foreign (asForeignPtr) -import System.IO.Unsafe (unsafePerformIO) -#endif - --- $setup --- >>> :set -package transformers --- >>> import Control.Monad.Trans.State --- >>> import Data.Text --- >>> import qualified Data.Text as T --- >>> :seti -XOverloadedStrings - --- $character_definition --- --- This package uses the term /character/ to denote Unicode /code points/. --- --- Note that this is not the same thing as a grapheme (e.g. a --- composition of code points that form one visual symbol). For --- instance, consider the grapheme \"ä\". This symbol has two --- Unicode representations: a single code-point representation --- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point), --- and a two code point representation @U+0061@ (the \"@A@\" code --- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point). - --- $strict --- --- This package provides both strict and lazy 'Text' types. The --- strict type is provided by the "Data.Text" module, while the lazy --- type is provided by the "Data.Text.Lazy" module. Internally, the --- lazy @Text@ type consists of a list of strict chunks. --- --- The strict 'Text' type requires that an entire string fit into --- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of --- streaming strings that are larger than memory using a small memory --- footprint. In many cases, the overhead of chunked streaming makes --- the lazy 'Data.Text.Lazy.Text' type slower than its strict --- counterpart, but this is not always the case. Sometimes, the time --- complexity of a function in one module may be different from the --- other, due to their differing internal structures. --- --- Each module provides an almost identical API, with the main --- difference being that the strict module uses 'Int' values for --- lengths and counts, while the lazy module uses 'Data.Int.Int64' --- lengths. - --- $replacement --- --- A 'Text' value is a sequence of Unicode scalar values, as defined --- in --- . --- As such, a 'Text' cannot contain values in the range U+D800 to --- U+DFFF inclusive. Haskell implementations admit all Unicode code --- points --- () --- as 'Char' values, including code points from this invalid range. --- This means that there are some 'Char' values --- (corresponding to 'Data.Char.Surrogate' category) that are not valid --- Unicode scalar values, and the functions in this module must handle --- those cases. --- --- Within this module, many functions construct a 'Text' from one or --- more 'Char' values. Those functions will substitute 'Char' values --- that are not valid Unicode scalar values with the replacement --- character \"�\" (U+FFFD). Functions that perform this --- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". The functions replace --- invalid scalar values, instead of dropping them, as a security --- measure. For details, see --- .) - --- $fusion --- --- Starting from @text-1.3@ fusion is no longer implicit, --- and pipelines of transformations usually allocate intermediate 'Text' values. --- Users, who observe significant changes to performances, --- are encouraged to use fusion framework explicitly, employing --- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". - -instance Eq Text where - Text arrA offA lenA == Text arrB offB lenB - | lenA == lenB = A.equal arrA offA arrB offB lenA - | otherwise = False - {-# INLINE (==) #-} - -instance Ord Text where - compare = compareText - -instance Read Text where - readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] - --- | @since 1.2.2.0 -instance Semigroup Text where - (<>) = append - -instance Monoid Text where - mempty = empty - mappend = (<>) - mconcat = concat - --- | Performs replacement on invalid scalar values: --- --- >>> :set -XOverloadedStrings --- >>> "\55555" :: Text --- "\65533" -instance IsString Text where - fromString = pack - --- | Performs replacement on invalid scalar values: --- --- >>> :set -XOverloadedLists --- >>> ['\55555'] :: Text --- "\65533" --- --- @since 1.2.0.0 -instance Exts.IsList Text where - type Item Text = Char - fromList = pack - toList = unpack - -instance NFData Text where rnf !_ = () - --- | @since 1.2.1.0 -instance Binary Text where - put t = put (encodeUtf8 t) - get = do - bs <- get - case decodeUtf8' bs of - P.Left exn -> P.fail (P.show exn) - P.Right a -> P.return a - --- | This instance preserves data abstraction at the cost of inefficiency. --- We omit reflection services for the sake of data abstraction. --- --- This instance was created by copying the updated behavior of --- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you --- feel a mistake has been made, please feel free to submit --- improvements. --- --- The original discussion is archived here: --- --- --- The followup discussion that changed the behavior of 'Data.Set.Set' --- and 'Data.Map.Map' is archived here: --- - -instance Data Text where - gfoldl f z txt = z pack `f` (unpack txt) - toConstr _ = packConstr - gunfold k z c = case constrIndex c of - 1 -> k (z pack) - _ -> P.error "gunfold" - dataTypeOf _ = textDataType - --- | @since 1.2.4.0 -instance TH.Lift Text where -#if MIN_VERSION_template_haskell(2,16,0) - lift txt = do - let (ptr, len) = unsafePerformIO $ asForeignPtr txt - case len of - 0 -> TH.varE 'empty - _ -> - let - bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len) - lenQ = liftInt (P.fromIntegral len) - liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n))) - in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ -#else - lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack -#endif -#if MIN_VERSION_template_haskell(2,17,0) - liftTyped = TH.unsafeCodeCoerce . TH.lift -#elif MIN_VERSION_template_haskell(2,16,0) - liftTyped = TH.unsafeTExpCoerce . TH.lift -#endif - -#if MIN_VERSION_template_haskell(2,16,0) -unpackCStringLen# :: Exts.Addr# -> Int -> Text -unpackCStringLen# addr# l = Text ba 0 l - where - ba = runST $ do - marr <- A.new l - A.copyFromPointer marr 0 (Exts.Ptr addr#) l - A.unsafeFreeze marr -{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat -#endif - --- | @since 1.2.2.0 -instance PrintfArg Text where - formatArg txt = formatString $ unpack txt - -packConstr :: Constr -packConstr = mkConstr textDataType "pack" [] Prefix - -textDataType :: DataType -textDataType = mkDataType "Data.Text.Text" [packConstr] - --- | /O(n)/ Compare two 'Text' values lexicographically. -compareText :: Text -> Text -> Ordering -compareText (Text arrA offA lenA) (Text arrB offB lenB) = - A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB --- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), --- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering --- of underlying bytearrays, no decoding is needed. - --- ----------------------------------------------------------------------------- --- * Basic functions - --- | /O(n)/ Adds a character to the front of a 'Text'. This function --- is more costly than its 'List' counterpart because it requires --- copying a new array. Performs replacement on --- invalid scalar values. -cons :: Char -> Text -> Text -cons c = unstream . S.cons (safe c) . stream -{-# INLINE [1] cons #-} - -infixr 5 `cons` - --- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process. --- Performs replacement on invalid scalar values. -snoc :: Text -> Char -> Text -snoc t c = unstream (S.snoc (stream t) (safe c)) -{-# INLINE snoc #-} - --- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. This is a partial function, consider using 'uncons' instead. -head :: HasCallStack => Text -> Char -head t = S.head (stream t) -{-# INLINE head #-} - --- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. -uncons :: Text -> Maybe (Char, Text) -uncons t@(Text arr off len) - | len <= 0 = Nothing - | otherwise = Just $ let !(Iter c d) = iter t 0 - in (c, text arr (off+d) (len-d)) -{-# INLINE [1] uncons #-} - --- | /O(1)/ Returns the last character of a 'Text', which must be --- non-empty. This is a partial function, consider using 'unsnoc' instead. -last :: HasCallStack => Text -> Char -last t@(Text _ _ len) - | null t = emptyError "last" - | otherwise = let Iter c _ = reverseIter t (len - 1) in c -{-# INLINE [1] last #-} - --- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. This is a partial function, consider using 'uncons' instead. -tail :: HasCallStack => Text -> Text -tail t@(Text arr off len) - | null t = emptyError "tail" - | otherwise = text arr (off+d) (len-d) - where d = iter_ t 0 -{-# INLINE [1] tail #-} - --- | /O(1)/ Returns all but the last character of a 'Text', which must --- be non-empty. This is a partial function, consider using 'unsnoc' instead. -init :: HasCallStack => Text -> Text -init t@(Text arr off len) - | null t = emptyError "init" - | otherwise = text arr off (len + reverseIter_ t (len - 1)) -{-# INLINE [1] init #-} - --- | /O(1)/ Returns all but the last character and the last character of a --- 'Text', or 'Nothing' if empty. --- --- @since 1.2.3.0 -unsnoc :: Text -> Maybe (Text, Char) -unsnoc t@(Text arr off len) - | null t = Nothing - | otherwise = Just (text arr off (len + d), c) - where - Iter c d = reverseIter t (len - 1) -{-# INLINE [1] unsnoc #-} - --- | /O(1)/ Tests whether a 'Text' is empty or not. -null :: Text -> Bool -null (Text _arr _off len) = -#if defined(ASSERTS) - assert (len >= 0) $ -#endif - len <= 0 -{-# INLINE [1] null #-} - -{-# RULES - "TEXT null/empty -> True" null empty = True -#-} - --- | /O(1)/ Tests whether a 'Text' contains exactly one character. -isSingleton :: Text -> Bool -isSingleton = S.isSingleton . stream -{-# INLINE isSingleton #-} - --- | /O(n)/ Returns the number of characters in a 'Text'. -length :: -#if defined(ASSERTS) - HasCallStack => -#endif - Text -> Int -length = P.negate . measureOff P.maxBound -{-# INLINE [1] length #-} --- length needs to be phased after the compareN/length rules otherwise --- it may inline before the rules have an opportunity to fire. - -{-# RULES -"TEXT length/filter -> S.length/S.filter" forall p t. - length (filter p t) = S.length (S.filter p (stream t)) -"TEXT length/unstream -> S.length" forall t. - length (unstream t) = S.length t -"TEXT length/pack -> P.length" forall t. - length (pack t) = P.length t -"TEXT length/map -> length" forall f t. - length (map f t) = length t -"TEXT length/zipWith -> length" forall f t1 t2. - length (zipWith f t1 t2) = min (length t1) (length t2) -"TEXT length/replicate -> n" forall n t. - length (replicate n t) = mul (max 0 n) (length t) -"TEXT length/cons -> length+1" forall c t. - length (cons c t) = 1 + length t -"TEXT length/intersperse -> 2*length-1" forall c t. - length (intersperse c t) = max 0 (mul 2 (length t) - 1) -"TEXT length/intercalate -> n*length" forall s ts. - length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) -"TEXT length/empty -> 0" - length empty = 0 - #-} - --- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. --- --- @ --- 'compareLength' t c = 'P.compare' ('length' t) c --- @ --- --- This function gives the same answer as comparing against the result --- of 'length', but can short circuit if the count of characters is --- greater than the number, and hence be more efficient. -compareLength :: Text -> Int -> Ordering -compareLength t c = S.compareLengthI (stream t) c -{-# INLINE [1] compareLength #-} - -{-# RULES -"TEXT compareN/length -> compareLength" [~1] forall t n. - compare (length t) n = compareLength t n - #-} - -{-# RULES -"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. - eqInt (length t) n = compareLength t n == EQ - #-} - -{-# RULES -"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. - neInt (length t) n = compareLength t n /= EQ - #-} - -{-# RULES -"TEXT compareLength/==LT" [~1] forall t n. - ltInt (length t) n = compareLength t n == LT - #-} - -{-# RULES -"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. - leInt (length t) n = compareLength t n /= GT - #-} - -{-# RULES -"TEXT >N/length -> compareLength/==GT" [~1] forall t n. - gtInt (length t) n = compareLength t n == GT - #-} - -{-# RULES -"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. - geInt (length t) n = compareLength t n /= LT - #-} - --- ----------------------------------------------------------------------------- --- * Transformations --- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. --- --- Example: --- --- >>> let message = pack "I am not angry. Not at all." --- >>> T.map (\c -> if c == '.' then '!' else c) message --- "I am not angry! Not at all!" --- --- Performs replacement on invalid scalar values. -map :: (Char -> Char) -> Text -> Text -map f = \t -> if null t then empty else mapNonEmpty f t -{-# INLINE [1] map #-} - -{-# RULES -"TEXT map/map -> map" forall f g t. - map f (map g t) = map (f . safe . g) t -#-} - --- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of --- 'Text's and concatenates the list after interspersing the first --- argument between each element of the list. --- --- Example: --- --- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] --- "WeNI!seekNI!theNI!HolyNI!Grail" -intercalate :: Text -> [Text] -> Text -intercalate t = concat . L.intersperse t -{-# INLINE [1] intercalate #-} - --- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. --- --- Example: --- --- >>> T.intersperse '.' "SHIELD" --- "S.H.I.E.L.D" --- --- Performs replacement on invalid scalar values. -intersperse :: Char -> Text -> Text -intersperse c t@(Text src o l) = if null t then empty else runST $ do - let !cLen = utf8Length c - dstLen = l + length t P.* cLen - - dst <- A.new dstLen - - let writeSep = case cLen of - 1 -> \dstOff -> - A.unsafeWrite dst dstOff (ord8 c) - 2 -> let (c0, c1) = ord2 c in \dstOff -> do - A.unsafeWrite dst dstOff c0 - A.unsafeWrite dst (dstOff + 1) c1 - 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do - A.unsafeWrite dst dstOff c0 - A.unsafeWrite dst (dstOff + 1) c1 - A.unsafeWrite dst (dstOff + 2) c2 - _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do - A.unsafeWrite dst dstOff c0 - A.unsafeWrite dst (dstOff + 1) c1 - A.unsafeWrite dst (dstOff + 2) c2 - A.unsafeWrite dst (dstOff + 3) c3 - let go !srcOff !dstOff = if srcOff >= o + l then return () else do - let m0 = A.unsafeIndex src srcOff - m1 = A.unsafeIndex src (srcOff + 1) - m2 = A.unsafeIndex src (srcOff + 2) - m3 = A.unsafeIndex src (srcOff + 3) - !d = utf8LengthByLeader m0 - case d of - 1 -> do - A.unsafeWrite dst dstOff m0 - writeSep (dstOff + 1) - go (srcOff + 1) (dstOff + 1 + cLen) - 2 -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - writeSep (dstOff + 2) - go (srcOff + 2) (dstOff + 2 + cLen) - 3 -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - writeSep (dstOff + 3) - go (srcOff + 3) (dstOff + 3 + cLen) - _ -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - A.unsafeWrite dst (dstOff + 3) m3 - writeSep (dstOff + 4) - go (srcOff + 4) (dstOff + 4 + cLen) - - go o 0 - arr <- A.unsafeFreeze dst - return (Text arr 0 (dstLen - cLen)) -{-# INLINE [1] intersperse #-} - --- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in --- @haystack@ with @replacement@. --- --- This function behaves as though it was defined as follows: --- --- @ --- replace needle replacement haystack = --- 'intercalate' replacement ('splitOn' needle haystack) --- @ --- --- As this suggests, each occurrence is replaced exactly once. So if --- @needle@ occurs in @replacement@, that occurrence will /not/ itself --- be replaced recursively: --- --- >>> replace "oo" "foo" "oo" --- "foo" --- --- In cases where several instances of @needle@ overlap, only the --- first one will be replaced: --- --- >>> replace "ofo" "bar" "ofofo" --- "barfo" --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -replace :: HasCallStack - => Text - -- ^ @needle@ to search for. If this string is empty, an - -- error will occur. - -> Text - -- ^ @replacement@ to replace @needle@ with. - -> Text - -- ^ @haystack@ in which to search. - -> Text -replace needle@(Text _ _ neeLen) - (Text repArr repOff repLen) - haystack@(Text hayArr hayOff hayLen) - | neeLen == 0 = emptyError "replace" - | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it - | L.null ixs = haystack - | otherwise = Text (A.run x) 0 len - where - ixs = indices needle haystack - len = hayLen - (neeLen - repLen) `mul` L.length ixs - x :: ST s (A.MArray s) - x = do - marr <- A.new len - let loop (i:is) o d = do - let d0 = d + i - o - d1 = d0 + repLen - A.copyI (i - o) marr d hayArr (hayOff+o) - A.copyI repLen marr d0 repArr repOff - loop is (i + neeLen) d1 - loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) - loop ixs 0 0 - return marr - --- ---------------------------------------------------------------------------- --- ** Case conversions (folds) - --- $case --- --- When case converting 'Text' values, do not use combinators like --- @map toUpper@ to case convert each character of a string --- individually, as this gives incorrect results according to the --- rules of some writing systems. The whole-string case conversion --- functions from this module, such as @toUpper@, obey the correct --- case conversion rules. As a result, these functions may map one --- input character to two or three output characters. For examples, --- see the documentation of each function. --- --- /Note/: In some languages, case conversion is a locale- and --- context-dependent operation. The case conversion functions in this --- module are /not/ locale sensitive. Programs that require locale --- sensitivity should use appropriate versions of the --- . - --- | /O(n)/ Convert a string to folded case. --- --- This function is mainly useful for performing caseless (also known --- as case insensitive) string comparisons. --- --- A string @x@ is a caseless match for a string @y@ if and only if: --- --- @toCaseFold x == toCaseFold y@ --- --- The result string may be longer than the input string, and may --- differ from applying 'toLower' to the input string. For instance, --- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case --- folded to the sequence \"մ\" (men, U+0574) followed by --- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, --- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) --- instead of itself. -toCaseFold :: Text -> Text -toCaseFold = \t -> - if null t then empty - else toCaseFoldNonEmpty t -{-# INLINE toCaseFold #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. --- --- The result string may be longer than the input string. For --- instance, \"İ\" (Latin capital letter I with dot above, --- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) --- followed by \" ̇\" (combining dot above, U+0307). -toLower :: Text -> Text -toLower = \t -> - if null t then empty - else toLowerNonEmpty t -{-# INLINE toLower #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. --- --- The result string may be longer than the input string. For --- instance, the German \"ß\" (eszett, U+00DF) maps to the --- two-letter sequence \"SS\". -toUpper :: Text -> Text -toUpper = \t -> - if null t then empty - else toUpperNonEmpty t -{-# INLINE toUpper #-} - --- | /O(n)/ Convert a string to title case, using simple case --- conversion. --- --- The first letter (as determined by 'Data.Char.isLetter') --- of the input is converted to title case, as is --- every subsequent letter that immediately follows a non-letter. --- Every letter that immediately follows another letter is converted --- to lower case. --- --- This function is not idempotent. --- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). --- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input --- is converted to title case, becoming two letters. --- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter --- and as such is recognised as a letter by 'Data.Char.isLetter', --- so 'T.toTitle' @"ʼN"@ = @"'n"@. --- --- The result string may be longer than the input string. For example, --- the Latin small ligature fl (U+FB02) is converted to the --- sequence Latin capital letter F (U+0046) followed by Latin small --- letter l (U+006C). --- --- /Note/: this function does not take language or culture specific --- rules into account. For instance, in English, different style --- guides disagree on whether the book name \"The Hill of the Red --- Fox\" is correctly title cased—but this function will --- capitalize /every/ word. --- --- @since 1.0.0.0 -toTitle :: Text -> Text -toTitle t = unstream (S.toTitle (stream t)) -{-# INLINE toTitle #-} - --- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. --- Performs replacement on invalid scalar values. --- --- Examples: --- --- >>> justifyLeft 7 'x' "foo" --- "fooxxxx" --- --- >>> justifyLeft 3 'x' "foobar" --- "foobar" -justifyLeft :: Int -> Char -> Text -> Text -justifyLeft k c t - | len >= k = t - | otherwise = t `append` replicateChar (k-len) c - where len = length t -{-# INLINE [1] justifyLeft #-} - --- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Performs replacement on --- invalid scalar values. --- --- Examples: --- --- >>> justifyRight 7 'x' "bar" --- "xxxxbar" --- --- >>> justifyRight 3 'x' "foobar" --- "foobar" -justifyRight :: Int -> Char -> Text -> Text -justifyRight k c t - | len >= k = t - | otherwise = replicateChar (k-len) c `append` t - where len = length t -{-# INLINE justifyRight #-} - --- | /O(n)/ Center a string to the given length, using the specified --- fill character on either side. Performs replacement on invalid --- scalar values. --- --- Examples: --- --- >>> center 8 'x' "HS" --- "xxxHSxxx" -center :: Int -> Char -> Text -> Text -center k c t - | len >= k = t - | otherwise = replicateChar l c `append` t `append` replicateChar r c - where len = length t - d = k - len - r = d `quot` 2 - l = d - r -{-# INLINE center #-} - --- | /O(n)/ The 'transpose' function transposes the rows and columns --- of its 'Text' argument. Note that this function uses 'pack', --- 'unpack', and the list version of transpose, and is thus not very --- efficient. --- --- Examples: --- --- >>> transpose ["green","orange"] --- ["go","rr","ea","en","ng","e"] --- --- >>> transpose ["blue","red"] --- ["br","le","ud","e"] -transpose :: [Text] -> [Text] -transpose ts = P.map pack (L.transpose (P.map unpack ts)) - --- ----------------------------------------------------------------------------- --- * Reducing 'Text's (folds) - --- | /O(n)/ 'foldl', applied to a binary operator, a starting value --- (typically the left-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from left to right. -foldl :: (a -> Char -> a) -> a -> Text -> a -foldl f z t = S.foldl f z (stream t) -{-# INLINE foldl #-} - --- | /O(n)/ A strict version of 'foldl'. -foldl' :: (a -> Char -> a) -> a -> Text -> a -foldl' f z t = S.foldl' f z (stream t) -{-# INLINE foldl' #-} - --- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. -foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldl1 f t = S.foldl1 f (stream t) -{-# INLINE foldl1 #-} - --- | /O(n)/ A strict version of 'foldl1'. -foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldl1' f t = S.foldl1' f (stream t) -{-# INLINE foldl1' #-} - --- | /O(n)/ 'foldr', applied to a binary operator, a starting value --- (typically the right-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from right to left. --- --- If the binary operator is strict in its second argument, use 'foldr'' --- instead. --- --- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually --- traverses the 'Text' from left to right, only as far as it needs to. --- --- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': --- --- @ --- head :: Text -> Char --- head = foldr const (error "head empty") --- @ --- --- Searches from left to right with short-circuiting behavior can --- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem'). -foldr :: (Char -> a -> a) -> a -> Text -> a -foldr f z t = S.foldr f z (stream t) -{-# INLINE foldr #-} - --- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. -foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldr1 f t = S.foldr1 f (stream t) -{-# INLINE foldr1 #-} - --- | /O(n)/ A strict version of 'foldr'. --- --- 'foldr'' evaluates as a right-to-left traversal using constant stack space. --- --- @since 2.0.1 -foldr' :: (Char -> a -> a) -> a -> Text -> a -foldr' f z t = S.foldl' (P.flip f) z (reverseStream t) -{-# INLINE foldr' #-} - --- ----------------------------------------------------------------------------- --- ** Special folds - --- | /O(n)/ Concatenate a list of 'Text's. -concat :: [Text] -> Text -concat ts = case ts of - [] -> empty - [t] -> t - _ | len == 0 -> empty - | otherwise -> Text (A.run go) 0 len - where - len = sumP "concat" $ L.map lengthWord8 ts - go :: ST s (A.MArray s) - go = do - arr <- A.new len - let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) - foldM step 0 ts >> return arr - --- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and --- concatenate the results. -concatMap :: (Char -> Text) -> Text -> Text -concatMap f = concat . foldr ((:) . f) [] -{-# INLINE concatMap #-} - --- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. -any :: (Char -> Bool) -> Text -> Bool -any p t = S.any p (stream t) -{-# INLINE any #-} - --- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. -all :: (Char -> Bool) -> Text -> Bool -all p t = S.all p (stream t) -{-# INLINE all #-} - --- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. -maximum :: HasCallStack => Text -> Char -maximum t = S.maximum (stream t) -{-# INLINE maximum #-} - --- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. -minimum :: HasCallStack => Text -> Char -minimum t = S.minimum (stream t) -{-# INLINE minimum #-} - --- ----------------------------------------------------------------------------- --- * Building 'Text's --- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. --- Performs replacement on invalid scalar values. --- --- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --- --- __Properties__ --- --- @'head' ('scanl' f z xs) = z@ --- --- @'last' ('scanl' f z xs) = 'foldl' f z xs@ -scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl g z (stream t)) - where g a b = safe (f a b) -{-# INLINE scanl #-} - --- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. Performs replacement on invalid scalar values. --- --- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] -scanl1 :: (Char -> Char -> Char) -> Text -> Text -scanl1 f t | null t = empty - | otherwise = scanl f (unsafeHead t) (unsafeTail t) -{-# INLINE scanl1 #-} - --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs --- replacement on invalid scalar values. --- --- > scanr f v == reverse . scanl (flip f) v . reverse -scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f z = S.reverse . S.reverseScanr g z . reverseStream - where g a b = safe (f a b) -{-# INLINE scanr #-} - --- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. Performs replacement on invalid scalar values. -scanr1 :: (Char -> Char -> Char) -> Text -> Text -scanr1 f t | null t = empty - | otherwise = scanr f (last t) (init t) -{-# INLINE scanr1 #-} - --- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a --- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. Performs --- replacement on invalid scalar values. -mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) -mapAccumL f z0 = go - where - go (Text src o l) = runST $ do - marr <- A.new (l + 4) - outer marr (l + 4) o 0 z0 - where - outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) - outer !dst !dstLen = inner - where - inner !srcOff !dstOff !z - | srcOff >= l + o = do - A.shrinkM dst dstOff - arr <- A.unsafeFreeze dst - return (z, Text arr 0 dstOff) - | dstOff + 4 > dstLen = do - let !dstLen' = dstLen + (l + o) - srcOff + 4 - dst' <- A.resizeM dst dstLen' - outer dst' dstLen' srcOff dstOff z - | otherwise = do - let !(Iter c d) = iterArray src srcOff - (z', c') = f z c - d' <- unsafeWrite dst dstOff (safe c') - inner (srcOff + d) (dstOff + d') z' -{-# INLINE mapAccumL #-} - --- | The 'mapAccumR' function behaves like a combination of 'map' and --- a strict 'foldr'; it applies a function to each element of a --- 'Text', passing an accumulating parameter from right to left, and --- returning a final value of this accumulator together with the new --- 'Text'. --- Performs replacement on invalid scalar values. -mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) -mapAccumR f z0 = go - where - go (Text src o l) = runST $ do - marr <- A.new (l + 4) - outer marr (l + o - 1) (l + 4 - 1) z0 - where - outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) - outer !dst = inner - where - inner !srcOff !dstOff !z - | srcOff < o = do - dstLen <- A.getSizeofMArray dst - arr <- A.unsafeFreeze dst - return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) - | dstOff < 3 = do - dstLen <- A.getSizeofMArray dst - let !dstLen' = dstLen + (srcOff - o) + 4 - dst' <- A.new dstLen' - A.copyM dst' (dstLen' - dstLen) dst 0 dstLen - outer dst' srcOff (dstOff + dstLen' - dstLen) z - | otherwise = do - let !(Iter c d) = reverseIterArray src (srcOff) - (z', c') = f z c - c'' = safe c' - !d' = utf8Length c'' - dstOff' = dstOff - d' - _ <- unsafeWrite dst (dstOff' + 1) c'' - inner (srcOff + d) dstOff' z' -{-# INLINE mapAccumR #-} - --- ----------------------------------------------------------------------------- --- ** Generating and unfolding 'Text's - --- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input --- @t@ repeated @n@ times. -replicate :: Int -> Text -> Text -replicate n t@(Text a o l) - | n <= 0 || l <= 0 = empty - | n == 1 = t - | isSingleton t = replicateChar n (unsafeHead t) - | otherwise = runST $ do - let totalLen = n `mul` l - marr <- A.new totalLen - A.copyI l marr 0 a o - A.tile marr l - arr <- A.unsafeFreeze marr - return $ Text arr 0 totalLen -{-# INLINE [1] replicate #-} - -{-# RULES -"TEXT replicate/singleton -> replicateChar" [~1] forall n c. - replicate n (singleton c) = replicateChar n c - #-} - --- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the --- value of every element. -replicateChar :: Int -> Char -> Text -replicateChar !len !c' - | len <= 0 = empty - | Char.isAscii c = runST $ do - marr <- A.newFilled len (Char.ord c) - arr <- A.unsafeFreeze marr - return $ Text arr 0 len - | otherwise = runST $ do - let cLen = utf8Length c - totalLen = cLen P.* len - marr <- A.new totalLen - _ <- unsafeWrite marr 0 c - A.tile marr cLen - arr <- A.unsafeFreeze marr - return $ Text arr 0 totalLen - where - c = safe c' -{-# INLINE replicateChar #-} - --- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' --- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a --- 'Text' from a seed value. The function takes the element and --- returns 'Nothing' if it is done producing the 'Text', otherwise --- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. --- Performs replacement on invalid scalar values. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) -{-# INLINE unfoldr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed --- value. However, the length of the result should be limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the maximum length of the result is known and --- correct, otherwise its performance is similar to 'unfoldr'. --- Performs replacement on invalid scalar values. -unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) -{-# INLINE unfoldrN #-} - --- ----------------------------------------------------------------------------- --- * Substrings - --- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the --- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. -take :: Int -> Text -> Text -take n t@(Text arr off len) - | n <= 0 = empty - | n >= len || m >= len || m < 0 = t - | otherwise = Text arr off m - where - m = measureOff n t -{-# INLINE [1] take #-} - --- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ --- returns a non-negative number, measuring their size in 'Word8'. Otherwise, --- if @t@ is shorter, return a non-positive number, which is a negated total count --- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. --- --- This function is used to implement 'take', 'drop', 'splitAt' and 'length' --- and is useful on its own in streaming and parsing libraries. --- --- @since 2.0 -measureOff :: Int -> Text -> Int -measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else - cSsizeToInt $ - measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) - --- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after --- taking @n@ characters from the end of @t@. --- --- Examples: --- --- >>> takeEnd 3 "foobar" --- "bar" --- --- @since 1.1.1.0 -takeEnd :: Int -> Text -> Text -takeEnd n t@(Text arr off len) - | n <= 0 = empty - | n >= len = t - | otherwise = text arr (off+i) (len-i) - where i = iterNEnd n t - -iterNEnd :: Int -> Text -> Int -iterNEnd n t@(Text _arr _off len) = loop (len-1) n - where loop i !m - | m <= 0 = i+1 - | i <= 0 = 0 - | otherwise = loop (i+d) (m-1) - where d = reverseIter_ t i - --- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the --- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. -drop :: Int -> Text -> Text -drop n t@(Text arr off len) - | n <= 0 = t - | n >= len || m >= len || m < 0 = empty - | otherwise = Text arr (off+m) (len-m) - where m = measureOff n t -{-# INLINE [1] drop #-} - --- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after --- dropping @n@ characters from the end of @t@. --- --- Examples: --- --- >>> dropEnd 3 "foobar" --- "foo" --- --- @since 1.1.1.0 -dropEnd :: Int -> Text -> Text -dropEnd n t@(Text arr off len) - | n <= 0 = t - | n >= len = empty - | otherwise = text arr off (iterNEnd n t) - --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', --- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. -takeWhile :: (Char -> Bool) -> Text -> Text -takeWhile p t@(Text arr off len) = loop 0 - where loop !i | i >= len = t - | p c = loop (i+d) - | otherwise = text arr off i - where Iter c d = iter t i -{-# INLINE [1] takeWhile #-} - --- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', --- returns the longest suffix (possibly empty) of elements that --- satisfy @p@. --- Examples: --- --- >>> takeWhileEnd (=='o') "foo" --- "oo" --- --- @since 1.2.2.0 -takeWhileEnd :: (Char -> Bool) -> Text -> Text -takeWhileEnd p t@(Text arr off len) = loop (len-1) len - where loop !i !l | l <= 0 = t - | p c = loop (i+d) (l+d) - | otherwise = text arr (off+l) (len-l) - where Iter c d = reverseIter t i -{-# INLINE [1] takeWhileEnd #-} - --- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. -dropWhile :: (Char -> Bool) -> Text -> Text -dropWhile p t@(Text arr off len) = loop 0 0 - where loop !i !l | l >= len = empty - | p c = loop (i+d) (l+d) - | otherwise = Text arr (off+i) (len-l) - where Iter c d = iter t i -{-# INLINE [1] dropWhile #-} - --- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after --- dropping characters that satisfy the predicate @p@ from the end of --- @t@. --- --- Examples: --- --- >>> dropWhileEnd (=='.') "foo..." --- "foo" -dropWhileEnd :: (Char -> Bool) -> Text -> Text -dropWhileEnd p t@(Text arr off len) = loop (len-1) len - where loop !i !l | l <= 0 = empty - | p c = loop (i+d) (l+d) - | otherwise = Text arr off l - where Iter c d = reverseIter t i -{-# INLINE [1] dropWhileEnd #-} - --- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after --- dropping characters that satisfy the predicate @p@ from both the --- beginning and end of @t@. -dropAround :: (Char -> Bool) -> Text -> Text -dropAround p = dropWhile p . dropWhileEnd p -{-# INLINE [1] dropAround #-} - --- | /O(n)/ Remove leading white space from a string. Equivalent to: --- --- > dropWhile isSpace -stripStart :: Text -> Text -stripStart = dropWhile Char.isSpace -{-# INLINE stripStart #-} - --- | /O(n)/ Remove trailing white space from a string. Equivalent to: --- --- > dropWhileEnd isSpace -stripEnd :: Text -> Text -stripEnd = dropWhileEnd Char.isSpace -{-# INLINE [1] stripEnd #-} - --- | /O(n)/ Remove leading and trailing white space from a string. --- Equivalent to: --- --- > dropAround isSpace -strip :: Text -> Text -strip = dropAround Char.isSpace -{-# INLINE [1] strip #-} - --- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a --- prefix of @t@ of length @n@, and whose second is the remainder of --- the string. It is equivalent to @('take' n t, 'drop' n t)@. -splitAt :: Int -> Text -> (Text, Text) -splitAt n t@(Text arr off len) - | n <= 0 = (empty, t) - | n >= len || m >= len || m < 0 = (t, empty) - | otherwise = (Text arr off m, Text arr (off+m) (len-m)) - where - m = measureOff n t - --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns --- a pair whose first element is the longest prefix (possibly empty) --- of @t@ of elements that satisfy @p@, and whose second is the --- remainder of the text. --- --- >>> T.span (=='0') "000AB" --- ("000","AB") -span :: (Char -> Bool) -> Text -> (Text, Text) -span p t = case span_ p t of - (# hd,tl #) -> (hd,tl) -{-# INLINE span #-} - --- | /O(n)/ 'break' is like 'span', but the prefix returned is --- over elements that fail the predicate @p@. --- --- >>> T.break (=='c') "180cm" --- ("180","cm") -break :: (Char -> Bool) -> Text -> (Text, Text) -break p = span (not . p) -{-# INLINE break #-} - --- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, --- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of --- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. --- --- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 --- (("abc","efg"),101) --- --- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': --- --- @ --- -- for all p :: Char -> Bool --- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) --- @ --- --- @since 2.0.1 -spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) -spanM p t@(Text arr off len) = go 0 - where - go !i | i < len = case iterArray arr (off+i) of - Iter c l -> do - continue <- p c - if continue then go (i+l) - else pure (text arr off i, text arr (off+i) (len-i)) - go _ = pure (t, empty) -{-# INLINE spanM #-} - --- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, --- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of --- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. --- --- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 --- (("tuv","xyz"),118) --- --- @ --- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p --- @ --- --- @since 2.0.1 -spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) -spanEndM p t@(Text arr off len) = go (len-1) - where - go !i | 0 <= i = case reverseIterArray arr (off+i) of - Iter c l -> do - continue <- p c - if continue then go (i+l) - else pure (text arr off (i+1), text arr (off+i+1) (len-i-1)) - go _ = pure (empty, t) -{-# INLINE spanEndM #-} - --- | /O(n)/ Group characters in a string according to a predicate. -groupBy :: (Char -> Char -> Bool) -> Text -> [Text] -groupBy p = loop - where - loop t@(Text arr off len) - | null t = [] - | otherwise = text arr off n : loop (text arr (off+n) (len-n)) - where Iter c d = iter t 0 - n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) - --- | Returns the /array/ index (in units of 'Word8') at which a --- character may be found. This is /not/ the same as the logical --- index returned by e.g. 'findIndex'. -findAIndexOrEnd :: (Char -> Bool) -> Text -> Int -findAIndexOrEnd q t@(Text _arr _off len) = go 0 - where go !i | i >= len || q c = i - | otherwise = go (i+d) - where Iter c d = iter t i - --- | /O(n)/ Group characters in a string by equality. -group :: Text -> [Text] -group = groupBy (==) - --- | /O(n)/ Return all initial segments of the given 'Text', shortest --- first. -inits :: Text -> [Text] -inits t = empty : case t of - Text arr off len -> - let loop i | i >= len = [] - | otherwise = let !j = i + iter_ t i in Text arr off j : loop j - in loop 0 - --- | /O(n)/ Return all final segments of the given 'Text', longest --- first. -tails :: Text -> [Text] -tails t | null t = [empty] - | otherwise = t : tails (unsafeTail t) - --- $split --- --- Splitting functions in this library do not perform character-wise --- copies to create substrings; they just construct new 'Text's that --- are slices of the original. - --- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' --- argument (which cannot be empty), consuming the delimiter. An empty --- delimiter is invalid, and will cause an error to be raised. --- --- Examples: --- --- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne" --- ["a","b","d","e"] --- --- >>> splitOn "aaa" "aaaXaaaXaaaXaaa" --- ["","X","X","X",""] --- --- >>> splitOn "x" "x" --- ["",""] --- --- and --- --- > intercalate s . splitOn s == id --- > splitOn (singleton c) == split (==c) --- --- (Note: the string @s@ to split on above cannot be empty.) --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -splitOn :: HasCallStack - => Text - -- ^ String to split on. If this string is empty, an error - -- will occur. - -> Text - -- ^ Input text. - -> [Text] -splitOn pat@(Text _ _ l) src@(Text arr off len) - | l <= 0 = emptyError "splitOn" - | isSingleton pat = split (== unsafeHead pat) src - | otherwise = go 0 (indices pat src) - where - go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs - go s _ = [text arr (s+off) (len-s)] -{-# INLINE [1] splitOn #-} - -{-# RULES -"TEXT splitOn/singleton -> split/==" [~1] forall c t. - splitOn (singleton c) t = split (==c) t - #-} - --- | /O(n)/ Splits a 'Text' into components delimited by separators, --- where the predicate returns True for a separator element. The --- resulting components do not contain the separators. Two adjacent --- separators result in an empty component in the output. eg. --- --- >>> split (=='a') "aabbaca" --- ["","","bb","c",""] --- --- >>> split (=='a') "" --- [""] -split :: (Char -> Bool) -> Text -> [Text] -split p t - | null t = [empty] - | otherwise = loop t - where loop s | null s' = [l] - | otherwise = l : loop (unsafeTail s') - where (# l, s' #) = span_ (not . p) s -{-# INLINE split #-} - --- | /O(n)/ Splits a 'Text' into components of length @k@. The last --- element may be shorter than the other chunks, depending on the --- length of the input. Examples: --- --- >>> chunksOf 3 "foobarbaz" --- ["foo","bar","baz"] --- --- >>> chunksOf 4 "haskell.org" --- ["hask","ell.","org"] -chunksOf :: Int -> Text -> [Text] -chunksOf k = go - where - go t = case splitAt k t of - (a,b) | null a -> [] - | otherwise -> a : go b -{-# INLINE chunksOf #-} - --- ---------------------------------------------------------------------------- --- * Searching - -------------------------------------------------------------------------------- --- ** Searching with a predicate - --- | /O(n)/ The 'elem' function takes a character and a 'Text', and --- returns 'True' if the element is found in the given 'Text', or --- 'False' otherwise. -elem :: Char -> Text -> Bool -elem c t = S.any (== c) (stream t) -{-# INLINE elem #-} - --- | /O(n)/ The 'find' function takes a predicate and a 'Text', and --- returns the first element matching the predicate, or 'Nothing' if --- there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.findBy p (stream t) -{-# INLINE find #-} - --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', --- and returns the pair of 'Text's with elements which do and do not --- satisfy the predicate, respectively; i.e. --- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} - --- | /O(n)/ 'filter', applied to a predicate and a 'Text', --- returns a 'Text' containing those characters that satisfy the --- predicate. -filter :: (Char -> Bool) -> Text -> Text -filter p = filter_ text p -{-# INLINE [1] filter #-} - -{-# RULES -"TEXT filter/filter -> filter" forall p q t. - filter p (filter q t) = filter (\c -> q c && p c) t -#-} - --- | /O(n+m)/ Find the first instance of @needle@ (which must be --- non-'null') in @haystack@. The first element of the returned tuple --- is the prefix of @haystack@ before @needle@ is matched. The second --- is the remainder of @haystack@, starting with the match. --- --- Examples: --- --- >>> breakOn "::" "a::b::c" --- ("a","::b::c") --- --- >>> breakOn "/" "foobar" --- ("foobar","") --- --- Laws: --- --- > append prefix match == haystack --- > where (prefix, match) = breakOn needle haystack --- --- If you need to break a string by a substring repeatedly (e.g. you --- want to break on every instance of a substring), use 'breakOnAll' --- instead, as it has lower startup overhead. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -breakOn :: HasCallStack => Text -> Text -> (Text, Text) -breakOn pat src@(Text arr off len) - | null pat = emptyError "breakOn" - | otherwise = case indices pat src of - [] -> (src, empty) - (x:_) -> (text arr off x, text arr (off+x) (len-x)) -{-# INLINE breakOn #-} - --- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the --- string. --- --- The first element of the returned tuple is the prefix of @haystack@ --- up to and including the last match of @needle@. The second is the --- remainder of @haystack@, following the match. --- --- >>> breakOnEnd "::" "a::b::c" --- ("a::b::","c") -breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) -breakOnEnd pat src = (reverse b, reverse a) - where (a,b) = breakOn (reverse pat) (reverse src) -{-# INLINE breakOnEnd #-} - --- | /O(n+m)/ Find all non-overlapping instances of @needle@ in --- @haystack@. Each element of the returned list consists of a pair: --- --- * The entire string prior to the /k/th match (i.e. the prefix) --- --- * The /k/th match, followed by the remainder of the string --- --- Examples: --- --- >>> breakOnAll "::" "" --- [] --- --- >>> breakOnAll "/" "a/b/c/" --- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")] --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. --- --- The @needle@ parameter may not be empty. -breakOnAll :: HasCallStack - => Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -breakOnAll pat src@(Text arr off slen) - | null pat = emptyError "breakOnAll" - | otherwise = L.map step (indices pat src) - where - step x = (chunk 0 x, chunk x (slen-x)) - chunk !n !l = text arr (n+off) l -{-# INLINE breakOnAll #-} - -------------------------------------------------------------------------------- --- ** Indexing 'Text's - --- $index --- --- If you think of a 'Text' value as an array of 'Char' values (which --- it is not), you run the risk of writing inefficient code. --- --- An idiom that is common in some languages is to find the numeric --- offset of a character or substring, then use that number to split --- or trim the searched string. With a 'Text' value, this approach --- would require two /O(n)/ operations: one to perform the search, and --- one to operate from wherever the search ended. --- --- For example, suppose you have a string that you want to split on --- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of --- searching for the index of @\"::\"@ and taking the substrings --- before and after that index, you would instead use @breakOnAll \"::\"@. - --- | /O(n)/ 'Text' index (subscript) operator, starting from 0. -index :: HasCallStack => Text -> Int -> Char -index t n = S.index (stream t) n -{-# INLINE index #-} - --- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' --- and returns the index of the first element in the 'Text' satisfying --- the predicate. -findIndex :: (Char -> Bool) -> Text -> Maybe Int -findIndex p t = S.findIndex p (stream t) -{-# INLINE findIndex #-} - --- | /O(n+m)/ The 'count' function returns the number of times the --- query string appears in the given 'Text'. An empty query string is --- invalid, and will cause an error to be raised. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -count :: HasCallStack => Text -> Text -> Int -count pat - | null pat = emptyError "count" - | isSingleton pat = countChar (unsafeHead pat) - | otherwise = L.length . indices pat -{-# INLINE [1] count #-} - -{-# RULES -"TEXT count/singleton -> countChar" [~1] forall c t. - count (singleton c) t = countChar c t - #-} - --- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. -countChar :: Char -> Text -> Int -countChar c t = S.countChar c (stream t) -{-# INLINE countChar #-} - -------------------------------------------------------------------------------- --- * Zipping - --- | /O(n)/ 'zip' takes two 'Text's and returns a list of --- corresponding pairs of bytes. If one input 'Text' is short, --- excess elements of the longer 'Text' are discarded. This is --- equivalent to a pair of 'unpack' operations. -zip :: Text -> Text -> [(Char,Char)] -zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) -{-# INLINE zip #-} - --- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function --- given as the first argument, instead of a tupling function. --- Performs replacement on invalid scalar values. -zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) - where g a b = safe (f a b) -{-# INLINE [1] zipWith #-} - --- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's --- representing white space. -words :: Text -> [Text] -words (Text arr off len) = loop 0 0 - where - loop !start !n - | n >= len = if start == n - then [] - else [Text arr (start + off) (n - start)] - -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 - | isAsciiSpace w0 = - if start == n - then loop (n + 1) (n + 1) - else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) - | w0 < 0x80 = loop start (n + 1) - -- or 2 bytes for 0xA0 - | w0 == 0xC2, w1 == 0xA0 = - if start == n - then loop (n + 2) (n + 2) - else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) - | w0 < 0xE0 = loop start (n + 2) - -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 - | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 - || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) - || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = - if start == n - then loop (n + 3) (n + 3) - else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) - | otherwise = loop start (n + utf8LengthByLeader w0) - where - w0 = A.unsafeIndex arr (off + n) - w1 = A.unsafeIndex arr (off + n + 1) - w2 = A.unsafeIndex arr (off + n + 2) -{-# INLINE words #-} - --- Adapted from Data.ByteString.Internal.isSpaceWord8 -isAsciiSpace :: Word8 -> Bool -isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) -{-# INLINE isAsciiSpace #-} - --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters --- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. --- --- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. -lines :: Text -> [Text] -lines (Text arr@(A.ByteArray arr#) off len) = go off - where - go !n - | n >= len + off = [] - | delta < 0 = [Text arr n (len + off - n)] - | otherwise = Text arr n delta : go (n + delta + 1) - where - delta = memchr arr# n (len + off - n) 0x0A -{-# INLINE lines #-} - --- | /O(n)/ Joins lines, after appending a terminating newline to --- each. -unlines :: [Text] -> Text -unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] -{-# INLINE unlines #-} - --- | /O(n)/ Joins words using single space characters. -unwords :: [Text] -> Text -unwords = intercalate (singleton ' ') -{-# INLINE unwords #-} - --- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' if and only if the first is a prefix of the second. -isPrefixOf :: Text -> Text -> Bool -isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) = - alen <= blen && S.isPrefixOf (stream a) (stream b) -{-# INLINE [1] isPrefixOf #-} - --- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns --- 'True' if and only if the first is a suffix of the second. -isSuffixOf :: Text -> Text -> Bool -isSuffixOf a@(Text _aarr _aoff alen) b@(Text barr boff blen) = - d >= 0 && a == b' - where d = blen - alen - b' | d == 0 = b - | otherwise = Text barr (boff+d) alen -{-# INLINE isSuffixOf #-} - --- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns --- 'True' if and only if the first is contained, wholly and intact, anywhere --- within the second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -isInfixOf :: -#if defined(ASSERTS) - HasCallStack => -#endif - Text -> Text -> Bool -isInfixOf needle haystack - | null needle = True - | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack - | otherwise = not . L.null . indices needle $ haystack -{-# INLINE [1] isInfixOf #-} - -------------------------------------------------------------------------------- --- * View patterns - --- | /O(n)/ Return the suffix of the second string if its prefix --- matches the entire first string. --- --- Examples: --- --- >>> stripPrefix "foo" "foobar" --- Just "bar" --- --- >>> stripPrefix "" "baz" --- Just "baz" --- --- >>> stripPrefix "foo" "quux" --- Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text as T --- > --- > fnordLength :: Text -> Int --- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf --- > fnordLength _ = -1 -stripPrefix :: Text -> Text -> Maybe Text -stripPrefix p@(Text _arr _off plen) t@(Text arr off len) - | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) - | otherwise = Nothing - --- | /O(n)/ Find the longest non-empty common prefix of two strings --- and return it, along with the suffixes of each string at which they --- no longer match. --- --- If the strings do not have a common prefix or either one is empty, --- this function returns 'Nothing'. --- --- Examples: --- --- >>> commonPrefixes "foobar" "fooquux" --- Just ("foo","bar","quux") --- --- >>> commonPrefixes "veeble" "fetzer" --- Nothing --- --- >>> commonPrefixes "" "baz" --- Nothing -commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) -commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) - | len0 == 0 = Nothing - | len1 == 0 = Nothing - | otherwise = go 0 0 - where - go !i !j - | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) - | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) - | a == b = go (i + 1) k - | k > 0 = Just (Text arr0 off0 k, - Text arr0 (off0 + k) (len0 - k), - Text arr1 (off1 + k) (len1 - k)) - | otherwise = Nothing - where - a = A.unsafeIndex arr0 (off0 + i) - b = A.unsafeIndex arr1 (off1 + i) - isLeader = word8ToInt8 a >= -64 - k = if isLeader then i else j -{-# INLINE commonPrefixes #-} - --- | /O(n)/ Return the prefix of the second string if its suffix --- matches the entire first string. --- --- Examples: --- --- >>> stripSuffix "bar" "foobar" --- Just "foo" --- --- >>> stripSuffix "" "baz" --- Just "baz" --- --- >>> stripSuffix "foo" "quux" --- Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text as T --- > --- > quuxLength :: Text -> Int --- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre --- > quuxLength _ = -1 -stripSuffix :: Text -> Text -> Maybe Text -stripSuffix p@(Text _arr _off plen) t@(Text arr off len) - | p `isSuffixOf` t = Just $! text arr off (len-plen) - | otherwise = Nothing - --- | Add a list of non-negative numbers. Errors out on overflow. -sumP :: String -> [Int] -> Int -sumP fun = L.foldl' add 0 - where add a x - | ax >= 0 = ax - | otherwise = overflowError fun - where ax = a + x -{-# INLINE sumP #-} -- Use foldl' and inline for fusion. - -emptyError :: HasCallStack => String -> a -emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" - -overflowError :: HasCallStack => String -> a -overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" - --- | /O(n)/ Make a distinct copy of the given string, sharing no --- storage with the original string. --- --- As an example, suppose you read a large string, of which you need --- only a small portion. If you do not use 'copy', the entire original --- array will be kept alive in memory by the smaller string. Making a --- copy \"breaks the link\" to the original array, allowing it to be --- garbage collected if there are no other live references to it. -copy :: Text -> Text -copy t@(Text arr off len) - | null t = empty - | otherwise = Text (A.run go) 0 len - where - go :: ST s (A.MArray s) - go = do - marr <- A.new len - A.copyI len marr 0 arr off - return marr - -ord8 :: Char -> Word8 -ord8 = P.fromIntegral . Char.ord - -intToCSize :: Int -> CSize -intToCSize = P.fromIntegral - -cSsizeToInt :: CSsize -> Int -cSsizeToInt = P.fromIntegral - -word8ToInt8 :: Word8 -> Int8 -word8ToInt8 = P.fromIntegral - -------------------------------------------------- --- NOTE: the named chunk below used by doctest; --- verify the doctests via `doctest -fobject-code Data/Text.hs` - --- $setup --- >>> :set -XOverloadedStrings --- >>> import qualified Data.Text as T +{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text +-- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text. +-- Suitable for performance critical use, both in terms of large data +-- quantities and high speed. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text as T +-- +-- To use an extended and very rich family of functions for working +-- with Unicode text (including normalization, regular expressions, +-- non-standard encodings, text breaking, and locales), see the +-- . +-- + +module Data.Text + ( + -- * Strict vs lazy types + -- $strict + + -- * Acceptable data + -- $replacement + + -- * Definition of character + -- $character_definition + + -- * Fusion + -- $fusion + + -- * Types + Text + , StrictText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr' + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , replicate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , breakOn + , breakOnEnd + , break + , span + , spanM + , spanEndM + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + + -- ** Breaking into lines and words + , lines + --, lines' + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , breakOnAll + , find + , elem + , partition + + -- , findSubstring + + -- * Indexing + -- $index + , index + , findIndex + , count + + -- * Zipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + + -- * Low level operations + , copy + , unpackCString# + , unpackCStringAscii# + + , measureOff + ) where + +import Prelude (Char, Bool(..), Int, Maybe(..), String, + Eq, (==), (/=), Ord(..), Ordering(..), (++), + Monad(..), pure, Read(..), Show, + (&&), (||), (+), (-), (.), ($), ($!), (>>), + not, return, otherwise, quot) +import Control.DeepSeq (NFData(rnf)) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits ((.&.)) +import qualified Data.Char as Char +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Control.Monad (foldM) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import qualified Data.List as L hiding (head, tail) +import qualified Data.List.NonEmpty as NonEmptyList +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import Data.Text.Internal.ArrayUtils (memchr) +import Data.Text.Internal.IsAscii (isAscii) +import Data.Text.Internal.Reverse (reverse) +import Data.Text.Internal.Measure (measure_off) +import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, unstream) +import Data.Text.Internal.Private (span_) +import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, + reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray) +import Data.Text.Internal.Search (indices) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_) +#if defined(__HADDOCK__) +import Data.ByteString (ByteString) +import qualified Data.Text.Lazy as L +#endif +import Data.Word (Word8) +import Foreign.C.Types +import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) +import qualified GHC.Exts as Exts +import GHC.Int (Int8) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Lib as TH +import qualified Language.Haskell.TH.Syntax as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) +import System.Posix.Types (CSsize(..)) + +#if __GLASGOW_HASKELL__ >= 810 +import Data.Text.Foreign (asForeignPtr) +import System.IO.Unsafe (unsafePerformIO) +#endif + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +-- $character_definition +-- +-- This package uses the term /character/ to denote Unicode /code points/. +-- +-- Note that this is not the same thing as a grapheme (e.g. a +-- composition of code points that form one visual symbol). For +-- instance, consider the grapheme \"ä\". This symbol has two +-- Unicode representations: a single code-point representation +-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point), +-- and a two code point representation @U+0061@ (the \"@A@\" code +-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point). + +-- $strict +-- +-- This package provides both strict and lazy 'Text' types. The +-- strict type is provided by the "Data.Text" module, while the lazy +-- type is provided by the "Data.Text.Lazy" module. Internally, the +-- lazy @Text@ type consists of a list of strict chunks. +-- +-- The strict 'Text' type requires that an entire string fit into +-- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of +-- streaming strings that are larger than memory using a small memory +-- footprint. In many cases, the overhead of chunked streaming makes +-- the lazy 'Data.Text.Lazy.Text' type slower than its strict +-- counterpart, but this is not always the case. Sometimes, the time +-- complexity of a function in one module may be different from the +-- other, due to their differing internal structures. +-- +-- Each module provides an almost identical API, with the main +-- difference being that the strict module uses 'Int' values for +-- lengths and counts, while the lazy module uses 'Data.Int.Int64' +-- lengths. + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- . +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- () +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- .) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +instance Eq Text where + Text arrA offA lenA == Text arrB offB lenB + | lenA == lenB = A.equal arrA offA arrB offB lenA + | otherwise = False + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +-- +-- Beware: @stimes@ will crash if the given number does not fit into +-- an @Int@. +instance Semigroup Text where + (<>) = append + + stimes howManyTimes + | howManyTimes < 0 = P.error "Data.Text.stimes: given number is negative!" + | otherwise = + let howManyTimesInt = P.fromIntegral howManyTimes :: Int + in if P.fromIntegral howManyTimesInt == howManyTimes && howManyTimesInt >= 0 + then replicate howManyTimesInt + else P.error "Data.Text.stimes: given number does not fit into an Int!" + + sconcat = concat . NonEmptyList.toList + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where rnf !_ = () + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (lengthWord8 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you +-- feel a mistake has been made, please feel free to submit +-- improvements. +-- +-- The original discussion is archived here: +-- +-- +-- The followup discussion that changed the behavior of 'Data.Set.Set' +-- and 'Data.Map.Map' is archived here: +-- + +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> P.error "gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 914 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> [| empty |] + _ -> + let + bytesQ = TH.liftAddrCompat ptr 0 (P.fromIntegral len) + lenQ = TH.liftIntCompat (P.fromIntegral len) + in [| unpackCStringLen# $bytesQ $lenQ |] +#elif __GLASGOW_HASKELL__ >= 810 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> TH.varE 'empty + _ -> + let + bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len) + lenQ = liftInt (P.fromIntegral len) + liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n))) + in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ +#else + lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +#if __GLASGOW_HASKELL__ >= 810 +unpackCStringLen# :: Exts.Addr# -> Int -> Text +unpackCStringLen# addr# l = Text ba 0 l + where + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Exts.Ptr addr#) l + A.unsafeFreeze marr +{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Text" [packConstr] + +-- | /O(n)/ Compare two 'Text' values lexicographically. +compareText :: Text -> Text -> Ordering +compareText (Text arrA offA lenA) (Text arrB offB lenB) = + A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(n)/ Adds a character to the front of a 'Text'. This function +-- is more costly than its 'List' counterpart because it requires +-- copying a new array. Performs replacement on +-- invalid scalar values. +cons :: Char -> Text -> Text +cons c (Text srcArr srcOff srcLen) = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = chLen + srcLen + marr <- A.new totalLen + _ <- unsafeWrite marr 0 ch + A.copyI srcLen marr chLen srcArr srcOff + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +-- Performs replacement on invalid scalar values. +snoc :: Text -> Char -> Text +snoc (Text srcArr srcOff srcLen) c = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = srcLen + chLen + marr <- A.new totalLen + A.copyI srcLen marr 0 srcArr srcOff + _ <- unsafeWrite marr srcLen ch + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE snoc #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t + | null t = emptyError "head" + | otherwise = let Iter c _ = iter t 0 in c +{-# INLINE head #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons t@(Text arr off len) + | len <= 0 = Nothing + | otherwise = Just $ let !(Iter c d) = iter t 0 + in (c, text arr (off+d) (len-d)) +{-# INLINE [1] uncons #-} + +-- | /O(1)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last t@(Text _ _ len) + | null t = emptyError "last" + | otherwise = let Iter c _ = reverseIter t (len - 1) in c +{-# INLINE [1] last #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail t@(Text arr off len) + | null t = emptyError "tail" + | otherwise = text arr (off+d) (len-d) + where d = iter_ t 0 +{-# INLINE [1] tail #-} + +-- | /O(1)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init t@(Text arr off len) + | null t = emptyError "init" + | otherwise = text arr off (len + reverseIter_ t (len - 1)) +{-# INLINE [1] init #-} + +-- | /O(1)/ Returns all but the last character and the last character of a +-- 'Text', or 'Nothing' if empty. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc t@(Text arr off len) + | null t = Nothing + | otherwise = Just (text arr off (len + d), c) + where + Iter c d = reverseIter t (len - 1) +{-# INLINE [1] unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null (Text _arr _off len) = +#if defined(ASSERTS) + assert (len >= 0) $ +#endif + len <= 0 +{-# INLINE [1] null #-} + +{-# RULES + "TEXT null/empty -> True" null empty = True +#-} + +-- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/), +-- to be used together with '(:<)' or '(:>)'. +-- +-- @since 2.1.2 +pattern Empty :: Text +pattern Empty <- (null -> True) where + Empty = empty + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton (Text arr off len) = + len /= 0 && len == utf8LengthByLeader (A.unsafeIndex arr off) +{-# INLINE isSingleton #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Int +length = P.negate . measureOff P.maxBound +{-# INLINE [1] length #-} +-- length needs to be phased after the compareN/length rules otherwise +-- it may inline before the rules have an opportunity to fire. + +{-# RULES +"TEXT length/filter -> S.length/S.filter" forall p t. + length (filter p t) = S.length (S.filter p (stream t)) +"TEXT length/unstream -> S.length" forall t. + length (unstream t) = S.length t +"TEXT length/pack -> P.length" forall t. + length (pack t) = P.length t +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = mul (max 0 n) (length t) +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (mul 2 (length t) - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) +"TEXT length/empty -> 0" + length empty = 0 + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +{-# RULES +"TEXT compareN/length -> compareLength" [~1] forall t n. + compare (length t) n = compareLength t n + #-} + +{-# RULES +"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. + eqInt (length t) n = compareLength t n == EQ + #-} + +{-# RULES +"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. + neInt (length t) n = compareLength t n /= EQ + #-} + +{-# RULES +"TEXT compareLength/==LT" [~1] forall t n. + ltInt (length t) n = compareLength t n == LT + #-} + +{-# RULES +"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. + leInt (length t) n = compareLength t n /= GT + #-} + +{-# RULES +"TEXT >N/length -> compareLength/==GT" [~1] forall t n. + gtInt (length t) n = compareLength t n == GT + #-} + +{-# RULES +"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. + geInt (length t) n = compareLength t n /= LT + #-} + +-- ----------------------------------------------------------------------------- +-- * Transformations +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- +-- Example: +-- +-- >>> let message = pack "I am not angry. Not at all." +-- >>> T.map (\c -> if c == '.' then '!' else c) message +-- "I am not angry! Not at all!" +-- +-- Performs replacement on invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = \t -> if null t then empty else mapNonEmpty f t +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +-- +-- Example: +-- +-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] +-- "WeNI!seekNI!theNI!HolyNI!Grail" +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. +-- +-- Example: +-- +-- >>> T.intersperse '.' "SHIELD" +-- "S.H.I.E.L.D" +-- +-- Performs replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t@(Text src o l) = if null t then empty else runST $ do + let !cLen = utf8Length c + dstLen = l + length t P.* cLen + + dst <- A.new dstLen + + let writeSep = case cLen of + 1 -> \dstOff -> + A.unsafeWrite dst dstOff (ord8 c) + 2 -> let (c0, c1) = ord2 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + A.unsafeWrite dst (dstOff + 3) c3 + let go !srcOff !dstOff = if srcOff >= o + l then return () else do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff m0 + writeSep (dstOff + 1) + go (srcOff + 1) (dstOff + 1 + cLen) + 2 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + writeSep (dstOff + 2) + go (srcOff + 2) (dstOff + 2 + cLen) + 3 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + writeSep (dstOff + 3) + go (srcOff + 3) (dstOff + 3 + cLen) + _ -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + writeSep (dstOff + 4) + go (srcOff + 4) (dstOff + 4 + cLen) + + go o 0 + arr <- A.unsafeFreeze dst + return (Text arr 0 (dstLen - cLen)) +{-# INLINE [1] intersperse #-} + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- >>> replace "oo" "foo" "oo" +-- "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- >>> replace "ofo" "bar" "ofofo" +-- "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace needle@(Text _ _ neeLen) + (Text repArr repOff repLen) + haystack@(Text hayArr hayOff hayLen) + | neeLen == 0 = emptyError "replace" + | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it + | L.null ixs = haystack + | otherwise = Text (A.run x) 0 len + where + ixs = indices needle haystack + len = hayLen - (neeLen - repLen) `mul` L.length ixs + x :: ST s (A.MArray s) + x = do + marr <- A.new len + let loop (i:is) o d = do + let d0 = d + i - o + d1 = d0 + repLen + A.copyI (i - o) marr d hayArr (hayOff+o) + A.copyI repLen marr d0 repArr repOff + loop is (i + neeLen) d1 + loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) + loop ixs 0 0 + return marr + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- When case converting 'Text' values, do not use combinators like +-- @map toUpper@ to case convert each character of a string +-- individually, as this gives incorrect results according to the +-- rules of some writing systems. The whole-string case conversion +-- functions from this module, such as @toUpper@, obey the correct +-- case conversion rules. As a result, these functions may map one +-- input character to two or three output characters. For examples, +-- see the documentation of each function. +-- +-- /Note/: In some languages, case conversion is a locale- and +-- context-dependent operation. The case conversion functions in this +-- module are /not/ locale sensitive. Programs that require locale +-- sensitivity should use appropriate versions of the +-- . + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (also known +-- as case insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case +-- folded to the sequence \"մ\" (men, U+0574) followed by +-- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, +-- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) +-- instead of itself. +toCaseFold :: Text -> Text +toCaseFold = \t -> + if null t then empty + else toCaseFoldNonEmpty t +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, \"İ\" (Latin capital letter I with dot above, +-- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) +-- followed by \" ̇\" (combining dot above, U+0307). +toLower :: Text -> Text +toLower = \t -> + if null t then empty + else toLowerNonEmpty t +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German \"ß\" (eszett, U+00DF) maps to the +-- two-letter sequence \"SS\". +toUpper :: Text -> Text +toUpper = \t -> + if null t then empty + else toUpperNonEmpty t +{-# INLINE toUpper #-} + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = \t -> + if null t then empty + else toTitleNonEmpty t +{-# INLINE toTitle #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. +-- Performs replacement on invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyLeft 7 'x' "foo" +-- "fooxxxx" +-- +-- >>> justifyLeft 3 'x' "foobar" +-- "foobar" +justifyLeft :: Int -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChar (k-len) c + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyRight 7 'x' "bar" +-- "xxxxbar" +-- +-- >>> justifyRight 3 'x' "foobar" +-- "foobar" +justifyRight :: Int -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChar (k-len) c `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- >>> center 8 'x' "HS" +-- "xxxHSxxx" +center :: Int -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChar l c `append` t `append` replicateChar r c + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +-- +-- Examples: +-- +-- >>> transpose ["green","orange"] +-- ["go","rr","ea","en","ng","e"] +-- +-- >>> transpose ["blue","red"] +-- ["br","le","ud","e"] +transpose :: [Text] -> [Text] +transpose ts = P.map pack (L.transpose (P.map unpack ts)) + +-- ----------------------------------------------------------------------------- +-- * Reducing 'Text's (folds) + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z (Text arr off len) = go (off + len - 1) + where + go !i + | i < off = z + | otherwise = let !(Iter c l) = reverseIterArray arr i in f (go (i + l)) c +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) (f acc c) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = case uncons t of + Nothing -> emptyError "foldl" + Just (c, t') -> foldl f c t' +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = case uncons t of + Nothing -> emptyError "foldl'" + Just (c, t') -> foldl' f c t' +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = pure acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) P.=<< f acc c +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- If the binary operator is strict in its second argument, use 'foldr'' +-- instead. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +-- +-- Searches from left to right with short-circuiting behavior can +-- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem'). +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = case unsnoc t of + Nothing -> emptyError "foldr1" + Just (t', c) -> foldr f c t' +{-# INLINE foldr1 #-} + +-- | /O(n)/ A strict version of 'foldr'. +-- +-- 'foldr'' evaluates as a right-to-left traversal using constant stack space. +-- +-- @since 2.0.1 +foldr' :: (Char -> a -> a) -> a -> Text -> a +foldr' f z (Text arr off len) = go (off + len - 1) z + where + go !i !acc + | i < off = acc + | otherwise = let !(Iter c l) = reverseIterArray arr i in go (i + l) (f c acc) +{-# INLINE foldr' #-} + +-- ----------------------------------------------------------------------------- +-- ** Special folds + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat ts = case ts of + [] -> empty + [t] -> t + _ | len == 0 -> empty + | otherwise -> Text (A.run go) 0 len + where + len = sumP "concat" $ L.map lengthWord8 ts + go :: ST s (A.MArray s) + go = do + arr <- A.new len + let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) + foldM step 0 ts >> return arr + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p = foldr (\c acc -> p c || acc) False +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p = foldr (\c acc -> p c && acc) True +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum = foldl1' max +-- This could be implemented faster: look for the longest +-- and largest UTF-8 sequence, then decode it to Char only once, +-- instead of decoding all characters, but I doubt anyone cares +-- about the performance of 'maximum' much. +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum = foldl1' min +-- This could be implemented faster, see the comment for 'maximum' above. +{-# INLINE minimum #-} + +-- ----------------------------------------------------------------------------- +-- * Building 'Text's +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + marr <- A.new l' + d' <- unsafeWrite marr 0 c0' + outer marr l' o d' c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Char -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !c + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + pure $ Text arr 0 dstOff + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff c + | otherwise = do + let !(Iter c' d) = iterArray src srcOff + c'' = safe $ f c c' + d' <- unsafeWrite dst dstOff c'' + inner (srcOff + d) (dstOff + d') c'' + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t | null t = empty + | otherwise = scanl f (unsafeHead t) (unsafeTail t) +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + !d' = utf8Length c0' + marr <- A.new l' + _ <- unsafeWrite marr (l' - d') c0' + outer marr (l + o - 1) (l' - d' - 1) c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Char -> ST s Text + outer !dst = inner + where + inner !srcOff !dstOff !c + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + pure $ Text arr (dstOff + 1) (dstLen - dstOff - 1) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) c + | otherwise = do + let !(Iter c' d) = reverseIterArray src srcOff + c'' = safe $ f c' c + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' c'' + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) +{-# INLINE scanr1 #-} + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumL f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !z + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (z, Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff z + | otherwise = do + let !(Iter c d) = iterArray src srcOff + (z', c') = f z c + d' <- unsafeWrite dst dstOff (safe c') + inner (srcOff + d) (dstOff + d') z' + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. +-- Performs replacement on invalid scalar values. +mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumR f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + o - 1) (l + 4 - 1) z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) + outer !dst = inner + where + inner !srcOff !dstOff !z + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) z + | otherwise = do + let !(Iter c d) = reverseIterArray src srcOff + (z', c') = f z c + c'' = safe c' + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' z' + +-- ----------------------------------------------------------------------------- +-- ** Generating and unfolding 'Text's + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int -> Text -> Text +replicate n t@(Text a o l) + | n <= 0 || l <= 0 = empty + | n == 1 = t + | isSingleton t = replicateChar n (unsafeHead t) + | otherwise = runST $ do + let totalLen = n `mul` l + marr <- A.new totalLen + A.copyI l marr 0 a o + A.tile marr l + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen +{-# INLINE [1] replicate #-} + +{-# RULES +"TEXT replicate/singleton -> replicateChar" [~1] forall n c. + replicate n (singleton c) = replicateChar n c + #-} + +-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the +-- value of every element. +replicateChar :: Int -> Char -> Text +replicateChar !len !c' + | len <= 0 = empty + | Char.isAscii c = runST $ do + marr <- A.newFilled len (Char.ord c) + arr <- A.unsafeFreeze marr + return $ Text arr 0 len + | otherwise = runST $ do + let cLen = utf8Length c + totalLen = cLen P.* len + marr <- A.new totalLen + _ <- unsafeWrite marr 0 c + A.tile marr cLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen + where + c = safe c' +{-# INLINE replicateChar #-} + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- ----------------------------------------------------------------------------- +-- * Substrings + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int -> Text -> Text +take n t@(Text arr off len) + | n <= 0 = empty + | n >= len || m >= len || m < 0 = t + | otherwise = Text arr off m + where + m = measureOff n t +{-# INLINE [1] take #-} + +-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ +-- returns a non-negative number, measuring their size in 'Word8'. Otherwise, +-- if @t@ is shorter, return a non-positive number, which is a negated total count +-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. +-- +-- This function is used to implement 'take', 'drop', 'splitAt' and 'length' +-- and is useful on its own in streaming and parsing libraries. +-- +-- @since 2.0 +measureOff :: Int -> Text -> Int +measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else + cSsizeToInt $ + measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> takeEnd 3 "foobar" +-- "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int -> Text -> Text +takeEnd n t@(Text arr off len) + | n <= 0 = empty + | n >= len = t + | otherwise = text arr (off+i) (len-i) + where i = iterNEnd n t + +iterNEnd :: Int -> Text -> Int +iterNEnd n t@(Text _arr _off len) = loop (len-1) n + where loop i !m + | m <= 0 = i+1 + | i <= 0 = 0 + | otherwise = loop (i+d) (m-1) + where d = reverseIter_ t i + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int -> Text -> Text +drop n t@(Text arr off len) + | n <= 0 = t + | n >= len || m >= len || m < 0 = empty + | otherwise = Text arr (off+m) (len-m) + where m = measureOff n t +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> dropEnd 3 "foobar" +-- "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int -> Text -> Text +dropEnd n t@(Text arr off len) + | n <= 0 = t + | n >= len = empty + | otherwise = text arr off (iterNEnd n t) + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t@(Text arr off len) = loop 0 + where loop !i | i >= len = t + | p c = loop (i+d) + | otherwise = text arr off i + where Iter c d = iter t i +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- >>> takeWhileEnd (=='o') "foo" +-- "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = t + | p c = loop (i+d) (l+d) + | otherwise = text arr (off+l) (len-l) + where Iter c d = reverseIter t i +{-# INLINE [1] takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t@(Text arr off len) = loop 0 0 + where loop !i !l | l >= len = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr (off+i) (len-l) + where Iter c d = iter t i +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- >>> dropWhileEnd (=='.') "foo..." +-- "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr off l + where Iter c d = reverseIter t i +{-# INLINE [1] dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile Char.isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd Char.isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround Char.isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int -> Text -> (Text, Text) +splitAt n t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len || m < 0 = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m = measureOff n t + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p t = case span_ p t of + (# hd,tl #) -> (hd,tl) +{-# INLINE span #-} + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is +-- over elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p = span (not . p) +{-# INLINE break #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t@(Text arr off len) = go 0 + where + go !i | i < len = case iterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off i, text arr (off+i) (len-i)) + go _ = pure (t, empty) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t@(Text arr off len) = go (len-1) + where + go !i | 0 <= i = case reverseIterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off (i+1), text arr (off+i+1) (len-i-1)) + go _ = pure (empty, t) +{-# INLINE spanEndM #-} + +-- | /O(n)/ Group characters in a string according to a predicate. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy p = loop + where + loop t@(Text arr off len) + | null t = [] + | otherwise = text arr off n : loop (text arr (off+n) (len-n)) + where Iter c d = iter t 0 + n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) + +-- | Returns the /array/ index (in units of 'Word8') at which a +-- character may be found. This is /not/ the same as the logical +-- index returned by e.g. 'findIndex'. +findAIndexOrEnd :: (Char -> Bool) -> Text -> Int +findAIndexOrEnd q t@(Text _arr _off len) = go 0 + where go !i | i >= len || q c = i + | otherwise = go (i+d) + where Iter c d = iter t i + +-- | /O(n)/ Group characters in a string by equality. +group :: Text -> [Text] +group = groupBy (==) + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +inits :: Text -> [Text] +inits = (NonEmptyList.toList $!) . initsNE + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmptyList.NonEmpty Text +initsNE t = empty NonEmptyList.:| case t of + Text arr off len -> + let loop i + | i >= len = [] + | otherwise = let !j = i + iter_ t i in Text arr off j : loop j + in loop 0 + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NonEmptyList.toList $!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmptyList.NonEmpty Text +tailsNE t + | null t = empty NonEmptyList.:| [] + | otherwise = t NonEmptyList.:| tails (unsafeTail t) + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be empty), consuming the delimiter. An empty +-- delimiter is invalid, and will cause an error to be raised. +-- +-- Examples: +-- +-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne" +-- ["a","b","d","e"] +-- +-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa" +-- ["","X","X","X",""] +-- +-- >>> splitOn "x" "x" +-- ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "splitOn" + | isSingleton pat = split (== unsafeHead pat) src + | otherwise = go 0 (indices pat src) + where + go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs + go s _ = [text arr (s+off) (len-s)] +{-# INLINE [1] splitOn #-} + +{-# RULES +"TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- >>> split (=='a') "aabbaca" +-- ["","","bb","c",""] +-- +-- >>> split (=='a') "" +-- [""] +split :: (Char -> Bool) -> Text -> [Text] +split p t + | null t = [empty] + | otherwise = loop t + where loop s | null s' = [l] + | otherwise = l : loop (unsafeTail s') + where (# l, s' #) = span_ (not . p) s +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- >>> chunksOf 3 "foobarbaz" +-- ["foo","bar","baz"] +-- +-- >>> chunksOf 4 "haskell.org" +-- ["hask","ell.","org"] +chunksOf :: Int -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- ---------------------------------------------------------------------------- +-- * Searching + +------------------------------------------------------------------------------- +-- ** Searching with a predicate + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem = any . (==) +-- TODO This can be implemented much faster: there is no need to decode +-- any UTF-8 sequences at all. +{-# INLINE elem #-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element matching the predicate, or 'Nothing' if +-- there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p = foldr (\c acc -> if p c then Just c else acc) Nothing +{-# INLINE find #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = filter_ text p +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- >>> breakOn "::" "a::b::c" +-- ("a","::b::c") +-- +-- >>> breakOn "/" "foobar" +-- ("foobar","") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src@(Text arr off len) + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> (text arr off x, text arr (off+x) (len-x)) +{-# INLINE breakOn #-} + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the +-- string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- >>> breakOnEnd "::" "a::b::c" +-- ("a::b::","c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = (reverse b, reverse a) + where (a,b) = breakOn (reverse pat) (reverse src) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- >>> breakOnAll "::" "" +-- [] +-- +-- >>> breakOnAll "/" "a/b/c/" +-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")] +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src@(Text arr off slen) + | null pat = emptyError "breakOnAll" + | otherwise = L.map step (indices pat src) + where + step x = (chunk 0 x, chunk x (slen-x)) + chunk !n !l = text arr (n+off) l +{-# INLINE breakOnAll #-} + +------------------------------------------------------------------------------- +-- ** Indexing 'Text's + +-- $index +-- +-- If you think of a 'Text' value as an array of 'Char' values (which +-- it is not), you run the risk of writing inefficient code. +-- +-- An idiom that is common in some languages is to find the numeric +-- offset of a character or substring, then use that number to split +-- or trim the searched string. With a 'Text' value, this approach +-- would require two /O(n)/ operations: one to perform the search, and +-- one to operate from wherever the search ended. +-- +-- For example, suppose you have a string that you want to split on +-- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of +-- searching for the index of @\"::\"@ and taking the substrings +-- before and after that index, you would instead use @breakOnAll \"::\"@. + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int -> Char +index t@(Text _ _ lenInBytes) ix + | ix < 0 + = P.error $ "Data.Text.index: negative index " ++ P.show ix + | off < 0 || off == lenInBytes + = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + | otherwise = ch + where + off = measureOff ix t + Iter ch _ = iter t off +{-# INLINE index #-} + +-- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' +-- and returns the index of the first element in the 'Text' satisfying +-- the predicate. +findIndex :: (Char -> Bool) -> Text -> Maybe Int +findIndex p t = S.findIndex p (stream t) +{-# INLINE findIndex #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int +count pat + | null pat = emptyError "count" + | isSingleton pat = countChar (unsafeHead pat) + | otherwise = L.length . indices pat +{-# INLINE [1] count #-} + +{-# RULES +"TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int +countChar c = foldl' (\acc c' -> if c == c' then acc + 1 else acc) 0 +{-# INLINE countChar #-} + +------------------------------------------------------------------------------- +-- * Zipping + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [1] zipWith #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words (Text arr off len) = loop 0 0 + where + loop !start !n + | n >= len = if start == n + then [] + else [Text arr (start + off) (n - start)] + -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 + | isAsciiSpace w0 = + if start == n + then loop (n + 1) (n + 1) + else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) + | w0 < 0x80 = loop start (n + 1) + -- or 2 bytes for 0xA0 + | w0 == 0xC2, w1 == 0xA0 = + if start == n + then loop (n + 2) (n + 2) + else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) + | w0 < 0xE0 = loop start (n + 2) + -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 + | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 + || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) + || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = + if start == n + then loop (n + 3) (n + 3) + else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) + | otherwise = loop start (n + utf8LengthByLeader w0) + where + w0 = A.unsafeIndex arr (off + n) + w1 = A.unsafeIndex arr (off + n + 1) + w2 = A.unsafeIndex arr (off + n + 2) +{-# INLINE words #-} + +-- Adapted from Data.ByteString.Internal.isSpaceWord8 +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) +{-# INLINE isAsciiSpace #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines (Text arr@(A.ByteArray arr#) off len) = go off + where + go !n + | n >= len + off = [] + | delta < 0 = [Text arr n (len + off - n)] + | otherwise = Text arr n delta : go (n + delta + 1) + where + delta = memchr arr# n (len + off - n) 0x0A +{-# INLINE lines #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr bOff aLen +{-# INLINE [1] isPrefixOf #-} + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr (bOff+d) aLen +{-# INLINE isSuffixOf #-} + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripPrefix "foo" "foobar" +-- Just "bar" +-- +-- >>> stripPrefix "" "baz" +-- Just "baz" +-- +-- >>> stripPrefix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p@(Text _arr _off plen) t@(Text arr off len) + | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) + | otherwise = Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- >>> commonPrefixes "foobar" "fooquux" +-- Just ("foo","bar","quux") +-- +-- >>> commonPrefixes "veeble" "fetzer" +-- Nothing +-- +-- >>> commonPrefixes "" "baz" +-- Nothing +commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) +commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) + | len0 == 0 = Nothing + | len1 == 0 = Nothing + | otherwise = go 0 0 + where + go !i !j + | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) + | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) + | a == b = go (i + 1) k + | k > 0 = Just (Text arr0 off0 k, + Text arr0 (off0 + k) (len0 - k), + Text arr1 (off1 + k) (len1 - k)) + | otherwise = Nothing + where + a = A.unsafeIndex arr0 (off0 + i) + b = A.unsafeIndex arr1 (off1 + i) + isLeader = word8ToInt8 a >= -64 + k = if isLeader then i else j +{-# INLINE commonPrefixes #-} + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripSuffix "bar" "foobar" +-- Just "foo" +-- +-- >>> stripSuffix "" "baz" +-- Just "baz" +-- +-- >>> stripSuffix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p@(Text _arr _off plen) t@(Text arr off len) + | p `isSuffixOf` t = Just $! text arr off (len-plen) + | otherwise = Nothing + +-- | Add a list of non-negative numbers. Errors out on overflow. +sumP :: String -> [Int] -> Int +sumP fun = L.foldl' add 0 + where add a x + | ax >= 0 = ax + | otherwise = overflowError fun + where ax = a + x +{-# INLINE sumP #-} -- Use foldl' and inline for fusion. + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" + +overflowError :: HasCallStack => String -> a +overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" + +-- | Convert a value to 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +-- | /O(n)/ Make a distinct copy of the given string, sharing no +-- storage with the original string. +-- +-- As an example, suppose you read a large string, of which you need +-- only a small portion. If you do not use 'copy', the entire original +-- array will be kept alive in memory by the smaller string. Making a +-- copy \"breaks the link\" to the original array, allowing it to be +-- garbage collected if there are no other live references to it. +copy :: Text -> Text +copy t@(Text arr off len) + | null t = empty + | otherwise = Text (A.run go) 0 len + where + go :: ST s (A.MArray s) + go = do + marr <- A.new len + A.copyI len marr 0 arr off + return marr + +ord8 :: Char -> Word8 +ord8 = P.fromIntegral . Char.ord + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = P.fromIntegral + +word8ToInt8 :: Word8 -> Int8 +word8ToInt8 = P.fromIntegral + +------------------------------------------------- +-- NOTE: the named chunk below used by doctest; +-- verify the doctests via `doctest -fobject-code Data/Text.hs` + +-- $setup +-- >>> :set -XOverloadedStrings +-- >>> import qualified Data.Text as T diff --git a/src/Data/Text/Array.hs b/src/Data/Text/Array.hs --- a/src/Data/Text/Array.hs +++ b/src/Data/Text/Array.hs @@ -1,339 +1,361 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE CPP #-} -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE PatternSynonyms #-} -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE RecordWildCards #-} -{-# LANGUAGE UnboxedTuples #-} -{-# LANGUAGE UnliftedFFITypes #-} - -{-# OPTIONS_GHC -fno-warn-unused-matches #-} --- | --- Module : Data.Text.Array --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : portable --- --- Packed, unboxed, heap-resident arrays. Suitable for performance --- critical use, both in terms of large data quantities and high --- speed. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions, e.g. --- --- > import qualified Data.Text.Array as A --- --- The names in this module resemble those in the 'Data.Array' family --- of modules, but are shorter due to the assumption of qualified --- naming. -module Data.Text.Array - ( - -- * Types - Array - , pattern ByteArray - , MArray - , pattern MutableByteArray - -- * Functions - , resizeM - , shrinkM - , copyM - , copyI - , copyFromPointer - , copyToPointer - , empty - , equal - , compare - , run - , run2 - , toList - , unsafeFreeze - , unsafeIndex - , new - , newPinned - , newFilled - , unsafeWrite - , tile - , getSizeofMArray - ) where - -#if defined(ASSERTS) -import GHC.Stack (HasCallStack) -#endif -#if !MIN_VERSION_base(4,11,0) -import Foreign.C.Types (CInt(..)) -#endif -import GHC.Exts hiding (toList) -import GHC.ST (ST(..), runST) -import GHC.Word (Word8(..)) -import qualified Prelude -import Prelude hiding (length, read, compare) -import Data.Array.Byte (ByteArray(..), MutableByteArray(..)) - --- | Immutable array type. -type Array = ByteArray - --- | Mutable array type, for use in the ST monad. -type MArray = MutableByteArray - --- | Create an uninitialized mutable array. -new :: forall s. Int -> ST s (MArray s) -new (I# len#) -#if defined(ASSERTS) - | I# len# < 0 = error "Data.Text.Array.new: size overflow" -#endif - | otherwise = ST $ \s1# -> - case newByteArray# len# s1# of - (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) -{-# INLINE new #-} - --- | Create an uninitialized mutable pinned array. --- --- @since 2.0 -newPinned :: forall s. Int -> ST s (MArray s) -newPinned (I# len#) -#if defined(ASSERTS) - | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" -#endif - | otherwise = ST $ \s1# -> - case newPinnedByteArray# len# s1# of - (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) -{-# INLINE newPinned #-} - --- | @since 2.0 -newFilled :: Int -> Int -> ST s (MArray s) -newFilled (I# len#) (I# c#) = ST $ \s1# -> - case newByteArray# len# s1# of - (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of - s3# -> (# s3#, MutableByteArray marr# #) -{-# INLINE newFilled #-} - --- | @since 2.0 -tile :: MArray s -> Int -> ST s () -tile marr tileLen = do - totalLen <- getSizeofMArray marr - let go l - | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) - | otherwise = copyM marr l marr 0 l >> go (2 * l) - go tileLen -{-# INLINE tile #-} - --- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! -unsafeFreeze :: MArray s -> ST s Array -unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> - case unsafeFreezeByteArray# marr s1# of - (# s2#, ba# #) -> (# s2#, ByteArray ba# #) -{-# INLINE unsafeFreeze #-} - --- | Unchecked read of an immutable array. May return garbage or --- crash on an out-of-bounds access. -unsafeIndex :: -#if defined(ASSERTS) - HasCallStack => -#endif - Array -> Int -> Word8 -unsafeIndex (ByteArray arr) i@(I# i#) = -#if defined(ASSERTS) - let word8len = I# (sizeofByteArray# arr) in - if i < 0 || i >= word8len - then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) - else -#endif - case indexWord8Array# arr i# of r# -> (W8# r#) -{-# INLINE unsafeIndex #-} - --- | @since 2.0 -getSizeofMArray :: MArray s -> ST s Int -getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> - -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is - -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. - case getSizeofMutableByteArray# marr s0# of - (# s1#, word8len# #) -> (# s1#, I# word8len# #) - -#if defined(ASSERTS) -checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () -checkBoundsM ma i elSize = do - len <- getSizeofMArray ma - if i < 0 || i + elSize > len - then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) - else return () -#endif - --- | Unchecked write of a mutable array. May return garbage or crash --- on an out-of-bounds access. -unsafeWrite :: -#if defined(ASSERTS) - HasCallStack => -#endif - MArray s -> Int -> Word8 -> ST s () -unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = -#if defined(ASSERTS) - checkBoundsM ma i 1 >> -#endif - (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of - s2# -> (# s2#, () #)) -{-# INLINE unsafeWrite #-} - --- | Convert an immutable array to a list. -toList :: Array -> Int -> Int -> [Word8] -toList ary off len = loop 0 - where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) - | otherwise = [] - --- | An empty immutable array. -empty :: Array -empty = runST (new 0 >>= unsafeFreeze) - --- | Run an action in the ST monad and return an immutable array of --- its result. -run :: (forall s. ST s (MArray s)) -> Array -run k = runST (k >>= unsafeFreeze) - --- | Run an action in the ST monad and return an immutable array of --- its result paired with whatever else the action returns. -run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) -run2 k = runST (do - (marr,b) <- k - arr <- unsafeFreeze marr - return (arr,b)) -{-# INLINE run2 #-} - --- | @since 2.0 -resizeM :: MArray s -> Int -> ST s (MArray s) -resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> - case resizeMutableByteArray# ma i# s1# of - (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) -{-# INLINE resizeM #-} - --- | @since 2.0 -shrinkM :: -#if defined(ASSERTS) - HasCallStack => -#endif - MArray s -> Int -> ST s () -shrinkM (MutableByteArray marr) i@(I# newSize) = do -#if defined(ASSERTS) - oldSize <- getSizeofMArray (MutableByteArray marr) - if I# newSize > oldSize - then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) - else return () -#endif - ST $ \s1# -> - case shrinkMutableByteArray# marr newSize s1# of - s2# -> (# s2#, () #) -{-# INLINE shrinkM #-} - --- | Copy some elements of a mutable array. -copyM :: MArray s -- ^ Destination - -> Int -- ^ Destination offset - -> MArray s -- ^ Source - -> Int -- ^ Source offset - -> Int -- ^ Count - -> ST s () -copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) -#if defined(ASSERTS) - | count < 0 = error $ - "copyM: count must be >= 0, but got " ++ show count -#endif - | otherwise = do -#if defined(ASSERTS) - srcLen <- getSizeofMArray src - dstLen <- getSizeofMArray dst - if srcOff + count > srcLen - then error "copyM: source is too short" - else return () - if dstOff + count > dstLen - then error "copyM: destination is too short" - else return () -#endif - ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of - s2# -> (# s2#, () #) -{-# INLINE copyM #-} - --- | Copy some elements of an immutable array. -copyI :: Int -- ^ Count - -> MArray s -- ^ Destination - -> Int -- ^ Destination offset - -> Array -- ^ Source - -> Int -- ^ Source offset - -> ST s () -copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) -#if defined(ASSERTS) - | count < 0 = error $ - "copyI: count must be >= 0, but got " ++ show count -#endif - | otherwise = ST $ \s1# -> - case copyByteArray# src# srcOff# dst# dstOff# count# s1# of - s2# -> (# s2#, () #) -{-# INLINE copyI #-} - --- | Copy from pointer. --- --- @since 2.0 -copyFromPointer - :: MArray s -- ^ Destination - -> Int -- ^ Destination offset - -> Ptr Word8 -- ^ Source - -> Int -- ^ Count - -> ST s () -copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) -#if defined(ASSERTS) - | count < 0 = error $ - "copyFromPointer: count must be >= 0, but got " ++ show count -#endif - | otherwise = ST $ \s1# -> - case copyAddrToByteArray# src# dst# dstOff# count# s1# of - s2# -> (# s2#, () #) -{-# INLINE copyFromPointer #-} - --- | Copy to pointer. --- --- @since 2.0 -copyToPointer - :: Array -- ^ Source - -> Int -- ^ Source offset - -> Ptr Word8 -- ^ Destination - -> Int -- ^ Count - -> ST s () -copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) -#if defined(ASSERTS) - | count < 0 = error $ - "copyToPointer: count must be >= 0, but got " ++ show count -#endif - | otherwise = ST $ \s1# -> - case copyByteArrayToAddr# src# srcOff# dst# count# s1# of - s2# -> (# s2#, () #) -{-# INLINE copyToPointer #-} - --- | Compare portions of two arrays for equality. No bounds checking --- is performed. -equal :: Array -> Int -> Array -> Int -> Int -> Bool -equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 -{-# INLINE equal #-} - --- | Compare portions of two arrays. No bounds checking is performed. --- --- @since 2.0 -compare :: Array -> Int -> Array -> Int -> Int -> Ordering -compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 -{-# INLINE compare #-} - -compareInternal - :: Array -- ^ First - -> Int -- ^ Offset into first - -> Array -- ^ Second - -> Int -- ^ Offset into second - -> Int -- ^ Count - -> Int -compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i - where -#if MIN_VERSION_base(4,11,0) - i = I# (compareByteArrays# src1# off1# src2# off2# count#) -#else - i = fromIntegral (memcmp src1# off1# src2# off2# count#) - -foreign import ccall unsafe "_hs_text_memcmp2" memcmp - :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt -#endif -{-# INLINE compareInternal #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-unused-matches #-} +-- | +-- Module : Data.Text.Array +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Packed, unboxed, heap-resident arrays. Suitable for performance +-- critical use, both in terms of large data quantities and high +-- speed. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text.Array as A +-- +-- The names in this module resemble those in the 'Data.Array' family +-- of modules, but are shorter due to the assumption of qualified +-- naming. +module Data.Text.Array + ( + -- * Types + Array + , pattern ByteArray + , MArray + , pattern MutableByteArray + -- * Functions + , resizeM + , shrinkM + , copyM + , copyI + , copyFromPointer + , copyToPointer + , empty + , equal + , compare + , run + , run2 + , toList + , unsafeFreeze + , unsafeIndex + , new + , newPinned + , newFilled + , unsafeWrite + , tile + , getSizeofMArray + ) where + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif +#if !MIN_VERSION_base(4,11,0) +import Foreign.C.Types (CInt(..)) +#endif +import GHC.Exts hiding (toList) +import GHC.ST (ST(..), runST) +import GHC.Word (Word8(..)) +import qualified Prelude +import Prelude hiding (length, read, compare) +import Data.Array.Byte (ByteArray(..), MutableByteArray(..)) + +-- | Immutable array type. +type Array = ByteArray + +-- | Mutable array type, for use in the ST monad. +type MArray = MutableByteArray + +-- | Create an uninitialized mutable array. +new :: forall s. Int -> ST s (MArray s) +new (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.new: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE new #-} + +-- | Create an uninitialized mutable pinned array. +-- +-- @since 2.0 +newPinned :: forall s. Int -> ST s (MArray s) +newPinned (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newPinnedByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE newPinned #-} + +-- | @since 2.0 +newFilled :: Int -> Int -> ST s (MArray s) +newFilled (I# len#) (I# c#) = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of + s3# -> (# s3#, MutableByteArray marr# #) +{-# INLINE newFilled #-} + +-- | @since 2.0 +tile :: MArray s -> Int -> ST s () +tile marr tileLen = do + totalLen <- getSizeofMArray marr + let go l + | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) + | otherwise = copyM marr l marr 0 l >> go (2 * l) + go tileLen +{-# INLINE tile #-} + +-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! +unsafeFreeze :: MArray s -> ST s Array +unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> + case unsafeFreezeByteArray# marr s1# of + (# s2#, ba# #) -> (# s2#, ByteArray ba# #) +{-# INLINE unsafeFreeze #-} + +-- | Unchecked read of an immutable array. May return garbage or +-- crash on an out-of-bounds access. +unsafeIndex :: +#if defined(ASSERTS) + HasCallStack => +#endif + Array -> Int -> Word8 +unsafeIndex (ByteArray arr) i@(I# i#) = +#if defined(ASSERTS) + let word8len = I# (sizeofByteArray# arr) in + if i < 0 || i >= word8len + then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) + else +#endif + case indexWord8Array# arr i# of r# -> (W8# r#) +{-# INLINE unsafeIndex #-} + +-- | @since 2.0 +getSizeofMArray :: MArray s -> ST s Int +getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> + -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is + -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. + case getSizeofMutableByteArray# marr s0# of + (# s1#, word8len# #) -> (# s1#, I# word8len# #) + +#if defined(ASSERTS) +checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () +checkBoundsM ma i elSize = do + len <- getSizeofMArray ma + if i < 0 || i + elSize > len + then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) + else return () +#endif + +-- | Unchecked write of a mutable array. May return garbage or crash +-- on an out-of-bounds access. +unsafeWrite :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> Word8 -> ST s () +unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = +#if defined(ASSERTS) + checkBoundsM ma i 1 >> +#endif + (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of + s2# -> (# s2#, () #)) +{-# INLINE unsafeWrite #-} + +-- | Convert an immutable array to a list. +toList :: Array -> Int -> Int -> [Word8] +toList ary off len = loop 0 + where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) + | otherwise = [] + +-- | An empty immutable array. +empty :: Array +empty = runST (new 0 >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result. +run :: (forall s. ST s (MArray s)) -> Array +run k = runST (k >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result paired with whatever else the action returns. +run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) +run2 k = runST (do + (marr,b) <- k + arr <- unsafeFreeze marr + return (arr,b)) +{-# INLINE run2 #-} + +-- | @since 2.0 +resizeM :: MArray s -> Int -> ST s (MArray s) +resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> + case resizeMutableByteArray# ma i# s1# of + (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) +{-# INLINE resizeM #-} + +-- | @since 2.0 +shrinkM :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> ST s () +shrinkM (MutableByteArray marr) i@(I# newSize) = do +#if defined(ASSERTS) + oldSize <- getSizeofMArray (MutableByteArray marr) + if I# newSize > oldSize + then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) + else return () +#endif + ST $ \s1# -> + case shrinkMutableByteArray# marr newSize s1# of + s2# -> (# s2#, () #) +{-# INLINE shrinkM #-} + +-- | Copy some elements of a mutable array. +copyM :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> MArray s -- ^ Source + -> Int -- ^ Source offset + -> Int -- ^ Count + -> ST s () +copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyM: count must be >= 0, but got " ++ show count +#endif + | otherwise = do +#if defined(ASSERTS) + srcLen <- getSizeofMArray src + dstLen <- getSizeofMArray dst + if srcOff + count > srcLen + then error "copyM: source is too short" + else return () + if dstOff + count > dstLen + then error "copyM: destination is too short" + else return () +#endif + ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyM #-} + +-- | Copy some elements of an immutable array. +copyI :: Int -- ^ Count + -> MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Array -- ^ Source + -> Int -- ^ Source offset + -> ST s () +copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyI: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyI #-} + +-- | Copy from pointer. +-- +-- @since 2.0 +copyFromPointer + :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Ptr Word8 -- ^ Source + -> Int -- ^ Count + -> ST s () +copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyFromPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyAddrToByteArray# src# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyFromPointer #-} + +-- | Copy to pointer. +-- +-- @since 2.0 +copyToPointer + :: Array -- ^ Source + -> Int -- ^ Source offset + -> Ptr Word8 -- ^ Destination + -> Int -- ^ Count + -> ST s () +copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyToPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArrayToAddr# src# srcOff# dst# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyToPointer #-} + +-- | Compare portions of two arrays for equality. No bounds checking +-- is performed. +equal + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Bool +equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 +{-# INLINE equal #-} + +-- | Compare portions of two arrays. No bounds checking is performed. +-- +-- @since 2.0 +compare + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Ordering +compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 +{-# INLINE compare #-} + +compareInternal + :: Array -- ^ First + -> Int -- ^ Offset into first + -> Array -- ^ Second + -> Int -- ^ Offset into second + -> Int -- ^ Count + -> Int +compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i + where +#if MIN_VERSION_base(4,11,0) + i = I# (compareByteArrays# src1# off1# src2# off2# count#) +#else + i = fromIntegral (memcmp src1# off1# src2# off2# count#) + +foreign import ccall unsafe "_hs_text_memcmp2" memcmp + :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt +#endif +{-# INLINE compareInternal #-} diff --git a/src/Data/Text/Encoding.hs b/src/Data/Text/Encoding.hs --- a/src/Data/Text/Encoding.hs +++ b/src/Data/Text/Encoding.hs @@ -1,578 +1,588 @@ -{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, - UnliftedFFITypes #-} -{-# LANGUAGE Trustworthy #-} -{-# LANGUAGE TypeApplications #-} -{-# LANGUAGE ScopedTypeVariables #-} --- | --- Module : Data.Text.Encoding --- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts, --- (c) 2008, 2009 Tom Harper --- (c) 2021 Andrew Lelechenko --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : portable --- --- Functions for converting 'Text' values to and from 'ByteString', --- using several standard encodings. --- --- To gain access to a much larger family of encodings, use the --- . - -module Data.Text.Encoding - ( - -- * Decoding ByteStrings to Text - -- $strict - - -- ** Total Functions #total# - -- $total - decodeLatin1 - , decodeASCIIPrefix - , decodeUtf8Lenient - , decodeUtf8' - , decodeASCII' - - -- *** Controllable error handling - , decodeUtf8With - , decodeUtf16LEWith - , decodeUtf16BEWith - , decodeUtf32LEWith - , decodeUtf32BEWith - - -- *** Stream oriented decoding - -- $stream - , streamDecodeUtf8With - , Decoding(..) - - -- *** Incremental UTF-8 decoding - -- $incremental - , decodeUtf8Chunk - , decodeUtf8More - , Utf8State - , startUtf8State - , StrictBuilder - , strictBuilderToText - , textToStrictBuilder - - -- ** Partial Functions - -- $partial - , decodeASCII - , decodeUtf8 - , decodeUtf16LE - , decodeUtf16BE - , decodeUtf32LE - , decodeUtf32BE - - -- *** Stream oriented decoding - , streamDecodeUtf8 - - -- * Encoding Text to ByteStrings - , encodeUtf8 - , encodeUtf16LE - , encodeUtf16BE - , encodeUtf32LE - , encodeUtf32BE - - -- * Encoding Text using ByteString Builders - , encodeUtf8Builder - , encodeUtf8BuilderEscaped - - -- * ByteString validation - -- $validation - , validateUtf8Chunk - , validateUtf8More - ) where - -import Control.Exception (evaluate, try) -import Data.Word (Word8) -import GHC.Exts (byteArrayContents#, unsafeCoerce#) -import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) -import Data.ByteString (ByteString) -#if defined(PURE_HASKELL) -import Control.Monad.ST.Unsafe (unsafeSTToIO) -import Data.ByteString.Char8 (unpack) -import Data.Text.Internal (pack) -import Foreign.Ptr (minusPtr, plusPtr) -import Foreign.Storable (poke) -#else -import Control.Monad.ST (runST) -import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) -import Data.Bits (shiftR, (.&.)) -import Data.Text.Internal.ByteStringCompat (withBS) -import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) -import Foreign.C.Types (CSize(..)) -import Foreign.Ptr (Ptr, minusPtr, plusPtr) -import Foreign.Storable (poke, peekByteOff) -#endif -import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) -import Data.Text.Internal (Text(..), empty) -import Data.Text.Internal.Encoding -import Data.Text.Internal.IsAscii (asciiPrefixLength) -import Data.Text.Unsafe (unsafeDupablePerformIO) -import Data.Text.Show () -import qualified Data.ByteString as B -import qualified Data.ByteString.Internal as B -import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) -import qualified Data.ByteString.Builder.Prim as BP -import qualified Data.ByteString.Builder.Prim.Internal as BP -import qualified Data.ByteString.Short.Internal as SBS -import qualified Data.Text.Array as A -import qualified Data.Text.Internal.Encoding.Fusion as E -import qualified Data.Text.Internal.Fusion as F -#if defined(ASSERTS) -import GHC.Stack (HasCallStack) -#endif - --- $validation --- These functions are for validating 'ByteString's as encoded text. - --- $strict --- --- All of the single-parameter functions for decoding bytestrings --- encoded in one of the Unicode Transformation Formats (UTF) operate --- in a /strict/ mode: each will throw an exception if given invalid --- input. --- --- Each function has a variant, whose name is suffixed with -'With', --- that gives greater control over the handling of decoding errors. --- For instance, 'decodeUtf8' will throw an exception, but --- 'decodeUtf8With' allows the programmer to determine what to do on a --- decoding error. - --- $total --- --- These functions facilitate total decoding and should be preferred --- over their partial counterparts. - --- $partial --- --- These functions are partial and should only be used with great caution --- (preferably not at all). See "Data.Text.Encoding#g:total" for better --- solutions. - --- | Decode a 'ByteString' containing ASCII text. --- --- This is a total function which returns a pair of the longest ASCII prefix --- as 'Text', and the remaining suffix as 'ByteString'. --- --- Important note: the pair is lazy. This lets you check for errors by testing --- whether the second component is empty, without forcing the first component --- (which does a copy). --- To drop references to the input bytestring, force the prefix --- (using 'seq' or @BangPatterns@) and drop references to the suffix. --- --- === Properties --- --- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@. --- - Either @suffix@ is empty, or @'B.head' suffix > 127@. --- --- @since 2.0.2 -decodeASCIIPrefix :: ByteString -> (Text, ByteString) -decodeASCIIPrefix bs = if B.null bs - then (empty, B.empty) - else - let len = asciiPrefixLength bs - prefix = - let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in - Text (A.ByteArray arr) 0 len - suffix = B.drop len bs in - (prefix, suffix) -{-# INLINE decodeASCIIPrefix #-} - --- | Decode a 'ByteString' containing 7-bit ASCII encoded text. --- --- This is a total function which returns either the 'ByteString' converted to a --- 'Text' containing ASCII text, or 'Nothing'. --- --- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid --- input instead of discarding it. --- --- @since 2.0.2 -decodeASCII' :: ByteString -> Maybe Text -decodeASCII' bs = - let (prefix, suffix) = decodeASCIIPrefix bs in - if B.null suffix then Just prefix else Nothing -{-# INLINE decodeASCII' #-} - --- | Decode a 'ByteString' containing 7-bit ASCII encoded text. --- --- This is a partial function: it checks that input does not contain --- anything except ASCII and copies buffer or throws an error otherwise. -decodeASCII :: ByteString -> Text -decodeASCII bs = - let (prefix, suffix) = decodeASCIIPrefix bs in - case B.uncons suffix of - Nothing -> prefix - Just (word, _) -> - let !errPos = B.length bs - B.length suffix in - error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos - --- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. --- --- 'decodeLatin1' is semantically equivalent to --- @Data.Text.pack . Data.ByteString.Char8.unpack@ --- --- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) --- characters to UTf-8 requires actual work and is not just buffer copying. --- -decodeLatin1 :: -#if defined(ASSERTS) - HasCallStack => -#endif - ByteString -> Text -#if defined(PURE_HASKELL) -decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs) -#else -decodeLatin1 bs = withBS bs $ \fp len -> runST $ do - dst <- A.new (2 * len) - let inner srcOff dstOff = if srcOff >= len then return dstOff else do - asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> - c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) - if asciiPrefixLen == 0 - then do - byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff - A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) - A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) - inner (srcOff + 1) (dstOff + 2) - else do - unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> - unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen - inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) - actualLen <- inner 0 0 - dst' <- A.resizeM dst actualLen - arr <- A.unsafeFreeze dst' - return $ Text arr 0 actualLen -#endif - -#if !defined(PURE_HASKELL) -foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii - :: Ptr Word8 -> Ptr Word8 -> IO CSize -#endif - --- $stream --- --- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept --- a 'ByteString' that represents a possibly incomplete input (e.g. a --- packet from a network stream) that may not end on a UTF-8 boundary. --- --- 1. The maximal prefix of 'Text' that could be decoded from the --- given input. --- --- 2. The suffix of the 'ByteString' that could not be decoded due to --- insufficient input. --- --- 3. A function that accepts another 'ByteString'. That string will --- be assumed to directly follow the string that was passed as --- input to the original function, and it will in turn be decoded. --- --- To help understand the use of these functions, consider the Unicode --- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi --- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. --- --- Now suppose that we receive this encoded string as 3 packets that --- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", --- \"\\x83\"]@. We cannot decode the entire Unicode string until we --- have received all three packets, but we would like to make progress --- as we receive each one. --- --- @ --- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" --- ghci> s0 --- 'Some' \"hi \" \"\\xe2\" _ --- @ --- --- We use the continuation @f0@ to decode our second packet. --- --- @ --- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" --- ghci> s1 --- 'Some' \"\" \"\\xe2\\x98\" --- @ --- --- We could not give @f0@ enough input to decode anything, so it --- returned an empty string. Once we feed our second continuation @f1@ --- the last byte of input, it will make progress. --- --- @ --- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" --- ghci> s2 --- 'Some' \"\\x2603\" \"\" _ --- @ --- --- If given invalid input, an exception will be thrown by the function --- or continuation where it is encountered. - --- | A stream oriented decoding result. --- --- @since 1.0.0.0 -data Decoding = Some !Text !ByteString (ByteString -> Decoding) - -instance Show Decoding where - showsPrec d (Some t bs _) = showParen (d > prec) $ - showString "Some " . showsPrec prec' t . - showChar ' ' . showsPrec prec' bs . - showString " _" - where prec = 10; prec' = prec + 1 - --- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8 --- encoded text that is known to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown (either by this function or a continuation) that cannot be --- caught in pure code. For more control over the handling of invalid --- data, use 'streamDecodeUtf8With'. --- --- @since 1.0.0.0 -streamDecodeUtf8 :: -#if defined(ASSERTS) - HasCallStack => -#endif - ByteString -> Decoding -streamDecodeUtf8 = streamDecodeUtf8With strictDecode - --- | Decode, in a stream oriented way, a lazy 'ByteString' containing UTF-8 --- encoded text. --- --- @since 1.0.0.0 -streamDecodeUtf8With :: -#if defined(ASSERTS) - HasCallStack => -#endif - OnDecodeError -> ByteString -> Decoding -streamDecodeUtf8With onErr = loop startUtf8State - where - loop s chunk = - let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk - in Some (strictBuilderToText builder) undecoded (loop s') - --- | Decode a 'ByteString' containing UTF-8 encoded text. --- --- Surrogate code points in replacement character returned by 'OnDecodeError' --- will be automatically remapped to the replacement char @U+FFFD@. -decodeUtf8With :: -#if defined(ASSERTS) - HasCallStack => -#endif - OnDecodeError -> ByteString -> Text -decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg - -invalidUtf8Msg :: String -invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream" - --- | Decode a 'ByteString' containing UTF-8 encoded text that is known --- to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown that cannot be caught in pure code. For more control over --- the handling of invalid data, use 'decodeUtf8'' or --- 'decodeUtf8With'. --- --- This is a partial function: it checks that input is a well-formed --- UTF-8 sequence and copies buffer or throws an error otherwise. --- -decodeUtf8 :: ByteString -> Text -decodeUtf8 = decodeUtf8With strictDecode -{-# INLINE[0] decodeUtf8 #-} - --- | Decode a 'ByteString' containing UTF-8 encoded text. --- --- If the input contains any invalid UTF-8 data, the relevant --- exception will be returned, otherwise the decoded text. -decodeUtf8' :: -#if defined(ASSERTS) - HasCallStack => -#endif - ByteString -> Either UnicodeException Text -decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode -{-# INLINE decodeUtf8' #-} - --- | Decode a 'ByteString' containing UTF-8 encoded text. --- --- Any invalid input bytes will be replaced with the Unicode replacement --- character U+FFFD. -decodeUtf8Lenient :: ByteString -> Text -decodeUtf8Lenient = decodeUtf8With lenientDecode - --- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. --- --- @since 1.1.0.0 -encodeUtf8Builder :: Text -> B.Builder -encodeUtf8Builder = - -- manual eta-expansion to ensure inlining works as expected - \txt -> B.builder (step txt) - where - step txt@(Text arr off len) !k br@(B.BufferRange op ope) - -- Ensure that the common case is not recursive and therefore yields - -- better code. - | op' <= ope = do - unsafeSTToIO $ A.copyToPointer arr off op len - k (B.BufferRange op' ope) - | otherwise = textCopyStep txt k br - where - op' = op `plusPtr` len -{-# INLINE encodeUtf8Builder #-} - -textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a -textCopyStep (Text arr off len) k = - go off (off + len) - where - go !ip !ipe (B.BufferRange op ope) - | inpRemaining <= outRemaining = do - unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining - let !br = B.BufferRange (op `plusPtr` inpRemaining) ope - k br - | otherwise = do - unsafeSTToIO $ A.copyToPointer arr ip op outRemaining - let !ip' = ip + outRemaining - return $ B.bufferFull 1 ope (go ip' ipe) - where - outRemaining = ope `minusPtr` op - inpRemaining = ipe - ip - --- | Encode text using UTF-8 encoding and escape the ASCII characters using --- a 'BP.BoundedPrim'. --- --- Use this function is to implement efficient encoders for text-based formats --- like JSON or HTML. --- --- @since 1.1.0.0 -{-# INLINE encodeUtf8BuilderEscaped #-} --- TODO: Extend documentation with references to source code in @blaze-html@ --- or @aeson@ that uses this function. -encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder -encodeUtf8BuilderEscaped be = - -- manual eta-expansion to ensure inlining works as expected - \txt -> B.builder (mkBuildstep txt) - where - bound = max 4 $ BP.sizeBound be - - mkBuildstep (Text arr off len) !k = - outerLoop off - where - iend = off + len - - outerLoop !i0 !br@(B.BufferRange op0 ope) - | i0 >= iend = k br - | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) - -- TODO: Use a loop with an integrated bound's check if outRemaining - -- is smaller than 8, as this will save on divisions. - | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) - where - outRemaining = (ope `minusPtr` op0) `quot` bound - inpRemaining = iend - i0 - - goPartial !iendTmp = go i0 op0 - where - go !i !op - | i < iendTmp = do - let w = A.unsafeIndex arr i - if w < 0x80 - then BP.runB be w op >>= go (i + 1) - else poke op w >> go (i + 1) (op `plusPtr` 1) - | otherwise = outerLoop i (B.BufferRange op ope) - --- | Encode text using UTF-8 encoding. -encodeUtf8 :: Text -> ByteString -encodeUtf8 (Text arr off len) - | len == 0 = B.empty - -- It would be easier to use Data.ByteString.Short.fromShort and slice later, - -- but this is undesirable when len is significantly smaller than length arr. - | otherwise = unsafeDupablePerformIO $ do - marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len - unsafeSTToIO $ A.copyI len marr 0 arr off - let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) - (PlainPtr mba) - pure $ B.fromForeignPtr fp 0 len - --- | Decode text from little endian UTF-16 encoding. -decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text -decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) -{-# INLINE decodeUtf16LEWith #-} - --- | Decode text from little endian UTF-16 encoding. --- --- If the input contains any invalid little endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16LEWith'. -decodeUtf16LE :: ByteString -> Text -decodeUtf16LE = decodeUtf16LEWith strictDecode -{-# INLINE decodeUtf16LE #-} - --- | Decode text from big endian UTF-16 encoding. -decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text -decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) -{-# INLINE decodeUtf16BEWith #-} - --- | Decode text from big endian UTF-16 encoding. --- --- If the input contains any invalid big endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16BEWith'. -decodeUtf16BE :: ByteString -> Text -decodeUtf16BE = decodeUtf16BEWith strictDecode -{-# INLINE decodeUtf16BE #-} - --- | Encode text using little endian UTF-16 encoding. -encodeUtf16LE :: Text -> ByteString -encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) -{-# INLINE encodeUtf16LE #-} - --- | Encode text using big endian UTF-16 encoding. -encodeUtf16BE :: Text -> ByteString -encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) -{-# INLINE encodeUtf16BE #-} - --- | Decode text from little endian UTF-32 encoding. -decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text -decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) -{-# INLINE decodeUtf32LEWith #-} - --- | Decode text from little endian UTF-32 encoding. --- --- If the input contains any invalid little endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32LEWith'. -decodeUtf32LE :: ByteString -> Text -decodeUtf32LE = decodeUtf32LEWith strictDecode -{-# INLINE decodeUtf32LE #-} - --- | Decode text from big endian UTF-32 encoding. -decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text -decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) -{-# INLINE decodeUtf32BEWith #-} - --- | Decode text from big endian UTF-32 encoding. --- --- If the input contains any invalid big endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32BEWith'. -decodeUtf32BE :: ByteString -> Text -decodeUtf32BE = decodeUtf32BEWith strictDecode -{-# INLINE decodeUtf32BE #-} - --- | Encode text using little endian UTF-32 encoding. -encodeUtf32LE :: Text -> ByteString -encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) -{-# INLINE encodeUtf32LE #-} - --- | Encode text using big endian UTF-32 encoding. -encodeUtf32BE :: Text -> ByteString -encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) -{-# INLINE encodeUtf32BE #-} - --- $incremental --- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more --- control for error-handling and streaming. --- --- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error. --- For example this lets you insert or delete arbitrary text, or do some --- stateful operations before resuming, such as keeping track of error locations. --- In contrast, the older stream-oriented interface only lets you substitute --- a single fixed 'Char' for each invalid byte in 'OnDecodeError'. --- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks --- before doing the copying work to construct a 'Text', or you can --- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'. --- --- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'. +{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, + UnliftedFFITypes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +-- | +-- Module : Data.Text.Encoding +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting 'Text' values to and from 'ByteString', +-- using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- . + +module Data.Text.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeASCIIPrefix + , decodeUtf8Lenient + , decodeUtf8' + , decodeASCII' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- *** Incremental UTF-8 decoding + -- $incremental + , decodeUtf8Chunk + , decodeUtf8More + , Utf8State + , startUtf8State + , StrictBuilder + , StrictTextBuilder + , strictBuilderToText + , textToStrictBuilder + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- ** Stream oriented decoding + -- $stream + , streamDecodeUtf8 + , streamDecodeUtf8With + , Decoding(..) + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + + -- * ByteString validation + -- $validation + , validateUtf8Chunk + , validateUtf8More + ) where + +import Control.Exception (evaluate, try) +import Data.Word (Word8) +import GHC.Exts (byteArrayContents#, unsafeCoerce#) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) +import Data.ByteString (ByteString) +#if defined(PURE_HASKELL) +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Char8 (unpack) +import Data.Text.Internal (pack) +import Foreign.Ptr (minusPtr, plusPtr) +import Foreign.Storable (poke) +#else +import Control.Monad.ST (runST) +import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) +import Data.Bits (shiftR, (.&.)) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Foreign.C.Types (CSize(..)) +import Foreign.Ptr (Ptr, minusPtr, plusPtr) +import Foreign.Storable (poke, peekByteOff) +#endif +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Encoding +import Data.Text.Internal.IsAscii (asciiPrefixLength) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Show () +import qualified Data.ByteString as B +import qualified Data.ByteString.Internal as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Builder.Prim.Internal as BP +import qualified Data.ByteString.Short.Internal as SBS +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Encoding.Fusion as E +import qualified Data.Text.Internal.Fusion as F +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +-- $validation +-- These functions are for validating 'ByteString's as encoded text. + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing ASCII text. +-- +-- This is a total function which returns a pair of the longest ASCII prefix +-- as 'Text', and the remaining suffix as 'ByteString'. +-- +-- Important note: the pair is lazy. This lets you check for errors by testing +-- whether the second component is empty, without forcing the first component +-- (which does a copy). +-- To drop references to the input bytestring, force the prefix +-- (using 'seq' or @BangPatterns@) and drop references to the suffix. +-- +-- === Properties +-- +-- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@. +-- - Either @suffix@ is empty, or @'B.head' suffix > 127@. +-- +-- @since 2.0.2 +decodeASCIIPrefix :: ByteString -> (Text, ByteString) +decodeASCIIPrefix bs = if B.null bs + then (empty, B.empty) + else + let len = asciiPrefixLength bs + prefix = + let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in + Text (A.ByteArray arr) 0 len + suffix = B.drop len bs in + (prefix, suffix) +{-# INLINE decodeASCIIPrefix #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a total function which returns either the 'ByteString' converted to a +-- 'Text' containing ASCII text, or 'Nothing'. +-- +-- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid +-- input instead of discarding it. +-- +-- @since 2.0.2 +decodeASCII' :: ByteString -> Maybe Text +decodeASCII' bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + if B.null suffix then Just prefix else Nothing +{-# INLINE decodeASCII' #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a partial function: it checks that input does not contain +-- anything except ASCII and copies buffer or throws an error otherwise. +decodeASCII :: ByteString -> Text +decodeASCII bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + case B.uncons suffix of + Nothing -> prefix + Just (word, _) -> + let !errPos = B.length bs - B.length suffix in + error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +-- +-- 'decodeLatin1' is semantically equivalent to +-- @'Data.Text.pack' . 'Data.ByteString.Char8.unpack'@ +-- +-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) +-- characters to UTf-8 requires actual work and is not just buffer copying. +-- +decodeLatin1 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Text +#if defined(PURE_HASKELL) +decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs) +#else +decodeLatin1 bs = withBS bs $ \fp len -> runST $ do + dst <- A.new (2 * len) + let inner srcOff dstOff = if srcOff >= len then return dstOff else do + asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) + if asciiPrefixLen == 0 + then do + byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff + A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) + A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) + inner (srcOff + 1) (dstOff + 2) + else do + unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen + inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) + actualLen <- inner 0 0 + dst' <- A.resizeM dst actualLen + arr <- A.unsafeFreeze dst' + return $ Text arr 0 actualLen +#endif + +#if !defined(PURE_HASKELL) +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize +#endif + +-- $stream +-- +-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept +-- a strict 'ByteString' that represents a possibly incomplete input (e.g. a +-- packet from a network stream) that may not end on a UTF-8 boundary +-- and return 'Decoding', which consists of: +-- +-- * The maximal prefix of 'Text' that could be decoded from the +-- given input. +-- +-- * The suffix of the 'ByteString' that could not be decoded due to +-- insufficient input. +-- +-- * A function that accepts another 'ByteString'. That string will +-- be assumed to directly follow the string that was passed as +-- input to the original function, and it will in turn be decoded. +-- +-- To help understand the use of these functions, consider the Unicode +-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi +-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. +-- +-- Now suppose that we receive this encoded string as 3 packets that +-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", +-- \"\\x83\"]@. We cannot decode the entire Unicode string until we +-- have received all three packets, but we would like to make progress +-- as we receive each one. +-- +-- @ +-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" +-- ghci> s0 +-- 'Some' \"hi \" \"\\xe2\" _ +-- @ +-- +-- We use the continuation @f0@ to decode our second packet. +-- +-- @ +-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" +-- ghci> s1 +-- 'Some' \"\" \"\\xe2\\x98\" +-- @ +-- +-- We could not give @f0@ enough input to decode anything, so it +-- returned an empty string. Once we feed our second continuation @f1@ +-- the last byte of input, it will make progress. +-- +-- @ +-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" +-- ghci> s2 +-- 'Some' \"\\x2603\" \"\" _ +-- @ +-- +-- If given invalid input, an exception will be thrown by the function +-- or continuation where it is encountered. + +-- | A stream-oriented decoding result (see 'streamDecodeUtf8' and 'streamDecodeUtf8With'). +-- +-- @since 1.0.0.0 +data Decoding = Some + !Text + -- ^ The maximal prefix that could be decoded from the given input. + !ByteString + -- ^ The remaining suffix of the input that could not be decoded + -- (usually because the input breaks in the middle of UTF-8 character) + (ByteString -> Decoding) + -- ^ The continuation call which should be fed with the next + -- chunk of the input. + +instance Show Decoding where + showsPrec d (Some t bs _) = showParen (d > prec) $ + showString "Some " . showsPrec prec' t . + showChar ' ' . showsPrec prec' bs . + showString " _" + where prec = 10; prec' = prec + 1 + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data that is known to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown (either by this function or a continuation) that cannot be +-- caught in pure code. For more control over the handling of invalid +-- data, use 'streamDecodeUtf8With'. +-- +-- @since 1.0.0.0 +streamDecodeUtf8 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Decoding +streamDecodeUtf8 = streamDecodeUtf8With strictDecode + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data. +-- +-- @since 1.0.0.0 +streamDecodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Decoding +streamDecodeUtf8With onErr = loop startUtf8State + where + loop s chunk = + let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk + in Some (strictBuilderToText builder) undecoded (loop s') + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Surrogate code points in replacement character returned by 'OnDecodeError' +-- will be automatically remapped to the replacement char @U+FFFD@. +decodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Text +decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg + +invalidUtf8Msg :: String +invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +-- +-- This is a partial function: it checks that input is a well-formed +-- UTF-8 sequence and copies buffer or throws an error otherwise. +-- +decodeUtf8 :: ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +decodeUtf8' :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Either UnicodeException Text +decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode +{-# INLINE decodeUtf8' #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.0 +decodeUtf8Lenient :: ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (step txt) + where + step txt@(Text arr off len) !k br@(B.BufferRange op ope) + -- Ensure that the common case is not recursive and therefore yields + -- better code. + | op' <= ope = do + unsafeSTToIO $ A.copyToPointer arr off op len + k (B.BufferRange op' ope) + | otherwise = textCopyStep txt k br + where + op' = op `plusPtr` len +{-# INLINE encodeUtf8Builder #-} + +textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a +textCopyStep (Text arr off len) k = + go off (off + len) + where + go !ip !ipe (B.BufferRange op ope) + | inpRemaining <= outRemaining = do + unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining + let !br = B.BufferRange (op `plusPtr` inpRemaining) ope + k br + | otherwise = do + unsafeSTToIO $ A.copyToPointer arr ip op outRemaining + let !ip' = ip + outRemaining + return $ B.bufferFull 1 ope (go ip' ipe) + where + outRemaining = ope `minusPtr` op + inpRemaining = ipe - ip + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +-- TODO: Extend documentation with references to source code in @blaze-html@ +-- or @aeson@ that uses this function. +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped be = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (mkBuildstep txt) + where + bound = max 4 $ BP.sizeBound be + + mkBuildstep (Text arr off len) !k = + outerLoop off + where + iend = off + len + + outerLoop !i0 !br@(B.BufferRange op0 ope) + | i0 >= iend = k br + | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) + -- TODO: Use a loop with an integrated bound's check if outRemaining + -- is smaller than 8, as this will save on divisions. + | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) + where + outRemaining = (ope `minusPtr` op0) `quot` bound + inpRemaining = iend - i0 + + goPartial !iendTmp = go i0 op0 + where + go !i !op + | i < iendTmp = do + let w = A.unsafeIndex arr i + if w < 0x80 + then BP.runB be w op >>= go (i + 1) + else poke op w >> go (i + 1) (op `plusPtr` 1) + | otherwise = outerLoop i (B.BufferRange op ope) + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> ByteString +encodeUtf8 (Text arr off len) + | len == 0 = B.empty + -- It would be easier to use Data.ByteString.Short.fromShort and slice later, + -- but this is undesirable when len is significantly smaller than length arr. + | otherwise = unsafeDupablePerformIO $ do + marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len + unsafeSTToIO $ A.copyI len marr 0 arr off + let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) + (PlainPtr mba) + pure $ B.fromForeignPtr fp 0 len + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> ByteString +encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> ByteString +encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> ByteString +encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> ByteString +encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) +{-# INLINE encodeUtf32BE #-} + +-- $incremental +-- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more +-- control for error-handling and streaming. +-- +-- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error. +-- For example this lets you insert or delete arbitrary text, or do some +-- stateful operations before resuming, such as keeping track of error locations. +-- In contrast, the older stream-oriented interface only lets you substitute +-- a single fixed 'Char' for each invalid byte in 'OnDecodeError'. +-- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks +-- before doing the copying work to construct a 'Text', or you can +-- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'. +-- +-- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'. diff --git a/src/Data/Text/Encoding/Error.hs b/src/Data/Text/Encoding/Error.hs --- a/src/Data/Text/Encoding/Error.hs +++ b/src/Data/Text/Encoding/Error.hs @@ -1,120 +1,119 @@ -{-# LANGUAGE CPP, DeriveDataTypeable #-} -{-# LANGUAGE Safe #-} --- | --- Module : Data.Text.Encoding.Error --- Copyright : (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : GHC --- --- Types and functions for dealing with encoding and decoding errors --- in Unicode text. --- --- The standard functions for encoding and decoding text are strict, --- which is to say that they throw exceptions on invalid input. This --- is often unhelpful on real world input, so alternative functions --- exist that accept custom handlers for dealing with invalid inputs. --- These 'OnError' handlers are normal Haskell functions. You can use --- one of the presupplied functions in this module, or you can write a --- custom handler of your own. - -module Data.Text.Encoding.Error - ( - -- * Error handling types - UnicodeException(..) - , OnError - , OnDecodeError - , OnEncodeError - -- * Useful error handling functions - , lenientDecode - , strictDecode - , strictEncode - , ignore - , replace - ) where - -import Control.DeepSeq (NFData (..)) -import Control.Exception (Exception, throw) -import Data.Typeable (Typeable) -import Data.Word (Word8) -import Numeric (showHex) - --- | Function type for handling a coding error. It is supplied with --- two inputs: --- --- * A 'String' that describes the error. --- --- * The input value that caused the error. If the error arose --- because the end of input was reached or could not be identified --- precisely, this value will be 'Nothing'. --- --- If the handler returns a value wrapped with 'Just', that value will --- be used in the output as the replacement for the invalid input. If --- it returns 'Nothing', no value will be used in the output. --- --- Should the handler need to abort processing, it should use 'error' --- or 'throw' an exception (preferably a 'UnicodeException'). It may --- use the description provided to construct a more helpful error --- report. -type OnError a b = String -> Maybe a -> Maybe b - --- | A handler for a decoding error. -type OnDecodeError = OnError Word8 Char - --- | A handler for an encoding error. -{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} -type OnEncodeError = OnError Char Word8 - --- | An exception type for representing Unicode encoding errors. -data UnicodeException = - DecodeError String (Maybe Word8) - -- ^ Could not decode a byte sequence because it was invalid under - -- the given encoding, or ran out of input in mid-decode. - | EncodeError String (Maybe Char) - -- ^ Tried to encode a character that could not be represented - -- under the given encoding, or ran out of input in mid-encode. - deriving (Eq, Typeable) - -{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} - -showUnicodeException :: UnicodeException -> String -showUnicodeException (DecodeError desc (Just w)) - = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) -showUnicodeException (DecodeError desc Nothing) - = "Cannot decode input: " ++ desc -showUnicodeException (EncodeError desc (Just c)) - = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) -showUnicodeException (EncodeError desc Nothing) - = "Cannot encode input: " ++ desc - -instance Show UnicodeException where - show = showUnicodeException - -instance Exception UnicodeException - -instance NFData UnicodeException where - rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () - rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () - --- | Throw a 'UnicodeException' if decoding fails. -strictDecode :: OnDecodeError -strictDecode desc c = throw (DecodeError desc c) - --- | Replace an invalid input byte with the Unicode replacement --- character U+FFFD. -lenientDecode :: OnDecodeError -lenientDecode _ _ = Just '\xfffd' - --- | Throw a 'UnicodeException' if encoding fails. -{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} -strictEncode :: OnEncodeError -strictEncode desc c = throw (EncodeError desc c) - --- | Ignore an invalid input, substituting nothing in the output. -ignore :: OnError a b -ignore _ _ = Nothing - --- | Replace an invalid input with a valid output. -replace :: b -> OnError a b -replace c _ _ = Just c +{-# LANGUAGE CPP #-} +{-# LANGUAGE Safe #-} +-- | +-- Module : Data.Text.Encoding.Error +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Types and functions for dealing with encoding and decoding errors +-- in Unicode text. +-- +-- The standard functions for encoding and decoding text are strict, +-- which is to say that they throw exceptions on invalid input. This +-- is often unhelpful on real world input, so alternative functions +-- exist that accept custom handlers for dealing with invalid inputs. +-- These 'OnError' handlers are normal Haskell functions. You can use +-- one of the presupplied functions in this module, or you can write a +-- custom handler of your own. + +module Data.Text.Encoding.Error + ( + -- * Error handling types + UnicodeException(..) + , OnError + , OnDecodeError + , OnEncodeError + -- * Useful error handling functions + , lenientDecode + , strictDecode + , strictEncode + , ignore + , replace + ) where + +import Control.DeepSeq (NFData (..)) +import Control.Exception (Exception, throw) +import Data.Word (Word8) +import Numeric (showHex) + +-- | Function type for handling a coding error. It is supplied with +-- two inputs: +-- +-- * A 'String' that describes the error. +-- +-- * The input value that caused the error. If the error arose +-- because the end of input was reached or could not be identified +-- precisely, this value will be 'Nothing'. +-- +-- If the handler returns a value wrapped with 'Just', that value will +-- be used in the output as the replacement for the invalid input. If +-- it returns 'Nothing', no value will be used in the output. +-- +-- Should the handler need to abort processing, it should use 'error' +-- or 'throw' an exception (preferably a 'UnicodeException'). It may +-- use the description provided to construct a more helpful error +-- report. +type OnError a b = String -> Maybe a -> Maybe b + +-- | A handler for a decoding error. +type OnDecodeError = OnError Word8 Char + +-- | A handler for an encoding error. +{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} +type OnEncodeError = OnError Char Word8 + +-- | An exception type for representing Unicode encoding errors. +data UnicodeException = + DecodeError String (Maybe Word8) + -- ^ Could not decode a byte sequence because it was invalid under + -- the given encoding, or ran out of input in mid-decode. + | EncodeError String (Maybe Char) + -- ^ Tried to encode a character that could not be represented + -- under the given encoding, or ran out of input in mid-encode. + deriving (Eq) + +{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} + +showUnicodeException :: UnicodeException -> String +showUnicodeException (DecodeError desc (Just w)) + = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) +showUnicodeException (DecodeError desc Nothing) + = "Cannot decode input: " ++ desc +showUnicodeException (EncodeError desc (Just c)) + = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) +showUnicodeException (EncodeError desc Nothing) + = "Cannot encode input: " ++ desc + +instance Show UnicodeException where + show = showUnicodeException + +instance Exception UnicodeException + +instance NFData UnicodeException where + rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () + rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () + +-- | Throw a 'UnicodeException' if decoding fails. +strictDecode :: OnDecodeError +strictDecode desc c = throw (DecodeError desc c) + +-- | Replace an invalid input byte with the Unicode replacement +-- character U+FFFD. +lenientDecode :: OnDecodeError +lenientDecode _ _ = Just '\xfffd' + +-- | Throw a 'UnicodeException' if encoding fails. +{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} +strictEncode :: OnEncodeError +strictEncode desc c = throw (EncodeError desc c) + +-- | Ignore an invalid input, substituting nothing in the output. +ignore :: OnError a b +ignore _ _ = Nothing + +-- | Replace an invalid input with a valid output. +replace :: b -> OnError a b +replace c _ _ = Just c diff --git a/src/Data/Text/Foreign.hs b/src/Data/Text/Foreign.hs --- a/src/Data/Text/Foreign.hs +++ b/src/Data/Text/Foreign.hs @@ -1,192 +1,212 @@ -{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-} --- | --- Module : Data.Text.Foreign --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : GHC --- --- Support for using 'Text' data with native code via the Haskell --- foreign function interface. - -module Data.Text.Foreign - ( - -- * Interoperability with native code - -- $interop - I8 - -- * Safe conversion functions - , fromPtr - , fromPtr0 - , useAsPtr - , asForeignPtr - -- ** Encoding as UTF-8 - , withCString - , peekCStringLen - , withCStringLen - -- * Unsafe conversion code - , lengthWord8 - , unsafeCopyToPtr - -- * Low-level manipulation - -- $lowlevel - , dropWord8 - , takeWord8 - ) where - -import Control.Monad.ST.Unsafe (unsafeSTToIO) -import Data.ByteString.Unsafe (unsafePackCStringLen, unsafeUseAsCStringLen) -import Data.Text.Encoding (decodeUtf8, encodeUtf8) -import Data.Text.Internal (Text(..), empty) -import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) -import Data.Text.Show (addrLen) -import Data.Text.Unsafe (lengthWord8) -import Data.Word (Word8) -import Foreign.C.String (CString, CStringLen) -import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) -import Foreign.Marshal.Alloc (allocaBytes) -import Foreign.Ptr (Ptr, castPtr) -import Foreign.Storable (pokeByteOff) -import GHC.Exts (Ptr(..)) -import qualified Data.Text.Array as A - --- $interop --- --- The 'Text' type is implemented using arrays that are not guaranteed --- to have a fixed address in the Haskell heap. All communication with --- native code must thus occur by copying data back and forth. --- --- The 'Text' type's internal representation is UTF-8. --- To interoperate with native libraries that use different --- internal representations, such as UTF-16 or UTF-32, consider using --- the functions in the 'Data.Text.Encoding' module. - --- | A type representing a number of UTF-8 code units. --- --- @since 2.0 -newtype I8 = I8 Int - deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) - --- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the --- contents of the array. -fromPtr :: Ptr Word8 -- ^ source array - -> I8 -- ^ length of source array (in 'Word8' units) - -> IO Text -fromPtr _ (I8 0) = pure empty -fromPtr ptr (I8 len) = unsafeSTToIO $ do - dst <- A.new len - A.copyFromPointer dst 0 ptr len - arr <- A.unsafeFreeze dst - return $! Text arr 0 len - --- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the --- contents of the NUL-terminated array. --- --- @since 2.0.1 -fromPtr0 :: Ptr Word8 -- ^ source array - -> IO Text -fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#)) - --- $lowlevel --- --- Foreign functions that use UTF-8 internally may return indices in --- units of 'Word8' instead of characters. These functions may --- safely be used with such indices, as they will adjust offsets if --- necessary to preserve the validity of a Unicode string. - --- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in --- length. --- --- If @n@ would cause the 'Text' to end inside a code point, the --- end of the prefix will be advanced by several additional 'Word8' units --- to maintain its validity. --- --- @since 2.0 -takeWord8 :: I8 -> Text -> Text -takeWord8 = (fst .) . splitAtWord8 - --- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units --- dropped from its beginning. --- --- If @n@ would cause the 'Text' to begin inside a code point, the --- beginning of the suffix will be advanced by several additional 'Word8' --- unit to maintain its validity. --- --- @since 2.0 -dropWord8 :: I8 -> Text -> Text -dropWord8 = (snd .) . splitAtWord8 - -splitAtWord8 :: I8 -> Text -> (Text, Text) -splitAtWord8 (I8 n) t@(Text arr off len) - | n <= 0 = (empty, t) - | n >= len || m >= len = (t, empty) - | otherwise = (Text arr off m, Text arr (off+m) (len-m)) - where - m | w0 < 0x80 = n -- last char is ASCII - | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence - | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence - | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence - | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence - | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence - | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence - | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence - | otherwise = n -- pre-pre-last char starts 3-byte sequence - w0 = A.unsafeIndex arr (off+n-1) - w1 = A.unsafeIndex arr (off+n-2) - w2 = A.unsafeIndex arr (off+n-3) - --- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big --- enough to hold the contents of the entire 'Text'. -unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () -unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len - --- | /O(n)/ Perform an action on a temporary, mutable copy of a --- 'Text'. The copy is freed as soon as the action returns. -useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a -useAsPtr t@(Text _arr _off len) action = - allocaBytes len $ \buf -> do - unsafeCopyToPtr t buf - action (castPtr buf) (I8 len) - --- | /O(n)/ Make a mutable copy of a 'Text'. -asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) -asForeignPtr t@(Text _arr _off len) = do - fp <- mallocForeignPtrArray len - unsafeWithForeignPtr fp $ unsafeCopyToPtr t - return (fp, I8 len) - --- | Marshal a 'Text' into a C string with a trailing NUL byte, --- encoded as UTF-8 in temporary storage. --- --- The temporary storage is freed when the subcomputation terminates --- (either normally or via an exception), so the pointer to the --- temporary storage must /not/ be used after this function returns. --- --- @since 2.0.1 -withCString :: Text -> (CString -> IO a) -> IO a -withCString t@(Text _arr _off len) action = - allocaBytes (len + 1) $ \buf -> do - unsafeCopyToPtr t buf - pokeByteOff buf len (0 :: Word8) - action (castPtr buf) - --- | /O(n)/ Decode a C string with explicit length, which is assumed --- to have been encoded as UTF-8. If decoding fails, a --- 'UnicodeException' is thrown. --- --- @since 1.0.0.0 -peekCStringLen :: CStringLen -> IO Text -peekCStringLen cs = do - bs <- unsafePackCStringLen cs - return $! decodeUtf8 bs - --- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary --- storage, with explicit length information. The encoded string may --- contain NUL bytes, and is not followed by a trailing NUL byte. --- --- The temporary storage is freed when the subcomputation terminates --- (either normally or via an exception), so the pointer to the --- temporary storage must /not/ be used after this function returns. --- --- @since 1.0.0.0 -withCStringLen :: Text -> (CStringLen -> IO a) -> IO a -withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act +{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-} +-- | +-- Module : Data.Text.Foreign +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Support for using 'Text' data with native code via the Haskell +-- foreign function interface. + +module Data.Text.Foreign + ( + -- * Interoperability with native code + -- $interop + I8 + -- * Pointer conversion functions + , fromPtr + , fromPtr0 + , useAsPtr + , asForeignPtr + -- ** Encoding as UTF-8 + , peekCString + , withCString + , peekCStringLen + , withCStringLen + -- * Low-level manipulation + -- $lowlevel + , dropWord8 + , takeWord8 + , lengthWord8 + , unsafeCopyToPtr + ) where + +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Show (addrLen) +import Data.Text.Unsafe (lengthWord8) +import Data.Word (Word8) +import Foreign.C.String (CString, CStringLen) +import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) +import Foreign.Marshal.Alloc (allocaBytes) +import Foreign.Ptr (Ptr, castPtr) +import Foreign.Storable (pokeByteOff) +import GHC.Exts (Ptr(..)) +import qualified Data.Text.Array as A + +-- $interop +-- +-- The 'Text' type is implemented using arrays that are not guaranteed +-- to have a fixed address in the Haskell heap. All communication with +-- native code must thus occur by copying data back and forth. +-- +-- The 'Text' type's internal representation is UTF-8. +-- To interoperate with native libraries that use different +-- internal representations, such as UTF-16 or UTF-32, consider using +-- the functions in the 'Data.Text.Encoding' module. + +-- | A type representing a number of UTF-8 code units. +-- +-- @since 2.0 +newtype I8 = I8 Int + deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the array. +-- +-- __This function is unsafe.__ The source array must contain a valid +-- UTF-8 string of the given length. There are no guarantees about what +-- happens otherwise. +fromPtr :: Ptr Word8 -- ^ source array + -> I8 -- ^ length of source array (in 'Word8' units) + -> IO Text +fromPtr _ (I8 0) = pure empty +fromPtr ptr (I8 len) = unsafeSTToIO $ do + dst <- A.new len + A.copyFromPointer dst 0 ptr len + arr <- A.unsafeFreeze dst + return $! Text arr 0 len + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the NUL-terminated array. +-- +-- __This function is unsafe.__ The source array must contain a NULL-terminated +-- valid UTF-8 string. There are no guarantees about what happens otherwise. +-- +-- @since 2.0.1 +fromPtr0 :: Ptr Word8 -- ^ source array + -> IO Text +fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#)) + +-- $lowlevel +-- +-- Foreign functions that use UTF-8 internally may return indices in +-- units of 'Word8' instead of characters. These functions may +-- safely be used with such indices, as they will adjust offsets if +-- necessary to preserve the validity of a Unicode string. + +-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in +-- length. +-- +-- If @n@ would cause the 'Text' to end inside a code point, the +-- end of the prefix will be advanced by several additional 'Word8' units +-- to maintain its validity. +-- +-- @since 2.0 +takeWord8 :: I8 -> Text -> Text +takeWord8 = (fst .) . splitAtWord8 + +-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units +-- dropped from its beginning. +-- +-- If @n@ would cause the 'Text' to begin inside a code point, the +-- beginning of the suffix will be advanced by several additional 'Word8' +-- unit to maintain its validity. +-- +-- @since 2.0 +dropWord8 :: I8 -> Text -> Text +dropWord8 = (snd .) . splitAtWord8 + +splitAtWord8 :: I8 -> Text -> (Text, Text) +splitAtWord8 (I8 n) t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m | w0 < 0x80 = n -- last char is ASCII + | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence + | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence + | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence + | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence + | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence + | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence + | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence + | otherwise = n -- pre-pre-last char starts 3-byte sequence + w0 = A.unsafeIndex arr (off+n-1) + w1 = A.unsafeIndex arr (off+n-2) + w2 = A.unsafeIndex arr (off+n-3) + +-- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big +-- enough to hold the contents of the entire 'Text'. +unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () +unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len + +-- | /O(n)/ Perform an action on a temporary, mutable copy of a +-- 'Text'. The copy is freed as soon as the action returns. +useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a +useAsPtr t@(Text _arr _off len) action = + allocaBytes len $ \buf -> do + unsafeCopyToPtr t buf + action (castPtr buf) (I8 len) + +-- | /O(n)/ Make a mutable copy of a 'Text'. +asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) +asForeignPtr t@(Text _arr _off len) = do + fp <- mallocForeignPtrArray len + unsafeWithForeignPtr fp $ unsafeCopyToPtr t + return (fp, I8 len) + +-- | Marshal a 'Text' into a C string with a trailing NUL byte, +-- encoded as UTF-8 in temporary storage. +-- +-- The 'Text' itself must not contain any NUL bytes, this precondition +-- is not checked. Cf. 'withCStringLen'. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 2.0.1 +withCString :: Text -> (CString -> IO a) -> IO a +withCString t@(Text _arr _off len) action = + allocaBytes (len + 1) $ \buf -> do + unsafeCopyToPtr t buf + pokeByteOff buf len (0 :: Word8) + action (castPtr buf) + +-- | /O(n)/ Decode a C string with explicit length, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 1.0.0.0 +peekCStringLen :: CStringLen -> IO Text +peekCStringLen cs = do + bs <- unsafePackCStringLen cs + return $! decodeUtf8 bs + +-- | /O(n)/ Decode a null-terminated C string, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 2.1.2 +peekCString :: CString -> IO Text +peekCString cs = do + bs <- unsafePackCString cs + return $! decodeUtf8 bs + +-- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary +-- storage, with explicit length information. The encoded string may +-- contain NUL bytes, and is not followed by a trailing NUL byte. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 1.0.0.0 +withCStringLen :: Text -> (CStringLen -> IO a) -> IO a +withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act diff --git a/src/Data/Text/IO.hs b/src/Data/Text/IO.hs --- a/src/Data/Text/IO.hs +++ b/src/Data/Text/IO.hs @@ -1,312 +1,211 @@ -{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} -{-# LANGUAGE Trustworthy #-} --- | --- Module : Data.Text.IO --- Copyright : (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Simon Marlow --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : GHC --- --- Efficient locale-sensitive support for text I\/O. --- --- The functions in this module obey the runtime system's locale, --- character set encoding, and line ending conversion settings. --- --- If you want to do I\/O using the UTF-8 encoding, use @Data.Text.IO.Utf8@, --- which is faster than this module. --- --- If you know in advance that you will be working with data that has --- a specific encoding, and your application is highly --- performance sensitive, you may find that it is faster to perform --- I\/O with bytestrings and to encode and decode yourself than to use --- the functions in this module. - -module Data.Text.IO - ( - -- * File-at-a-time operations - readFile - , writeFile - , appendFile - -- * Operations on handles - , hGetContents - , hGetChunk - , hGetLine - , hPutStr - , hPutStrLn - -- * Special cases for standard input and output - , interact - , getContents - , getLine - , putStr - , putStrLn - ) where - -import Data.Text (Text) -import Prelude hiding (appendFile, getContents, getLine, interact, - putStr, putStrLn, readFile, writeFile) -import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout, - withFile) -import qualified Control.Exception as E -import Control.Monad (liftM2, when) -import Data.IORef (readIORef, writeIORef) -import qualified Data.Text as T -import Data.Text.Internal.Fusion (stream) -import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) -import Data.Text.Internal.IO (hGetLineWith, readChunk) -import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBufElem, CharBuffer, - RawCharBuffer, emptyBuffer, isEmptyBuffer, newCharBuffer, - writeCharBuf) -import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) -import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle, - wantWritableHandle) -import GHC.IO.Handle.Text (commitBuffer') -import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), - HandleType(..), Newline(..)) -import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) -import System.IO.Error (isEOFError) - --- | The 'readFile' function reads a file and returns the contents of --- the file as a string. The entire file is read strictly, as with --- 'getContents'. --- --- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. --- Unexpected system locale may cause your application to read corrupted data or --- throw runtime exceptions about "invalid argument (invalid byte sequence)" --- or "invalid argument (invalid character)". This is also slow, because GHC --- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. --- --- If your data is UTF-8, --- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' --- is a much faster and safer alternative. -readFile :: FilePath -> IO Text -readFile name = openFile name ReadMode >>= hGetContents - --- | Write a string to a file. The file is truncated to zero length --- before writing begins. -writeFile :: FilePath -> Text -> IO () -writeFile p = withFile p WriteMode . flip hPutStr - --- | Write a string to the end of a file. -appendFile :: FilePath -> Text -> IO () -appendFile p = withFile p AppendMode . flip hPutStr - -catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool) -catchError caller h Handle__{..} err - | isEOFError err = do - buf <- readIORef haCharBuffer - return $ if isEmptyBuffer buf - then (T.empty, True) - else (T.singleton '\r', True) - | otherwise = E.throwIO (augmentIOError err caller h) - --- | Wrap readChunk and return a value indicating if we're reached the EOF. --- This is needed because unpack_nl is unable to discern the difference --- between a buffer with just \r due to EOF or because not enough data was left --- for decoding. e.g. the final character decoded from the byte buffer was \r. -readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool) -readChunkEof hh buf = do t <- readChunk hh buf - return (t, False) - --- | /Experimental./ Read a single chunk of strict text from a --- 'Handle'. The size of the chunk depends on the amount of input --- currently buffered. --- --- This function blocks only if there is no data available, and EOF --- has not yet been reached. Once EOF is reached, this function --- returns an empty string instead of throwing an exception. -hGetChunk :: Handle -> IO Text -hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk - where - readSingleChunk hh@Handle__{..} = do - buf <- readIORef haCharBuffer - (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh - return (hh, t) - --- | Read the remaining contents of a 'Handle' as a string. The --- 'Handle' is closed once the contents have been read, or if an --- exception is thrown. --- --- Internally, this function reads a chunk at a time from the --- lower-level buffering abstraction, and concatenates the chunks into --- a single string once the entire file has been read. --- --- As a result, it requires approximately twice as much memory as its --- result to construct its result. For files more than a half of --- available RAM in size, this may result in memory exhaustion. -hGetContents :: Handle -> IO Text -hGetContents h = do - chooseGoodBuffering h - wantReadableHandle "hGetContents" h readAll - where - readAll hh@Handle__{..} = do - let readChunks = do - buf <- readIORef haCharBuffer - (t, eof) <- readChunkEof hh buf - `E.catch` catchError "hGetContents" h hh - if eof - then return [t] - else (t:) `fmap` readChunks - ts <- readChunks - (hh', _) <- hClose_help hh - return (hh'{haType=ClosedHandle}, T.concat ts) - --- | Use a more efficient buffer size if we're reading in --- block-buffered mode with the default buffer size. When we can --- determine the size of the handle we're reading, set the buffer size --- to that, so that we can read the entire file in one chunk. --- Otherwise, use a buffer size of at least 16KB. -chooseGoodBuffering :: Handle -> IO () -chooseGoodBuffering h = do - bufMode <- hGetBuffering h - case bufMode of - BlockBuffering Nothing -> do - d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> - if ioe_type e == InappropriateType - then return 16384 -- faster than the 2KB default - else E.throwIO e - when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d - _ -> return () - --- | Read a single line from a handle. -hGetLine :: Handle -> IO Text -hGetLine = hGetLineWith T.concat - --- | Write a string to a handle. -hPutStr :: Handle -> Text -> IO () --- This function is lifted almost verbatim from GHC.IO.Handle.Text. -hPutStr h t = do - (buffer_mode, nl) <- - wantWritableHandle "hPutStr" h $ \h_ -> do - bmode <- getSpareBuffer h_ - return (bmode, haOutputNL h_) - let str = stream t - case buffer_mode of - (NoBuffering, _) -> hPutChars h str - (LineBuffering, buf) -> writeLines h nl buf str - (BlockBuffering _, buf) - | nl == CRLF -> writeBlocksCRLF h buf str - | otherwise -> writeBlocksRaw h buf str - -hPutChars :: Handle -> Stream Char -> IO () -hPutChars h (Stream next0 s0 _len) = loop s0 - where - loop !s = case next0 s of - Done -> return () - Skip s' -> loop s' - Yield x s' -> hPutChar h x >> loop s' - --- The following functions are largely lifted from GHC.IO.Handle.Text, --- but adapted to a coinductive stream of data instead of an inductive --- list. --- --- We have several variations of more or less the same code for --- performance reasons. Splitting the original buffered write --- function into line- and block-oriented versions gave us a 2.1x --- performance improvement. Lifting out the raw/cooked newline --- handling gave a few more percent on top. - -writeLines :: Handle -> Newline -> Buffer CharBufElem -> Stream Char -> IO () -writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | x == '\n' -> do - n' <- if nl == CRLF - then do n1 <- writeCharBuf raw n '\r' - writeCharBuf raw n1 '\n' - else writeCharBuf raw n x - commit n' True{-needs flush-} False >>= outer s' - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - -writeBlocksCRLF :: Handle -> Buffer CharBufElem -> Stream Char -> IO () -writeBlocksCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | x == '\n' -> do n1 <- writeCharBuf raw n '\r' - writeCharBuf raw n1 '\n' >>= inner s' - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - -writeBlocksRaw :: Handle -> Buffer CharBufElem -> Stream Char -> IO () -writeBlocksRaw h buf0 (Stream next0 s0 _len) = outer s0 buf0 - where - outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) - where - inner !s !n = - case next0 s of - Done -> commit n False{-no flush-} True{-release-} >> return () - Skip s' -> inner s' n - Yield x s' - | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s - | otherwise -> writeCharBuf raw n x >>= inner s' - commit = commitBuffer h raw len - --- This function is completely lifted from GHC.IO.Handle.Text. -getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) -getSpareBuffer Handle__{haCharBuffer=ref, - haBuffers=spare_ref, - haBufferMode=mode} - = do - case mode of - NoBuffering -> return (mode, error "no buffer!") - _ -> do - bufs <- readIORef spare_ref - buf <- readIORef ref - case bufs of - BufferListCons b rest -> do - writeIORef spare_ref rest - return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) - BufferListNil -> do - new_buf <- newCharBuffer (bufSize buf) WriteBuffer - return (mode, new_buf) - - --- This function is completely lifted from GHC.IO.Handle.Text. -commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool - -> IO CharBuffer -commitBuffer hdl !raw !sz !count flush release = - wantWritableHandle "commitAndReleaseBuffer" hdl $ - commitBuffer' raw sz count flush release -{-# INLINE commitBuffer #-} - --- | Write a string to a handle, followed by a newline. -hPutStrLn :: Handle -> Text -> IO () -hPutStrLn h t = hPutStr h t >> hPutChar h '\n' - --- | The 'interact' function takes a function of type @Text -> Text@ --- as its argument. The entire input from the standard input device is --- passed to this function as its argument, and the resulting string --- is output on the standard output device. -interact :: (Text -> Text) -> IO () -interact f = putStr . f =<< getContents - --- | Read all user input on 'stdin' as a single string. -getContents :: IO Text -getContents = hGetContents stdin - --- | Read a single line of user input from 'stdin'. -getLine :: IO Text -getLine = hGetLine stdin - --- | Write a string to 'stdout'. -putStr :: Text -> IO () -putStr = hPutStr stdout - --- | Write a string to 'stdout', followed by a newline. -putStrLn :: Text -> IO () -putStrLn = hPutStrLn stdout +{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +-- | +-- Module : Data.Text.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Efficient locale-sensitive support for text I\/O. +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you want to do I\/O using the UTF-8 encoding, use "Data.Text.IO.Utf8", +-- which is faster than this module. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding, and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. + +module Data.Text.IO + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetChunk + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Data.Text (Text) +import Prelude hiding (appendFile, getContents, getLine, interact, + putStr, putStrLn, readFile, writeFile) +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, + withFile) +import qualified Control.Exception as E +import Control.Monad (liftM2, when) +import Data.IORef (readIORef) +import qualified Data.Text as T +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn) +import GHC.IO.Buffer (CharBuffer, isEmptyBuffer) +import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) +import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle) +import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..)) +import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) +import System.IO.Error (isEOFError) + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' +-- is a much faster and safer alternative. +readFile :: FilePath -> IO Text +readFile name = openFile name ReadMode >>= hGetContents + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile p = withFile p WriteMode . flip hPutStr + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile p = withFile p AppendMode . flip hPutStr + +catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool) +catchError caller h Handle__{..} err + | isEOFError err = do + buf <- readIORef haCharBuffer + return $ if isEmptyBuffer buf + then (T.empty, True) + else (T.singleton '\r', True) + | otherwise = E.throwIO (augmentIOError err caller h) + +-- | Wrap readChunk and return a value indicating if we're reached the EOF. +-- This is needed because unpack_nl is unable to discern the difference +-- between a buffer with just \r due to EOF or because not enough data was left +-- for decoding. e.g. the final character decoded from the byte buffer was \r. +readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool) +readChunkEof hh buf = do t <- readChunk hh buf + return (t, False) + +-- | Read a single chunk of strict text from a +-- 'Handle'. The size of the chunk depends on the amount of input +-- currently buffered. +-- +-- This function blocks only if there is no data available, and EOF +-- has not yet been reached. Once EOF is reached, this function +-- returns an empty string instead of throwing an exception. +-- +-- === Behavior +-- +-- Unlike byte-oriented functions, 'hGetChunk' operates on complete UTF-8 +-- characters. Since UTF-8 characters can occupy 1 to 4 bytes, this function +-- cannot guarantee reading an exact number of bytes. Instead, it reads +-- complete characters up to the handle's internal buffer limit. +-- +-- === Buffer Size +-- +-- The maximum chunk size is determined by the handle's internal character +-- buffer, which is set to 8192 bytes (2048 characters) by the GHC runtime +-- constant @dEFAULT_CHAR_BUFFER_SIZE@. This buffer size cannot be modified +-- through any public API. +-- +-- === UTF-8 Considerations +-- +-- When working with UTF-8 encoded text: +-- +-- * The function will never return a partial character +-- * The actual number of bytes read may vary depending on the character +-- encoding (ASCII characters = 1 byte, other Unicode characters = 2-4 bytes) +hGetChunk :: Handle -> IO Text +hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk + where + readSingleChunk hh@Handle__{..} = do + buf <- readIORef haCharBuffer + (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh + return (hh, t) + +-- | Read the remaining contents of a 'Handle' as a string. The +-- 'Handle' is closed once the contents have been read, or if an +-- exception is thrown. +-- +-- Internally, this function reads a chunk at a time from the +-- lower-level buffering abstraction, and concatenates the chunks into +-- a single string once the entire file has been read. +-- +-- As a result, it requires approximately twice as much memory as its +-- result to construct its result. For files more than a half of +-- available RAM in size, this may result in memory exhaustion. +hGetContents :: Handle -> IO Text +hGetContents h = do + chooseGoodBuffering h + wantReadableHandle "hGetContents" h readAll + where + readAll hh@Handle__{..} = do + let readChunks = do + buf <- readIORef haCharBuffer + (t, eof) <- readChunkEof hh buf + `E.catch` catchError "hGetContents" h hh + if eof + then return [t] + else (t:) `fmap` readChunks + ts <- readChunks + (hh', _) <- hClose_help hh + return (hh'{haType=ClosedHandle}, T.concat ts) + +-- | Use a more efficient buffer size if we're reading in +-- block-buffered mode with the default buffer size. When we can +-- determine the size of the handle we're reading, set the buffer size +-- to that, so that we can read the entire file in one chunk. +-- Otherwise, use a buffer size of at least 16KB. +chooseGoodBuffering :: Handle -> IO () +chooseGoodBuffering h = do + bufMode <- hGetBuffering h + case bufMode of + BlockBuffering Nothing -> do + d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> + if ioe_type e == InappropriateType + then return 16384 -- faster than the 2KB default + else E.throwIO e + when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d + _ -> return () + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = hGetLineWith T.concat + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = hGetContents stdin + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = hGetLine stdin + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = hPutStr stdout + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = hPutStrLn stdout diff --git a/src/Data/Text/IO/Utf8.hs b/src/Data/Text/IO/Utf8.hs --- a/src/Data/Text/IO/Utf8.hs +++ b/src/Data/Text/IO/Utf8.hs @@ -1,93 +1,94 @@ --- | --- Module : Data.Text.IO.Utf8 --- License : BSD-style --- Portability : GHC --- --- Efficient UTF-8 support for text I\/O. --- Unlike @Data.Text.IO@, these functions do not depend on the locale --- and do not do line ending conversion. -module Data.Text.IO.Utf8 - ( - -- * File-at-a-time operations - readFile - , writeFile - , appendFile - -- * Operations on handles - , hGetContents - , hGetLine - , hPutStr - , hPutStrLn - -- * Special cases for standard input and output - , interact - , getContents - , getLine - , putStr - , putStrLn - ) where - -import Prelude hiding (readFile, writeFile, appendFile, interact, getContents, getLine, putStr, putStrLn) -import Control.Exception (evaluate) -import Control.Monad ((<=<)) -import Data.ByteString (ByteString) -import qualified Data.ByteString.Char8 as B -import Data.Text (Text) -import Data.Text.Encoding (decodeUtf8, encodeUtf8) -import GHC.IO.Handle (Handle) -import qualified Data.ByteString.Char8 as B.Char8 - -decodeUtf8IO :: ByteString -> IO Text -decodeUtf8IO = evaluate . decodeUtf8 - --- | The 'readFile' function reads a file and returns the contents of --- the file as a string. The entire file is read strictly, as with --- 'getContents'. -readFile :: FilePath -> IO Text -readFile = decodeUtf8IO <=< B.readFile - --- | Write a string to a file. The file is truncated to zero length --- before writing begins. -writeFile :: FilePath -> Text -> IO () -writeFile fp = B.writeFile fp . encodeUtf8 - --- | Write a string to the end of a file. -appendFile :: FilePath -> Text -> IO () -appendFile fp = B.appendFile fp . encodeUtf8 - --- | Read the remaining contents of a 'Handle' as a string. -hGetContents :: Handle -> IO Text -hGetContents = decodeUtf8IO <=< B.hGetContents - --- | Read a single line from a handle. -hGetLine :: Handle -> IO Text -hGetLine = decodeUtf8IO <=< B.hGetLine - --- | Write a string to a handle. -hPutStr :: Handle -> Text -> IO () -hPutStr h = B.hPutStr h . encodeUtf8 - --- | Write a string to a handle, followed by a newline. -hPutStrLn :: Handle -> Text -> IO () -hPutStrLn h t = hPutStr h t >> B.hPutStr h (B.Char8.singleton '\n') - --- | The 'interact' function takes a function of type @Text -> Text@ --- as its argument. The entire input from the standard input device is --- passed to this function as its argument, and the resulting string --- is output on the standard output device. -interact :: (Text -> Text) -> IO () -interact f = putStr . f =<< getContents - --- | Read all user input on 'stdin' as a single string. -getContents :: IO Text -getContents = decodeUtf8IO =<< B.getContents - --- | Read a single line of user input from 'stdin'. -getLine :: IO Text -getLine = decodeUtf8IO =<< B.getLine - --- | Write a string to 'stdout'. -putStr :: Text -> IO () -putStr = B.putStr . encodeUtf8 - --- | Write a string to 'stdout', followed by a newline. -putStrLn :: Text -> IO () -putStrLn t = B.putStr (encodeUtf8 t) >> B.putStr (B.Char8.singleton '\n') +-- | +-- Module : Data.Text.IO.Utf8 +-- License : BSD-style +-- Portability : GHC +-- +-- Efficient UTF-8 support for text I\/O. +-- Unlike "Data.Text.IO", these functions do not depend on the locale +-- and do not do line ending conversion. +module Data.Text.IO.Utf8 + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Prelude () +import Control.Exception (evaluate) +import Control.Monad ((<=<), (=<<)) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Function ((.)) +import Data.Text (Text) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import GHC.IO.Handle (Handle) +import System.IO (IO, FilePath) + +decodeUtf8IO :: ByteString -> IO Text +decodeUtf8IO = evaluate . decodeUtf8 + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +readFile :: FilePath -> IO Text +readFile = decodeUtf8IO <=< B.readFile + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile fp = B.writeFile fp . encodeUtf8 + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile fp = B.appendFile fp . encodeUtf8 + +-- | Read the remaining contents of a 'Handle' as a string. +hGetContents :: Handle -> IO Text +hGetContents = decodeUtf8IO <=< B.hGetContents + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = decodeUtf8IO <=< B.hGetLine + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h = B.hPutStr h . encodeUtf8 + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h = B.hPutStrLn h . encodeUtf8 + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = decodeUtf8IO =<< B.getContents + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = decodeUtf8IO =<< B.getLine + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = B.putStr . encodeUtf8 + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = B.putStrLn . encodeUtf8 diff --git a/src/Data/Text/Internal.hs b/src/Data/Text/Internal.hs --- a/src/Data/Text/Internal.hs +++ b/src/Data/Text/Internal.hs @@ -1,275 +1,274 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE CPP #-} -{-# LANGUAGE DeriveDataTypeable #-} -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE UnboxedTuples #-} -{-# OPTIONS_HADDOCK not-home #-} - --- | --- Module : Data.Text.Internal --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- A module containing private 'Text' internals. This exposes the --- 'Text' representation and low level construction functions. --- Modules which extend the 'Text' system may need to use this module. --- --- You should not use this module unless you are determined to monkey --- with the internals, as the functions here do just about nothing to --- preserve data invariants. You have been warned! - -module Data.Text.Internal - ( - -- * Types - -- $internals - Text(..) - , StrictText - -- * Construction - , text - , textP - -- * Safety - , safe - -- * Code that must be here for accessibility - , empty - , append - -- * Utilities - , firstf - -- * Checked multiplication - , mul - , mul32 - , mul64 - -- * Debugging - , showText - -- * Conversions - , pack - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -import GHC.Stack (HasCallStack) -#endif -import Control.Monad.ST (ST, runST) -import Data.Bits -import Data.Int (Int32, Int64) -import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) -import Data.Typeable (Typeable) -import qualified Data.Text.Array as A - --- | A space efficient, packed, unboxed Unicode text type. -data Text = Text - {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8 - {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence - {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence - deriving (Typeable) - --- | Type synonym for the strict flavour of 'Text'. -type StrictText = Text - --- | Smart constructor. -text_ :: -#if defined(ASSERTS) - HasCallStack => -#endif - A.Array -- ^ bytearray encoded as UTF-8 - -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence - -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence - -> Text -text_ arr off len = -#if defined(ASSERTS) - let c = A.unsafeIndex arr off - in assert (len >= 0) . - assert (off >= 0) . - assert (len == 0 || c < 0x80 || c >= 0xC0) $ -#endif - Text arr off len -{-# INLINE text_ #-} - --- | /O(1)/ The empty 'Text'. -empty :: Text -empty = Text A.empty 0 0 -{-# NOINLINE empty #-} - --- | /O(n)/ Appends one 'Text' to the other by copying both of them --- into a new 'Text'. -append :: Text -> Text -> Text -append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) - | len1 == 0 = b - | len2 == 0 = a - | len > 0 = Text (A.run x) 0 len - | otherwise = error $ "Data.Text.append: size overflow" - where - len = len1+len2 - x :: ST s (A.MArray s) - x = do - arr <- A.new len - A.copyI len1 arr 0 arr1 off1 - A.copyI len2 arr len1 arr2 off2 - return arr -{-# NOINLINE append #-} - --- | Construct a 'Text' without invisibly pinning its byte array in --- memory if its length has dwindled to zero. --- It ensures that empty 'Text' values are shared. -text :: -#if defined(ASSERTS) - HasCallStack => -#endif - A.Array -- ^ bytearray encoded as UTF-8 - -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence - -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence - -> Text -text arr off len | len == 0 = empty - | otherwise = text_ arr off len -{-# INLINE [0] text #-} - -textP :: A.Array -> Int -> Int -> Text -{-# DEPRECATED textP "Use text instead" #-} -textP = text - --- | A useful 'show'-like function for debugging purposes. -showText :: Text -> String -showText (Text arr off len) = - "Text " ++ show (A.toList arr off len) ++ ' ' : - show off ++ ' ' : show len - --- | Map a 'Char' to a 'Text'-safe value. --- --- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode --- scalar values, but are unfortunately admitted as valid 'Char' --- values by Haskell. They cannot be represented in a 'Text'. This --- function remaps those code points to the Unicode replacement --- character (U+FFFD, \'�\'), and leaves other code points --- unchanged. -safe :: Char -> Char -safe c - | ord c .&. 0x1ff800 /= 0xd800 = c - | otherwise = '\xfffd' -{-# INLINE [0] safe #-} - --- | Apply a function to the first element of an optional pair. -firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) -firstf f (Just (a, b)) = Just (f a, b) -firstf _ Nothing = Nothing - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul :: Int -> Int -> Int -mul a b - | finiteBitSize (0 :: Word) == 64 - = int64ToInt $ intToInt64 a `mul64` intToInt64 b - | otherwise - = int32ToInt $ intToInt32 a `mul32` intToInt32 b -{-# INLINE mul #-} -infixl 7 `mul` - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul64 :: Int64 -> Int64 -> Int64 -mul64 a b - | a >= 0 && b >= 0 = mul64_ a b - | a >= 0 = -mul64_ a (-b) - | b >= 0 = -mul64_ (-a) b - | otherwise = mul64_ (-a) (-b) -{-# INLINE mul64 #-} -infixl 7 `mul64` - -mul64_ :: Int64 -> Int64 -> Int64 -mul64_ a b - | ahi > 0 && bhi > 0 = error "overflow" - | top > 0x7fffffff = error "overflow" - | total < 0 = error "overflow" - | otherwise = total - where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) - (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) - top = ahi * blo + alo * bhi - total = (top `shiftL` 32) + alo * blo -{-# INLINE mul64_ #-} - --- | Checked multiplication. Calls 'error' if the result would --- overflow. -mul32 :: Int32 -> Int32 -> Int32 -mul32 a b = case int32ToInt64 a * int32ToInt64 b of - ab | ab < min32 || ab > max32 -> error "overflow" - | otherwise -> int64ToInt32 ab - where min32 = -0x80000000 :: Int64 - max32 = 0x7fffffff -{-# INLINE mul32 #-} -infixl 7 `mul32` - -intToInt64 :: Int -> Int64 -intToInt64 = fromIntegral - -int64ToInt :: Int64 -> Int -int64ToInt = fromIntegral - -intToInt32 :: Int -> Int32 -intToInt32 = fromIntegral - -int32ToInt :: Int32 -> Int -int32ToInt = fromIntegral - -int32ToInt64 :: Int32 -> Int64 -int32ToInt64 = fromIntegral - -int64ToInt32 :: Int64 -> Int32 -int64ToInt32 = fromIntegral - --- $internals --- --- Internally, the 'Text' type is represented as an array of 'Word8' --- UTF-8 code units. The offset and length fields in the constructor --- are in these units, /not/ units of 'Char'. --- --- Invariants that all functions must maintain: --- --- * Since the 'Text' type uses UTF-8 internally, it cannot represent --- characters in the reserved surrogate code point range U+D800 to --- U+DFFF. To maintain this invariant, the 'safe' function maps --- 'Char' values in this range to the replacement character (U+FFFD, --- \'�\'). --- --- * Offset and length must point to a valid UTF-8 sequence of bytes. --- Violation of this may cause memory access violation and divergence. - --- ----------------------------------------------------------------------------- --- * Conversion to/from 'Text' - --- | /O(n)/ Convert a 'String' into a 'Text'. --- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id': --- --- >>> Data.Text.unpack (pack "\55555") --- "\65533" -pack :: String -> Text -pack [] = empty -pack xs = runST $ do - -- It's tempting to allocate a buffer of 4 * length xs bytes, - -- but not only it's wasteful for predominantly ASCII arguments, - -- the computation of length xs would force allocation of the entire xs at once. - let dstLen = 64 - dst <- A.new dstLen - outer dst dstLen 0 xs - where - outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text - outer !dst !dstLen = inner - where - inner !dstOff [] = do - A.shrinkM dst dstOff - arr <- A.unsafeFreeze dst - return (Text arr 0 dstOff) - inner !dstOff ccs@(c : cs) - -- Each 'Char' takes up to 4 bytes - | dstOff + 4 > dstLen = do - -- Double size of the buffer - let !dstLen' = dstLen * 2 - dst' <- A.resizeM dst dstLen' - outer dst' dstLen' dstOff ccs - | otherwise = do - d <- unsafeWrite dst dstOff (safe c) - inner (dstOff + d) cs -{-# NOINLINE [0] pack #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE UnboxedTuples #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. +-- +-- You should not use this module unless you are determined to monkey +-- with the internals, as the functions here do just about nothing to +-- preserve data invariants. You have been warned! + +module Data.Text.Internal + ( + -- * Types + -- $internals + Text(..) + , StrictText + -- * Construction + , text + , textP + -- * Safety + , safe + -- * Code that must be here for accessibility + , empty + , append + -- * Utilities + , firstf + -- * Checked multiplication + , mul + , mul32 + , mul64 + -- * Debugging + , showText + -- * Conversions + , pack + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Control.Monad.ST (ST, runST) +import Data.Bits +import Data.Int (Int32, Int64) +import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) +import qualified Data.Text.Array as A + +-- | A space efficient, packed, unboxed Unicode text type. +data Text = Text + {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8 + {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + +-- | Type synonym for the strict flavour of 'Text'. +-- +-- @since 2.1.1 +type StrictText = Text + +-- | Smart constructor. +text_ :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text_ arr off len = +#if defined(ASSERTS) + let c = A.unsafeIndex arr off + in assert (len >= 0) . + assert (off >= 0) . + assert (len == 0 || c < 0x80 || c >= 0xC0) $ +#endif + Text arr off len +{-# INLINE text_ #-} + +-- | /O(1)/ The empty 'Text'. +empty :: Text +empty = Text A.empty 0 0 +{-# NOINLINE empty #-} + +-- | /O(n)/ Appends one 'Text' to the other by copying both of them +-- into a new 'Text'. +append :: Text -> Text -> Text +append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) + | len1 == 0 = b + | len2 == 0 = a + | len > 0 = Text (A.run x) 0 len + | otherwise = error $ "Data.Text.append: size overflow" + where + len = len1+len2 + x :: ST s (A.MArray s) + x = do + arr <- A.new len + A.copyI len1 arr 0 arr1 off1 + A.copyI len2 arr len1 arr2 off2 + return arr +{-# NOINLINE append #-} + +-- | Construct a 'Text' without invisibly pinning its byte array in +-- memory if its length has dwindled to zero. +-- It ensures that empty 'Text' values are shared. +text :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text arr off len | len == 0 = empty + | otherwise = text_ arr off len +{-# INLINE [0] text #-} + +textP :: A.Array -> Int -> Int -> Text +{-# DEPRECATED textP "Use text instead" #-} +textP = text + +-- | A useful 'show'-like function for debugging purposes. +showText :: Text -> String +showText (Text arr off len) = + "Text " ++ show (A.toList arr off len) ++ ' ' : + show off ++ ' ' : show len + +-- | Map a 'Char' to a 'Text'-safe value. +-- +-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode +-- scalar values, but are unfortunately admitted as valid 'Char' +-- values by Haskell. They cannot be represented in a 'Text'. This +-- function remaps those code points to the Unicode replacement +-- character (U+FFFD, \'�\'), and leaves other code points +-- unchanged. +safe :: Char -> Char +safe c + | ord c .&. 0x1ff800 /= 0xd800 = c + | otherwise = '\xfffd' +{-# INLINE [0] safe #-} + +-- | Apply a function to the first element of an optional pair. +firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) +firstf f (Just (a, b)) = Just (f a, b) +firstf _ Nothing = Nothing + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul :: Int -> Int -> Int +mul a b + | finiteBitSize (0 :: Word) == 64 + = int64ToInt $ intToInt64 a `mul64` intToInt64 b + | otherwise + = int32ToInt $ intToInt32 a `mul32` intToInt32 b +{-# INLINE mul #-} +infixl 7 `mul` + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul64 :: Int64 -> Int64 -> Int64 +mul64 a b + | a >= 0 && b >= 0 = mul64_ a b + | a >= 0 = -mul64_ a (-b) + | b >= 0 = -mul64_ (-a) b + | otherwise = mul64_ (-a) (-b) +{-# INLINE mul64 #-} +infixl 7 `mul64` + +mul64_ :: Int64 -> Int64 -> Int64 +mul64_ a b + | ahi > 0 && bhi > 0 = error "overflow" + | top > 0x7fffffff = error "overflow" + | total < 0 = error "overflow" + | otherwise = total + where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) + (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) + top = ahi * blo + alo * bhi + total = (top `shiftL` 32) + alo * blo +{-# INLINE mul64_ #-} + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul32 :: Int32 -> Int32 -> Int32 +mul32 a b = case int32ToInt64 a * int32ToInt64 b of + ab | ab < min32 || ab > max32 -> error "overflow" + | otherwise -> int64ToInt32 ab + where min32 = -0x80000000 :: Int64 + max32 = 0x7fffffff +{-# INLINE mul32 #-} +infixl 7 `mul32` + +intToInt64 :: Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Int +int64ToInt = fromIntegral + +intToInt32 :: Int -> Int32 +intToInt32 = fromIntegral + +int32ToInt :: Int32 -> Int +int32ToInt = fromIntegral + +int32ToInt64 :: Int32 -> Int64 +int32ToInt64 = fromIntegral + +int64ToInt32 :: Int64 -> Int32 +int64ToInt32 = fromIntegral + +-- $internals +-- +-- Internally, the 'Text' type is represented as an array of 'Word8' +-- UTF-8 code units. The offset and length fields in the constructor +-- are in these units, /not/ units of 'Char'. +-- +-- Invariants that all functions must maintain: +-- +-- * Since the 'Text' type uses UTF-8 internally, it cannot represent +-- characters in the reserved surrogate code point range U+D800 to +-- U+DFFF. To maintain this invariant, the 'safe' function maps +-- 'Char' values in this range to the replacement character (U+FFFD, +-- \'�\'). +-- +-- * Offset and length must point to a valid UTF-8 sequence of bytes. +-- Violation of this may cause memory access violation and divergence. + +-- ----------------------------------------------------------------------------- +-- * Conversion to/from 'Text' + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.unpack (pack "\55555") +-- "\65533" +pack :: String -> Text +pack [] = empty +pack xs = runST $ do + -- It's tempting to allocate a buffer of 4 * length xs bytes, + -- but not only it's wasteful for predominantly ASCII arguments, + -- the computation of length xs would force allocation of the entire xs at once. + let dstLen = 64 + dst <- A.new dstLen + outer dst dstLen 0 xs + where + outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text + outer !dst !dstLen = inner + where + inner !dstOff [] = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + inner !dstOff ccs@(c : cs) + -- Each 'Char' takes up to 4 bytes + | dstOff + 4 > dstLen = do + -- Double size of the buffer + let !dstLen' = dstLen * 2 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' dstOff ccs + | otherwise = do + d <- unsafeWrite dst dstOff (safe c) + inner (dstOff + d) cs +{-# NOINLINE [0] pack #-} diff --git a/src/Data/Text/Internal/Builder.hs b/src/Data/Text/Internal/Builder.hs --- a/src/Data/Text/Internal/Builder.hs +++ b/src/Data/Text/Internal/Builder.hs @@ -1,343 +1,350 @@ -{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} -{-# OPTIONS_HADDOCK not-home #-} - ------------------------------------------------------------------------------ --- | --- Module : Data.Text.Internal.Builder --- Copyright : (c) 2013 Bryan O'Sullivan --- (c) 2010 Johan Tibell --- License : BSD-style (see LICENSE) --- --- Maintainer : Johan Tibell --- Stability : experimental --- Portability : portable to Hugs and GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Efficient construction of lazy @Text@ values. The principal --- operations on a @Builder@ are @singleton@, @fromText@, and --- @fromLazyText@, which construct new builders, and 'mappend', which --- concatenates two builders. --- --- To get maximum performance when building lazy @Text@ values using a --- builder, associate @mappend@ calls to the right. For example, --- prefer --- --- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') --- --- to --- --- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' --- --- as the latter associates @mappend@ to the left. --- ------------------------------------------------------------------------------ - -module Data.Text.Internal.Builder - ( -- * Public API - -- ** The Builder type - Builder - , toLazyText - , toLazyTextWith - - -- ** Constructing Builders - , singleton - , fromText - , fromLazyText - , fromString - - -- ** Flushing the buffer state - , flush - - -- * Internal functions - , append' - , ensureFree - , writeN - ) where - -import Control.Monad.ST (ST, runST) -import Data.Monoid (Monoid(..)) -#if !MIN_VERSION_base(4,11,0) -import Data.Semigroup (Semigroup(..)) -#endif -import Data.Text.Internal (Text(..), safe) -import Data.Text.Internal.Lazy (smallChunkSize) -import Data.Text.Unsafe (inlineInterleaveST) -import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import Prelude hiding (map, putChar) - -import qualified Data.String as String -import qualified Data.Text as S -import qualified Data.Text.Array as A -import qualified Data.Text.Lazy as L - -#if defined(ASSERTS) -import GHC.Stack (HasCallStack) -#endif - ------------------------------------------------------------------------- - --- | A @Builder@ is an efficient way to build lazy @Text@ values. --- There are several functions for constructing builders, but only one --- to inspect them: to extract any data, you have to turn them into --- lazy @Text@ values using @toLazyText@. --- --- Internally, a builder constructs a lazy @Text@ by filling arrays --- piece by piece. As each buffer is filled, it is \'popped\' off, to --- become a new chunk of the resulting lazy @Text@. All this is --- hidden from the user of the @Builder@. -newtype Builder = Builder { - -- Invariant (from Data.Text.Lazy): - -- The lists include no null Texts. - runBuilder :: forall s. (Buffer s -> ST s [S.Text]) - -> Buffer s - -> ST s [S.Text] - } - -instance Semigroup Builder where - (<>) = append - {-# INLINE (<>) #-} - -instance Monoid Builder where - mempty = empty - {-# INLINE mempty #-} - mappend = (<>) - {-# INLINE mappend #-} - mconcat = foldr mappend Data.Monoid.mempty - {-# INLINE mconcat #-} - --- | Performs replacement on invalid scalar values: --- --- >>> :set -XOverloadedStrings --- >>> "\55555" :: Builder --- "\65533" -instance String.IsString Builder where - fromString = fromString - {-# INLINE fromString #-} - -instance Show Builder where - show = show . toLazyText - -instance Eq Builder where - a == b = toLazyText a == toLazyText b - -instance Ord Builder where - a <= b = toLazyText a <= toLazyText b - ------------------------------------------------------------------------- - --- | /O(1)./ The empty @Builder@, satisfying --- --- * @'toLazyText' 'empty' = 'L.empty'@ --- -empty :: Builder -empty = Builder (\ k buf -> k buf) -{-# INLINE empty #-} - --- | /O(1)./ A @Builder@ taking a single character, satisfying --- --- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ --- -singleton :: -#if defined(ASSERTS) - HasCallStack => -#endif - Char -> Builder -singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) -{-# INLINE singleton #-} - ------------------------------------------------------------------------- - --- | /O(1)./ The concatenation of two builders, an associative --- operation with identity 'empty', satisfying --- --- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ --- -append :: Builder -> Builder -> Builder -append (Builder f) (Builder g) = Builder (f . g) -{-# INLINE [0] append #-} - --- TODO: Experiment to find the right threshold. -copyLimit :: Int -copyLimit = 128 - --- This function attempts to merge small @Text@ values instead of --- treating each value as its own chunk. We may not always want this. - --- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying --- --- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ --- -fromText :: S.Text -> Builder -fromText t@(Text arr off l) - | S.null t = empty - | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off - | otherwise = flush `append` mapBuilder (t :) -{-# INLINE [1] fromText #-} - -{-# RULES -"fromText/pack" forall s . - fromText (S.pack s) = fromString s - #-} - --- | /O(1)./ A Builder taking a @String@, satisfying --- --- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ --- --- Performs replacement on invalid scalar values: --- --- >>> fromString "\55555" --- "\65533" --- --- @since 1.2.0.0 -fromString :: String -> Builder -fromString str = Builder $ \k (Buffer p0 o0 u0 l0) -> - let loop !marr !o !u !l [] = k (Buffer marr o u l) - loop marr o u l s@(c:cs) - | l <= 3 = do - A.shrinkM marr (o + u) - arr <- A.unsafeFreeze marr - let !t = Text arr o u - marr' <- A.new chunkSize - ts <- inlineInterleaveST (loop marr' 0 0 chunkSize s) - return $ t : ts - | otherwise = do - n <- unsafeWrite marr (o+u) (safe c) - loop marr o (u+n) (l-n) cs - in loop p0 o0 u0 l0 str - where - chunkSize = smallChunkSize -{-# INLINE fromString #-} - --- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying --- --- * @'toLazyText' ('fromLazyText' t) = t@ --- -fromLazyText :: L.Text -> Builder -fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) -{-# INLINE fromLazyText #-} - ------------------------------------------------------------------------- - --- Our internal buffer type -data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) - {-# UNPACK #-} !Int -- offset - {-# UNPACK #-} !Int -- used units - {-# UNPACK #-} !Int -- length left - ------------------------------------------------------------------------- - --- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default --- buffer size. The construction work takes place if and when the --- relevant part of the lazy @Text@ is demanded. -toLazyText :: Builder -> L.Text -toLazyText = toLazyTextWith smallChunkSize - --- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given --- size for the initial buffer. The construction work takes place if --- and when the relevant part of the lazy @Text@ is demanded. --- --- If the initial buffer is too small to hold all data, subsequent --- buffers will be the default buffer size. -toLazyTextWith :: Int -> Builder -> L.Text -toLazyTextWith chunkSize m = L.fromChunks (runST $ - newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) - --- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, --- yielding a new chunk in the result lazy @Text@. -flush :: Builder -flush = Builder $ \ k buf@(Buffer p o u l) -> - if u == 0 - then k buf - else do arr <- A.unsafeFreeze p - let !b = Buffer p (o+u) 0 l - !t = Text arr o u - ts <- inlineInterleaveST (k b) - return $! t : ts -{-# INLINE [1] flush #-} --- defer inlining so that flush/flush rule may fire. - ------------------------------------------------------------------------- - --- | Sequence an ST operation on the buffer -withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder -withBuffer f = Builder $ \k buf -> f buf >>= k -{-# INLINE withBuffer #-} - --- | Get the size of the buffer -withSize :: (Int -> Builder) -> Builder -withSize f = Builder $ \ k buf@(Buffer _ _ _ l) -> - runBuilder (f l) k buf -{-# INLINE withSize #-} - --- | Map the resulting list of texts. -mapBuilder :: ([S.Text] -> [S.Text]) -> Builder -mapBuilder f = Builder (fmap f .) - ------------------------------------------------------------------------- - --- | Ensure that there are at least @n@ many elements available. -ensureFree :: Int -> Builder -ensureFree !n = withSize $ \ l -> - if n <= l - then empty - else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) -{-# INLINE [0] ensureFree #-} - -writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder -writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) -{-# INLINE [0] writeAtMost #-} - --- | Ensure that @n@ many elements are available, and then use @f@ to --- write some elements into the memory. -writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder -writeN n f = writeAtMost n (\ p o -> f p o >> return n) -{-# INLINE writeN #-} - -writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) -writeBuffer f (Buffer p o u l) = do - n <- f p (o+u) - return $! Buffer p o (u+n) (l-n) -{-# INLINE writeBuffer #-} - -newBuffer :: Int -> ST s (Buffer s) -newBuffer size = do - arr <- A.new size - return $! Buffer arr 0 0 size -{-# INLINE newBuffer #-} - ------------------------------------------------------------------------- --- Some nice rules for Builder - --- This function makes GHC understand that 'writeN' and 'ensureFree' --- are *not* recursive in the precense of the rewrite rules below. --- This is not needed with GHC 7+. -append' :: Builder -> Builder -> Builder -append' (Builder f) (Builder g) = Builder (f . g) -{-# INLINE append' #-} - -{-# RULES - -"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) - (g::forall s. A.MArray s -> Int -> ST s Int) ws. - append (writeAtMost a f) (append (writeAtMost b g) ws) = - append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> - g marr (o+n) >>= \ m -> - let s = n+m in s `seq` return s)) ws - -"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) - (g::forall s. A.MArray s -> Int -> ST s Int). - append (writeAtMost a f) (writeAtMost b g) = - writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> - g marr (o+n) >>= \ m -> - let s = n+m in s `seq` return s) - -"ensureFree/ensureFree" forall a b . - append (ensureFree a) (ensureFree b) = ensureFree (max a b) - -"flush/flush" - append flush flush = flush - - #-} +{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} +{-# OPTIONS_HADDOCK not-home #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Data.Text.Internal.Builder +-- Copyright : (c) 2013 Bryan O'Sullivan +-- (c) 2010 Johan Tibell +-- License : BSD-style (see LICENSE) +-- +-- Maintainer : Johan Tibell +-- Stability : experimental +-- Portability : portable to Hugs and GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. +-- +-- To get maximum performance when building lazy @Text@ values using a +-- builder, associate @mappend@ calls to the right. For example, +-- prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. +-- +----------------------------------------------------------------------------- + +module Data.Text.Internal.Builder + ( -- * Public API + -- ** The Builder type + Builder + , LazyTextBuilder + , toLazyText + , toLazyTextWith + + -- ** Constructing Builders + , singleton + , fromText + , fromLazyText + , fromString + + -- ** Flushing the buffer state + , flush + + -- * Internal functions + , append' + , ensureFree + , writeN + ) where + +import Control.Monad.ST (ST, runST) +import Data.Monoid (Monoid(..)) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Lazy (smallChunkSize) +import Data.Text.Unsafe (inlineInterleaveST) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Prelude hiding (map, putChar) + +import qualified Data.String as String +import qualified Data.Text as S +import qualified Data.Text.Array as A +import qualified Data.Text.Lazy as L + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +------------------------------------------------------------------------ + +-- | A @Builder@ is an efficient way to build lazy @Text@ values. +-- There are several functions for constructing builders, but only one +-- to inspect them: to extract any data, you have to turn them into +-- lazy @Text@ values using @toLazyText@. +-- +-- Internally, a builder constructs a lazy @Text@ by filling arrays +-- piece by piece. As each buffer is filled, it is \'popped\' off, to +-- become a new chunk of the resulting lazy @Text@. All this is +-- hidden from the user of the @Builder@. +newtype Builder = Builder { + -- Invariant (from Data.Text.Lazy): + -- The lists include no null Texts. + runBuilder :: forall s. (Buffer s -> ST s [S.Text]) + -> Buffer s + -> ST s [S.Text] + } + +-- | @since 2.1.2 +type LazyTextBuilder = Builder + +instance Semigroup Builder where + (<>) = append + {-# INLINE (<>) #-} + +instance Monoid Builder where + mempty = empty + {-# INLINE mempty #-} + mappend = (<>) + {-# INLINE mappend #-} + mconcat = foldr mappend Data.Monoid.mempty + {-# INLINE mconcat #-} + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Builder +-- "\65533" +instance String.IsString Builder where + fromString = fromString + {-# INLINE fromString #-} + +instance Show Builder where + show = show . toLazyText + +instance Eq Builder where + a == b = toLazyText a == toLazyText b + +instance Ord Builder where + a <= b = toLazyText a <= toLazyText b + +------------------------------------------------------------------------ + +-- | /O(1)./ The empty @Builder@, satisfying +-- +-- * @'toLazyText' 'empty' = 'L.empty'@ +-- +empty :: Builder +empty = Builder (\ k buf -> k buf) +{-# INLINE empty #-} + +-- | /O(1)./ A @Builder@ taking a single character, satisfying +-- +-- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ +-- +singleton :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> Builder +singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) +{-# INLINE singleton #-} + +------------------------------------------------------------------------ + +-- | /O(1)./ The concatenation of two builders, an associative +-- operation with identity 'empty', satisfying +-- +-- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ +-- +append :: Builder -> Builder -> Builder +append (Builder f) (Builder g) = Builder (f . g) +{-# INLINE [0] append #-} + +-- TODO: Experiment to find the right threshold. +copyLimit :: Int +copyLimit = 128 + +-- This function attempts to merge small @Text@ values instead of +-- treating each value as its own chunk. We may not always want this. + +-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying +-- +-- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ +-- +fromText :: S.Text -> Builder +fromText t@(Text arr off l) + | S.null t = empty + | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off + | otherwise = flush `append` mapBuilder (t :) +{-# INLINE [1] fromText #-} + +{-# RULES +"fromText/pack" forall s . + fromText (S.pack s) = fromString s + #-} + +-- | /O(1)./ A Builder taking a @String@, satisfying +-- +-- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ +-- +-- Performs replacement on invalid scalar values: +-- +-- >>> fromString "\55555" +-- "\65533" +-- +-- @since 1.2.0.0 +fromString :: String -> Builder +fromString str = Builder $ \k (Buffer p0 o0 u0) -> do + len <- A.getSizeofMArray p0 + -- `end` is 3 bytes before the actual end of `marr` + -- to make sure there's room for a 4-byte UTF-8 code point. + let loop !marr !o !u !_ [] = k (Buffer marr o u) + loop marr o u end s@(c:cs) + | u >= end = do + A.shrinkM marr (o + u) + arr <- A.unsafeFreeze marr + let !t = Text arr o u + marr' <- A.new chunkSize + ts <- inlineInterleaveST (loop marr' 0 0 (chunkSize - 3) s) + return $ t : ts + | otherwise = do + n <- unsafeWrite marr (o+u) (safe c) + loop marr o (u+n) end cs + loop p0 o0 u0 (len - o0 - 3) str + where + chunkSize = smallChunkSize +{-# INLINEABLE fromString #-} + +-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying +-- +-- * @'toLazyText' ('fromLazyText' t) = t@ +-- +fromLazyText :: L.Text -> Builder +fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) +{-# INLINE fromLazyText #-} + +------------------------------------------------------------------------ + +-- Our internal buffer type +data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) + {-# UNPACK #-} !Int -- offset + {-# UNPACK #-} !Int -- used units + +------------------------------------------------------------------------ + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default +-- buffer size. The construction work takes place if and when the +-- relevant part of the lazy @Text@ is demanded. +toLazyText :: Builder -> L.Text +toLazyText = toLazyTextWith smallChunkSize + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given +-- size for the initial buffer. The construction work takes place if +-- and when the relevant part of the lazy @Text@ is demanded. +-- +-- If the initial buffer is too small to hold all data, subsequent +-- buffers will be the default buffer size. +toLazyTextWith :: Int -> Builder -> L.Text +toLazyTextWith chunkSize m = L.fromChunks (runST $ + newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) + +-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, +-- yielding a new chunk in the result lazy @Text@. +flush :: Builder +flush = Builder $ \ k buf@(Buffer p o u) -> + if u == 0 + then k buf + else do arr <- A.unsafeFreeze p + let !b = Buffer p (o+u) 0 + !t = Text arr o u + ts <- inlineInterleaveST (k b) + return $! t : ts +{-# INLINE [1] flush #-} +-- defer inlining so that flush/flush rule may fire. + +------------------------------------------------------------------------ + +-- | Sequence an ST operation on the buffer +withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder +withBuffer f = Builder $ \k buf -> f buf >>= k +{-# INLINE withBuffer #-} + +-- | Get the size of the buffer +withSize :: (Int -> Builder) -> Builder +withSize f = Builder $ \ k buf@(Buffer arr offset used) -> do + len <- A.getSizeofMArray arr + runBuilder (f (len - offset - used)) k buf +{-# INLINE withSize #-} + +-- | Map the resulting list of texts. +mapBuilder :: ([S.Text] -> [S.Text]) -> Builder +mapBuilder f = Builder (fmap f .) + +------------------------------------------------------------------------ + +-- | Ensure that there are at least @n@ many elements available. +ensureFree :: Int -> Builder +ensureFree !n = withSize $ \ l -> + if n <= l + then empty + else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) +{-# INLINE [0] ensureFree #-} + +writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder +writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) +{-# INLINE [0] writeAtMost #-} + +-- | Ensure that @n@ many elements are available, and then use @f@ to +-- write some elements into the memory. +writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder +writeN n f = writeAtMost n (\ p o -> f p o >> return n) +{-# INLINE writeN #-} + +writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) +writeBuffer f (Buffer p o u) = do + n <- f p (o+u) + return $! Buffer p o (u+n) +{-# INLINE writeBuffer #-} + +newBuffer :: Int -> ST s (Buffer s) +newBuffer size = do + arr <- A.new size + return $! Buffer arr 0 0 +{-# INLINE newBuffer #-} + +------------------------------------------------------------------------ +-- Some nice rules for Builder + +-- This function makes GHC understand that 'writeN' and 'ensureFree' +-- are *not* recursive in the presence of the rewrite rules below. +-- This is not needed with GHC 7+. +append' :: Builder -> Builder -> Builder +append' (Builder f) (Builder g) = Builder (f . g) +{-# INLINE append' #-} + +{-# RULES + +"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int) ws. + append (writeAtMost a f) (append (writeAtMost b g) ws) = + append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s)) ws + +"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int). + append (writeAtMost a f) (writeAtMost b g) = + writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s) + +"ensureFree/ensureFree" forall a b . + append (ensureFree a) (ensureFree b) = ensureFree (max a b) + +"flush/flush" + append flush flush = flush + + #-} diff --git a/src/Data/Text/Internal/Builder/Int/Digits.hs b/src/Data/Text/Internal/Builder/Int/Digits.hs --- a/src/Data/Text/Internal/Builder/Int/Digits.hs +++ b/src/Data/Text/Internal/Builder/Int/Digits.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE OverloadedStrings #-} - -- Module: Data.Text.Internal.Builder.Int.Digits -- Copyright: (c) 2013 Bryan O'Sullivan -- License: BSD-style @@ -16,11 +14,12 @@ module Data.Text.Internal.Builder.Int.Digits (digits) where -import Data.ByteString.Char8 (ByteString) +import Data.ByteString.Char8 (ByteString, pack) digits :: ByteString -digits = "0001020304050607080910111213141516171819\ - \2021222324252627282930313233343536373839\ - \4041424344454647484950515253545556575859\ - \6061626364656667686970717273747576777879\ - \8081828384858687888990919293949596979899" +digits = pack + "0001020304050607080910111213141516171819\ + \2021222324252627282930313233343536373839\ + \4041424344454647484950515253545556575859\ + \6061626364656667686970717273747576777879\ + \8081828384858687888990919293949596979899" diff --git a/src/Data/Text/Internal/Encoding.hs b/src/Data/Text/Internal/Encoding.hs --- a/src/Data/Text/Internal/Encoding.hs +++ b/src/Data/Text/Internal/Encoding.hs @@ -26,7 +26,8 @@ , decodeUtf8With2 , Utf8State , startUtf8State - , StrictBuilder() + , StrictTextBuilder() + , StrictBuilder , strictBuilderToText , textToStrictBuilder @@ -50,7 +51,7 @@ import Data.Text.Internal (Text(..)) import Data.Text.Internal.Encoding.Utf8 (DecoderState, utf8AcceptState, utf8RejectState, updateDecoderState) -import Data.Text.Internal.StrictBuilder (StrictBuilder) +import Data.Text.Internal.StrictBuilder (StrictBuilder, StrictTextBuilder) import qualified Data.ByteString as B import qualified Data.ByteString.Internal as BI import qualified Data.ByteString.Short.Internal as SBS @@ -72,13 +73,13 @@ -- | Use 'StrictBuilder' to build 'Text'. -- -- @since 2.0.2 -strictBuilderToText :: StrictBuilder -> Text +strictBuilderToText :: StrictTextBuilder -> Text strictBuilderToText = SB.toText -- | Copy 'Text' in a 'StrictBuilder' -- -- @since 2.0.2 -textToStrictBuilder :: Text -> StrictBuilder +textToStrictBuilder :: Text -> StrictTextBuilder textToStrictBuilder = SB.fromText -- | State of decoding a 'ByteString' in UTF-8. @@ -352,11 +353,11 @@ | otherwise = k (- partUtf8Len part) (Just (Utf8State s (partUtf8UnsafeAppend part bs))) -- Eta-expanded to inline partUtf8Foldr -partUtf8ToStrictBuilder :: PartialUtf8CodePoint -> StrictBuilder +partUtf8ToStrictBuilder :: PartialUtf8CodePoint -> StrictTextBuilder partUtf8ToStrictBuilder c = partUtf8Foldr ((<>) . SB.unsafeFromWord8) mempty c -utf8StateToStrictBuilder :: Utf8State -> StrictBuilder +utf8StateToStrictBuilder :: Utf8State -> StrictTextBuilder utf8StateToStrictBuilder = partUtf8ToStrictBuilder . partialUtf8CodePoint -- | Decode another chunk in an ongoing UTF-8 stream. @@ -413,7 +414,7 @@ -- s2b (pre1 '<>' pre2) = s2b pre3 -- ms2 = ms3 -- @ -decodeUtf8More :: Utf8State -> ByteString -> (StrictBuilder, ByteString, Maybe Utf8State) +decodeUtf8More :: Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State) decodeUtf8More s bs = validateUtf8MoreCont s bs $ \len ms -> let builder | len <= 0 = mempty @@ -444,7 +445,7 @@ -- @ -- 'Data.Text.Encoding.encodeUtf8' ('Data.Text.Encoding.strictBuilderToText' builder) '<>' rest = chunk -- @ -decodeUtf8Chunk :: ByteString -> (StrictBuilder, ByteString, Maybe Utf8State) +decodeUtf8Chunk :: ByteString -> (StrictTextBuilder, ByteString, Maybe Utf8State) decodeUtf8Chunk = decodeUtf8More startUtf8State -- | Call the error handler on each byte of the partial code point stored in @@ -454,14 +455,14 @@ -- -- @since 2.0.2 {-# INLINE skipIncomplete #-} -skipIncomplete :: OnDecodeError -> String -> Utf8State -> StrictBuilder +skipIncomplete :: OnDecodeError -> String -> Utf8State -> StrictTextBuilder skipIncomplete onErr msg s = partUtf8Foldr ((<>) . handleUtf8Error onErr msg) mempty (partialUtf8CodePoint s) {-# INLINE handleUtf8Error #-} -handleUtf8Error :: OnDecodeError -> String -> Word8 -> StrictBuilder +handleUtf8Error :: OnDecodeError -> String -> Word8 -> StrictTextBuilder handleUtf8Error onErr msg w = case onErr msg (Just w) of Just c -> SB.fromChar c Nothing -> mempty @@ -505,7 +506,7 @@ #if defined(ASSERTS) HasCallStack => #endif - OnDecodeError -> String -> Utf8State -> ByteString -> (StrictBuilder, ByteString, Utf8State) + OnDecodeError -> String -> Utf8State -> ByteString -> (StrictTextBuilder, ByteString, Utf8State) decodeUtf8With2 onErr msg s0 bs = loop s0 0 mempty where loop s i !builder = diff --git a/src/Data/Text/Internal/Encoding/Utf8.hs b/src/Data/Text/Internal/Encoding/Utf8.hs --- a/src/Data/Text/Internal/Encoding/Utf8.hs +++ b/src/Data/Text/Internal/Encoding/Utf8.hs @@ -1,298 +1,318 @@ -{-# LANGUAGE CPP, MagicHash, BangPatterns #-} - --- | --- Module : Data.Text.Internal.Encoding.Utf8 --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- (c) 2021 Andrew Lelechenko --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Basic UTF-8 validation and character manipulation. -module Data.Text.Internal.Encoding.Utf8 - ( utf8Length - , utf8LengthByLeader - -- Decomposition - , ord2 - , ord3 - , ord4 - -- Construction - , chr2 - , chr3 - , chr4 - -- * Validation - , validate1 - , validate2 - , validate3 - , validate4 - -- * Naive decoding - , DecoderState(..) - , utf8AcceptState - , utf8RejectState - , updateDecoderState - , DecoderResult(..) - , CodePoint(..) - , utf8DecodeStart - , utf8DecodeContinue - ) where - -#if defined(ASSERTS) -import Control.Exception (assert) -import GHC.Stack (HasCallStack) -#endif -import Data.Bits (Bits(..), FiniteBits(..)) -import Data.Char (ord, chr) -import GHC.Exts -import GHC.Word (Word8(..)) - -#if !MIN_VERSION_base(4,16,0) --- harmless to import, except for warnings that it is unused. -import Data.Text.Internal.PrimCompat (word8ToWord#) -#endif - -default(Int) - -between :: Word8 -- ^ byte to check - -> Word8 -- ^ lower bound - -> Word8 -- ^ upper bound - -> Bool -between x y z = x >= y && x <= z -{-# INLINE between #-} - --- This is a branchless version of --- utf8Length c --- | ord c < 0x80 = 1 --- | ord c < 0x800 = 2 --- | ord c < 0x10000 = 3 --- | otherwise = 4 --- Implementation suggested by Alex Mason. - --- | @since 2.0 -utf8Length :: Char -> Int -utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) -{-# INLINE utf8Length #-} - --- This is a branchless version of --- utf8LengthByLeader w --- | w < 0x80 = 1 --- | w < 0xE0 = 2 --- | w < 0xF0 = 3 --- | otherwise = 4 --- --- c `xor` I# (c# <=# 0#) is a branchless equivalent of c `max` 1. --- It is crucial to write c# <=# 0# and not c# ==# 0#, otherwise --- GHC is tempted to "optimize" by introduction of branches. - --- | @since 2.0 -utf8LengthByLeader :: Word8 -> Int -utf8LengthByLeader w = c `xor` I# (c# <=# 0#) - where - !c@(I# c#) = countLeadingZeros (complement w) -{-# INLINE utf8LengthByLeader #-} - -ord2 :: -#if defined(ASSERTS) - HasCallStack => -#endif - Char -> (Word8,Word8) -ord2 c = -#if defined(ASSERTS) - assert (n >= 0x80 && n <= 0x07ff) -#endif - (x1,x2) - where - n = ord c - x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 - x2 = intToWord8 $ (n .&. 0x3F) + 0x80 -{-# INLINE ord2 #-} - -ord3 :: -#if defined(ASSERTS) - HasCallStack => -#endif - Char -> (Word8,Word8,Word8) -ord3 c = -#if defined(ASSERTS) - assert (n >= 0x0800 && n <= 0xffff) -#endif - (x1,x2,x3) - where - n = ord c - x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 - x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 - x3 = intToWord8 $ (n .&. 0x3F) + 0x80 -{-# INLINE ord3 #-} - -ord4 :: -#if defined(ASSERTS) - HasCallStack => -#endif - Char -> (Word8,Word8,Word8,Word8) -ord4 c = -#if defined(ASSERTS) - assert (n >= 0x10000) -#endif - (x1,x2,x3,x4) - where - n = ord c - x1 = intToWord8 $ (n `shiftR` 18) + 0xF0 - x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 - x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 - x4 = intToWord8 $ (n .&. 0x3F) + 0x80 -{-# INLINE ord4 #-} - -chr2 :: Word8 -> Word8 -> Char -chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) - where - !y1# = word2Int# (word8ToWord# x1#) - !y2# = word2Int# (word8ToWord# x2#) - !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# - !z2# = y2# -# 0x80# -{-# INLINE chr2 #-} - -chr3 :: Word8 -> Word8 -> Word8 -> Char -chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) - where - !y1# = word2Int# (word8ToWord# x1#) - !y2# = word2Int# (word8ToWord# x2#) - !y3# = word2Int# (word8ToWord# x3#) - !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# - !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# - !z3# = y3# -# 0x80# -{-# INLINE chr3 #-} - -chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char -chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = - C# (chr# (z1# +# z2# +# z3# +# z4#)) - where - !y1# = word2Int# (word8ToWord# x1#) - !y2# = word2Int# (word8ToWord# x2#) - !y3# = word2Int# (word8ToWord# x3#) - !y4# = word2Int# (word8ToWord# x4#) - !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# - !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# - !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# - !z4# = y4# -# 0x80# -{-# INLINE chr4 #-} - -validate1 :: Word8 -> Bool -validate1 x1 = x1 <= 0x7F -{-# INLINE validate1 #-} - -validate2 :: Word8 -> Word8 -> Bool -validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF -{-# INLINE validate2 #-} - -validate3 :: Word8 -> Word8 -> Word8 -> Bool -{-# INLINE validate3 #-} -validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 - where - validate3_1 = (x1 == 0xE0) && - between x2 0xA0 0xBF && - between x3 0x80 0xBF - validate3_2 = between x1 0xE1 0xEC && - between x2 0x80 0xBF && - between x3 0x80 0xBF - validate3_3 = x1 == 0xED && - between x2 0x80 0x9F && - between x3 0x80 0xBF - validate3_4 = between x1 0xEE 0xEF && - between x2 0x80 0xBF && - between x3 0x80 0xBF - -validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool -{-# INLINE validate4 #-} -validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 - where - validate4_1 = x1 == 0xF0 && - between x2 0x90 0xBF && - between x3 0x80 0xBF && - between x4 0x80 0xBF - validate4_2 = between x1 0xF1 0xF3 && - between x2 0x80 0xBF && - between x3 0x80 0xBF && - between x4 0x80 0xBF - validate4_3 = x1 == 0xF4 && - between x2 0x80 0x8F && - between x3 0x80 0xBF && - between x4 0x80 0xBF - -intToWord8 :: Int -> Word8 -intToWord8 = fromIntegral - -word8ToInt :: Word8 -> Int -word8ToInt = fromIntegral - -------------------------------------------------------------------------------- --- Naive UTF8 decoder. --- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. - -newtype ByteClass = ByteClass Word8 - -byteToClass :: Word8 -> ByteClass -byteToClass n = ByteClass (W8# el#) - where - !(I# n#) = word8ToInt n - el# = indexWord8OffAddr# table# n# - - table# :: Addr# - table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# - -newtype DecoderState = DecoderState Word8 - deriving (Eq, Show) - -utf8AcceptState :: DecoderState -utf8AcceptState = DecoderState 0 - -utf8RejectState :: DecoderState -utf8RejectState = DecoderState 12 - -updateState :: ByteClass -> DecoderState -> DecoderState -updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) - where - !(I# n#) = word8ToInt (c + s) - el# = indexWord8OffAddr# table# n# - - table# :: Addr# - table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# - -updateDecoderState :: Word8 -> DecoderState -> DecoderState -updateDecoderState b s = updateState (byteToClass b) s - -newtype CodePoint = CodePoint Int - --- | @since 2.0 -data DecoderResult - = Accept !Char - | Incomplete !DecoderState !CodePoint - | Reject - --- | @since 2.0 -utf8DecodeStart :: Word8 -> DecoderResult -utf8DecodeStart !w - | st == utf8AcceptState = Accept (chr (word8ToInt w)) - | st == utf8RejectState = Reject - | otherwise = Incomplete st (CodePoint cp) - where - cl@(ByteClass cl') = byteToClass w - st = updateState cl utf8AcceptState - cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w - --- | @since 2.0 -utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult -utf8DecodeContinue !w !st (CodePoint !cp) - | st' == utf8AcceptState = Accept (chr cp') - | st' == utf8RejectState = Reject - | otherwise = Incomplete st' (CodePoint cp') - where - cl = byteToClass w - st' = updateState cl st - cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f) +{-# LANGUAGE CPP, MagicHash, BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Utf8 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-8 validation and character manipulation. +module Data.Text.Internal.Encoding.Utf8 + ( utf8Length + , utf8LengthByLeader + -- Decomposition + , ord2 + , ord3 + , ord4 + -- Construction + , chr2 + , chr3 + , chr4 + -- * Validation + , validate1 + , validate2 + , validate3 + , validate4 + -- * Naive decoding + , DecoderState(..) + , utf8AcceptState + , utf8RejectState + , updateDecoderState + , DecoderResult(..) + , CodePoint(..) + , utf8DecodeStart + , utf8DecodeContinue + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Bits (Bits(..)) +import Data.Char (ord, chr) +import GHC.Exts +import GHC.Word (Word8(..)) + +#if !MIN_VERSION_base(4,16,0) +-- harmless to import, except for warnings that it is unused. +import Data.Text.Internal.PrimCompat (word8ToWord#) +#endif + +default(Int) + +between :: Word8 -- ^ byte to check + -> Word8 -- ^ lower bound + -> Word8 -- ^ upper bound + -> Bool +between x y z = x >= y && x <= z +{-# INLINE between #-} + +-- This is a branchless version of +-- utf8Length c +-- | ord c < 0x80 = 1 +-- | ord c < 0x800 = 2 +-- | ord c < 0x10000 = 3 +-- | otherwise = 4 +-- Implementation suggested by Alex Mason. + +-- | Measure byte length of UTF-8 encoding for a given character. +-- +-- @since 2.0 +utf8Length :: Char -> Int +utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) +{-# INLINE utf8Length #-} + +-- | Measure byte length of UTF-8 encoding for characters, +-- starting with a given byte. +-- +-- @since 2.0 +utf8LengthByLeader :: Word8 -> Int +utf8LengthByLeader w + | w < 0x80 = 1 + | w < 0xE0 = 2 + | w < 0xF0 = 3 + | otherwise = 4 +{-# INLINE utf8LengthByLeader #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 2 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8) +ord2 c = +#if defined(ASSERTS) + assert (n >= 0x80 && n <= 0x07ff) +#endif + (x1,x2) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 + x2 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord2 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 3 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord3 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8) +ord3 c = +#if defined(ASSERTS) + assert (n >= 0x0800 && n <= 0xffff) +#endif + (x1,x2,x3) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 + x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x3 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord3 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 4 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord4 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8,Word8) +ord4 c = +#if defined(ASSERTS) + assert (n >= 0x10000) +#endif + (x1,x2,x3,x4) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 18) + 0xF0 + x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 + x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x4 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord4 #-} + +-- | @since 1.1.0.0 +chr2 :: Word8 -> Word8 -> Char +chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# + !z2# = y2# -# 0x80# +{-# INLINE chr2 #-} + +-- | @since 1.1.0.0 +chr3 :: Word8 -> Word8 -> Word8 -> Char +chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# + !z3# = y3# -# 0x80# +{-# INLINE chr3 #-} + +-- | @since 1.1.0.0 +chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char +chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = + C# (chr# (z1# +# z2# +# z3# +# z4#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !y4# = word2Int# (word8ToWord# x4#) + !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# + !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# + !z4# = y4# -# 0x80# +{-# INLINE chr4 #-} + +-- | @since 1.1.0.0 +validate1 :: Word8 -> Bool +validate1 x1 = x1 <= 0x7F +{-# INLINE validate1 #-} + +-- | @since 1.1.0.0 +validate2 :: Word8 -> Word8 -> Bool +validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF +{-# INLINE validate2 #-} + +-- | @since 1.1.0.0 +validate3 :: Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate3 #-} +validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 + where + validate3_1 = (x1 == 0xE0) && + between x2 0xA0 0xBF && + between x3 0x80 0xBF + validate3_2 = between x1 0xE1 0xEC && + between x2 0x80 0xBF && + between x3 0x80 0xBF + validate3_3 = x1 == 0xED && + between x2 0x80 0x9F && + between x3 0x80 0xBF + validate3_4 = between x1 0xEE 0xEF && + between x2 0x80 0xBF && + between x3 0x80 0xBF + +-- | @since 1.1.0.0 +validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate4 #-} +validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 + where + validate4_1 = x1 == 0xF0 && + between x2 0x90 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_2 = between x1 0xF1 0xF3 && + between x2 0x80 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_3 = x1 == 0xF4 && + between x2 0x80 0x8F && + between x3 0x80 0xBF && + between x4 0x80 0xBF + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +------------------------------------------------------------------------------- +-- Naive UTF8 decoder. +-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. + +newtype ByteClass = ByteClass Word8 + +byteToClass :: Word8 -> ByteClass +byteToClass n = ByteClass (W8# el#) + where + !(I# n#) = word8ToInt n + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# + +-- | @since 2.0 +newtype DecoderState = DecoderState Word8 + deriving (Eq, Show) + +-- | @since 2.0.2 +utf8AcceptState :: DecoderState +utf8AcceptState = DecoderState 0 + +-- | @since 2.0.2 +utf8RejectState :: DecoderState +utf8RejectState = DecoderState 12 + +updateState :: ByteClass -> DecoderState -> DecoderState +updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) + where + !(I# n#) = word8ToInt (c + s) + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# + +-- | @since 2.0.2 +updateDecoderState :: Word8 -> DecoderState -> DecoderState +updateDecoderState b s = updateState (byteToClass b) s + +-- | @since 2.0 +newtype CodePoint = CodePoint Int + +-- | @since 2.0 +data DecoderResult + = Accept !Char + | Incomplete !DecoderState !CodePoint + | Reject + +-- | @since 2.0 +utf8DecodeStart :: Word8 -> DecoderResult +utf8DecodeStart !w + | st == utf8AcceptState = Accept (chr (word8ToInt w)) + | st == utf8RejectState = Reject + | otherwise = Incomplete st (CodePoint cp) + where + cl@(ByteClass cl') = byteToClass w + st = updateState cl utf8AcceptState + cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w + +-- | @since 2.0 +utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult +utf8DecodeContinue !w !st (CodePoint !cp) + | st' == utf8AcceptState = Accept (chr cp') + | st' == utf8RejectState = Reject + | otherwise = Incomplete st' (CodePoint cp') + where + cl = byteToClass w + st' = updateState cl st + cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f) diff --git a/src/Data/Text/Internal/Fusion.hs b/src/Data/Text/Internal/Fusion.hs --- a/src/Data/Text/Internal/Fusion.hs +++ b/src/Data/Text/Internal/Fusion.hs @@ -25,6 +25,7 @@ -- * Creation and elimination , stream + , streamLn , unstream , reverseStream @@ -49,8 +50,8 @@ , countChar ) where -import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int, - Num(..), Ord(..), ($), +import Prelude (Bool(..), Char, Eq(..), Maybe(..), Monad(..), Int, + Num(..), Ord(..), ($), (&&), otherwise) import Data.Bits (shiftL, shiftR) import Data.Text.Internal (Text(..)) @@ -78,12 +79,33 @@ HasCallStack => #endif Text -> Stream Char -stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) len) +stream = stream' False +{-# INLINE [0] stream #-} + +-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@ +-- +-- @since 2.1.2 +streamLn :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +streamLn = stream' True + +-- | Shared implementation of 'stream' and 'streamLn'. +stream' :: +#if defined(ASSERTS) + HasCallStack => +#endif + Bool -> Text -> Stream Char +stream' addNl (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) maxLen) where + maxLen = if addNl then len + 1 else len !end = off+len next !i - | i >= end = Done - | otherwise = Yield chr (i + l) + | i < end = Yield chr (i + l) + | addNl && i == end = Yield '\n' (i + 1) + | otherwise = Done where n0 = A.unsafeIndex arr i n1 = A.unsafeIndex arr (i + 1) @@ -96,7 +118,7 @@ 2 -> U8.chr2 n0 n1 3 -> U8.chr3 n0 n1 n2 _ -> U8.chr4 n0 n1 n2 n3 -{-# INLINE [0] stream #-} +{-# INLINE [0] stream' #-} -- | /O(n)/ Converts 'Text' into a 'Stream' 'Char', but iterates -- backwards through the text. diff --git a/src/Data/Text/Internal/Fusion/CaseMapping.hs b/src/Data/Text/Internal/Fusion/CaseMapping.hs --- a/src/Data/Text/Internal/Fusion/CaseMapping.hs +++ b/src/Data/Text/Internal/Fusion/CaseMapping.hs @@ -1,7652 +1,7984 @@ --- AUTOMATICALLY GENERATED - DO NOT EDIT --- Generated by scripts/CaseMapping.hs --- CaseFolding-14.0.0.txt --- Date: 2021-03-08, 19:35:41 GMT --- SpecialCasing-14.0.0.txt --- Date: 2021-03-08, 19:35:55 GMT - -{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} -{-# OPTIONS_GHC -Wno-partial-type-signatures #-} -module Data.Text.Internal.Fusion.CaseMapping where -import GHC.Int -import GHC.Exts -unI64 :: Int64 -> _ {- unboxed Int64 -} -unI64 (I64# n) = n - -upperMapping :: Char# -> _ {- unboxed Int64 -} -{-# NOINLINE upperMapping #-} -upperMapping = \case - -- LATIN SMALL LETTER SHARP S - '\x00df'# -> unI64 174063699 - -- LATIN SMALL LIGATURE FF - '\xfb00'# -> unI64 146800710 - -- LATIN SMALL LIGATURE FI - '\xfb01'# -> unI64 153092166 - -- LATIN SMALL LIGATURE FL - '\xfb02'# -> unI64 159383622 - -- LATIN SMALL LIGATURE FFI - '\xfb03'# -> unI64 321057542111302 - -- LATIN SMALL LIGATURE FFL - '\xfb04'# -> unI64 334251681644614 - -- LATIN SMALL LIGATURE LONG S T - '\xfb05'# -> unI64 176160851 - -- LATIN SMALL LIGATURE ST - '\xfb06'# -> unI64 176160851 - -- ARMENIAN SMALL LIGATURE ECH YIWN - '\x0587'# -> unI64 2856322357 - -- ARMENIAN SMALL LIGATURE MEN NOW - '\xfb13'# -> unI64 2831156548 - -- ARMENIAN SMALL LIGATURE MEN ECH - '\xfb14'# -> unI64 2795504964 - -- ARMENIAN SMALL LIGATURE MEN INI - '\xfb15'# -> unI64 2808087876 - -- ARMENIAN SMALL LIGATURE VEW NOW - '\xfb16'# -> unI64 2831156558 - -- ARMENIAN SMALL LIGATURE MEN XEH - '\xfb17'# -> unI64 2812282180 - -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE - '\x0149'# -> unI64 163578556 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS - '\x0390'# -> unI64 3382099394429849 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS - '\x03b0'# -> unI64 3382099394429861 - -- LATIN SMALL LETTER J WITH CARON - '\x01f0'# -> unI64 1635778634 - -- LATIN SMALL LETTER H WITH LINE BELOW - '\x1e96'# -> unI64 1713373256 - -- LATIN SMALL LETTER T WITH DIAERESIS - '\x1e97'# -> unI64 1627390036 - -- LATIN SMALL LETTER W WITH RING ABOVE - '\x1e98'# -> unI64 1631584343 - -- LATIN SMALL LETTER Y WITH RING ABOVE - '\x1e99'# -> unI64 1631584345 - -- LATIN SMALL LETTER A WITH RIGHT HALF RING - '\x1e9a'# -> unI64 1472200769 - -- GREEK SMALL LETTER UPSILON WITH PSILI - '\x1f50'# -> unI64 1650459557 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA - '\x1f52'# -> unI64 3377701370987429 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA - '\x1f54'# -> unI64 3382099417498533 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI - '\x1f56'# -> unI64 3667972440720293 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI - '\x1fb6'# -> unI64 1749025681 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI - '\x1fc6'# -> unI64 1749025687 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA - '\x1fd2'# -> unI64 3377701347918745 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA - '\x1fd3'# -> unI64 3382099394429849 - -- GREEK SMALL LETTER IOTA WITH PERISPOMENI - '\x1fd6'# -> unI64 1749025689 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI - '\x1fd7'# -> unI64 3667972417651609 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA - '\x1fe2'# -> unI64 3377701347918757 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA - '\x1fe3'# -> unI64 3382099394429861 - -- GREEK SMALL LETTER RHO WITH PSILI - '\x1fe4'# -> unI64 1650459553 - -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI - '\x1fe6'# -> unI64 1749025701 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI - '\x1fe7'# -> unI64 3667972417651621 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI - '\x1ff6'# -> unI64 1749025705 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI - '\x1f80'# -> unI64 1931484936 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI - '\x1f81'# -> unI64 1931484937 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1f82'# -> unI64 1931484938 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1f83'# -> unI64 1931484939 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1f84'# -> unI64 1931484940 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1f85'# -> unI64 1931484941 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f86'# -> unI64 1931484942 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f87'# -> unI64 1931484943 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI - '\x1f88'# -> unI64 1931484936 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI - '\x1f89'# -> unI64 1931484937 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1f8a'# -> unI64 1931484938 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1f8b'# -> unI64 1931484939 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1f8c'# -> unI64 1931484940 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1f8d'# -> unI64 1931484941 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f8e'# -> unI64 1931484942 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f8f'# -> unI64 1931484943 - -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI - '\x1f90'# -> unI64 1931484968 - -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI - '\x1f91'# -> unI64 1931484969 - -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1f92'# -> unI64 1931484970 - -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1f93'# -> unI64 1931484971 - -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1f94'# -> unI64 1931484972 - -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1f95'# -> unI64 1931484973 - -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f96'# -> unI64 1931484974 - -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f97'# -> unI64 1931484975 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI - '\x1f98'# -> unI64 1931484968 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI - '\x1f99'# -> unI64 1931484969 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1f9a'# -> unI64 1931484970 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1f9b'# -> unI64 1931484971 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1f9c'# -> unI64 1931484972 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1f9d'# -> unI64 1931484973 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f9e'# -> unI64 1931484974 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f9f'# -> unI64 1931484975 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI - '\x1fa0'# -> unI64 1931485032 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI - '\x1fa1'# -> unI64 1931485033 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1fa2'# -> unI64 1931485034 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1fa3'# -> unI64 1931485035 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1fa4'# -> unI64 1931485036 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1fa5'# -> unI64 1931485037 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1fa6'# -> unI64 1931485038 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1fa7'# -> unI64 1931485039 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI - '\x1fa8'# -> unI64 1931485032 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI - '\x1fa9'# -> unI64 1931485033 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1faa'# -> unI64 1931485034 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1fab'# -> unI64 1931485035 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1fac'# -> unI64 1931485036 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1fad'# -> unI64 1931485037 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1fae'# -> unI64 1931485038 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1faf'# -> unI64 1931485039 - -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI - '\x1fb3'# -> unI64 1931477905 - -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI - '\x1fbc'# -> unI64 1931477905 - -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI - '\x1fc3'# -> unI64 1931477911 - -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI - '\x1fcc'# -> unI64 1931477911 - -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI - '\x1ff3'# -> unI64 1931477929 - -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI - '\x1ffc'# -> unI64 1931477929 - -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI - '\x1fb2'# -> unI64 1931485114 - -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI - '\x1fb4'# -> unI64 1931477894 - -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI - '\x1fc2'# -> unI64 1931485130 - -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI - '\x1fc4'# -> unI64 1931477897 - -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI - '\x1ff2'# -> unI64 1931485178 - -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI - '\x1ff4'# -> unI64 1931477903 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fb7'# -> unI64 4050602585752465 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fc7'# -> unI64 4050602585752471 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1ff7'# -> unI64 4050602585752489 - '\x0061'# -> unI64 65 - '\x0062'# -> unI64 66 - '\x0063'# -> unI64 67 - '\x0064'# -> unI64 68 - '\x0065'# -> unI64 69 - '\x0066'# -> unI64 70 - '\x0067'# -> unI64 71 - '\x0068'# -> unI64 72 - '\x0069'# -> unI64 73 - '\x006a'# -> unI64 74 - '\x006b'# -> unI64 75 - '\x006c'# -> unI64 76 - '\x006d'# -> unI64 77 - '\x006e'# -> unI64 78 - '\x006f'# -> unI64 79 - '\x0070'# -> unI64 80 - '\x0071'# -> unI64 81 - '\x0072'# -> unI64 82 - '\x0073'# -> unI64 83 - '\x0074'# -> unI64 84 - '\x0075'# -> unI64 85 - '\x0076'# -> unI64 86 - '\x0077'# -> unI64 87 - '\x0078'# -> unI64 88 - '\x0079'# -> unI64 89 - '\x007a'# -> unI64 90 - '\x00b5'# -> unI64 924 - '\x00e0'# -> unI64 192 - '\x00e1'# -> unI64 193 - '\x00e2'# -> unI64 194 - '\x00e3'# -> unI64 195 - '\x00e4'# -> unI64 196 - '\x00e5'# -> unI64 197 - '\x00e6'# -> unI64 198 - '\x00e7'# -> unI64 199 - '\x00e8'# -> unI64 200 - '\x00e9'# -> unI64 201 - '\x00ea'# -> unI64 202 - '\x00eb'# -> unI64 203 - '\x00ec'# -> unI64 204 - '\x00ed'# -> unI64 205 - '\x00ee'# -> unI64 206 - '\x00ef'# -> unI64 207 - '\x00f0'# -> unI64 208 - '\x00f1'# -> unI64 209 - '\x00f2'# -> unI64 210 - '\x00f3'# -> unI64 211 - '\x00f4'# -> unI64 212 - '\x00f5'# -> unI64 213 - '\x00f6'# -> unI64 214 - '\x00f8'# -> unI64 216 - '\x00f9'# -> unI64 217 - '\x00fa'# -> unI64 218 - '\x00fb'# -> unI64 219 - '\x00fc'# -> unI64 220 - '\x00fd'# -> unI64 221 - '\x00fe'# -> unI64 222 - '\x00ff'# -> unI64 376 - '\x0101'# -> unI64 256 - '\x0103'# -> unI64 258 - '\x0105'# -> unI64 260 - '\x0107'# -> unI64 262 - '\x0109'# -> unI64 264 - '\x010b'# -> unI64 266 - '\x010d'# -> unI64 268 - '\x010f'# -> unI64 270 - '\x0111'# -> unI64 272 - '\x0113'# -> unI64 274 - '\x0115'# -> unI64 276 - '\x0117'# -> unI64 278 - '\x0119'# -> unI64 280 - '\x011b'# -> unI64 282 - '\x011d'# -> unI64 284 - '\x011f'# -> unI64 286 - '\x0121'# -> unI64 288 - '\x0123'# -> unI64 290 - '\x0125'# -> unI64 292 - '\x0127'# -> unI64 294 - '\x0129'# -> unI64 296 - '\x012b'# -> unI64 298 - '\x012d'# -> unI64 300 - '\x012f'# -> unI64 302 - '\x0131'# -> unI64 73 - '\x0133'# -> unI64 306 - '\x0135'# -> unI64 308 - '\x0137'# -> unI64 310 - '\x013a'# -> unI64 313 - '\x013c'# -> unI64 315 - '\x013e'# -> unI64 317 - '\x0140'# -> unI64 319 - '\x0142'# -> unI64 321 - '\x0144'# -> unI64 323 - '\x0146'# -> unI64 325 - '\x0148'# -> unI64 327 - '\x014b'# -> unI64 330 - '\x014d'# -> unI64 332 - '\x014f'# -> unI64 334 - '\x0151'# -> unI64 336 - '\x0153'# -> unI64 338 - '\x0155'# -> unI64 340 - '\x0157'# -> unI64 342 - '\x0159'# -> unI64 344 - '\x015b'# -> unI64 346 - '\x015d'# -> unI64 348 - '\x015f'# -> unI64 350 - '\x0161'# -> unI64 352 - '\x0163'# -> unI64 354 - '\x0165'# -> unI64 356 - '\x0167'# -> unI64 358 - '\x0169'# -> unI64 360 - '\x016b'# -> unI64 362 - '\x016d'# -> unI64 364 - '\x016f'# -> unI64 366 - '\x0171'# -> unI64 368 - '\x0173'# -> unI64 370 - '\x0175'# -> unI64 372 - '\x0177'# -> unI64 374 - '\x017a'# -> unI64 377 - '\x017c'# -> unI64 379 - '\x017e'# -> unI64 381 - '\x017f'# -> unI64 83 - '\x0180'# -> unI64 579 - '\x0183'# -> unI64 386 - '\x0185'# -> unI64 388 - '\x0188'# -> unI64 391 - '\x018c'# -> unI64 395 - '\x0192'# -> unI64 401 - '\x0195'# -> unI64 502 - '\x0199'# -> unI64 408 - '\x019a'# -> unI64 573 - '\x019e'# -> unI64 544 - '\x01a1'# -> unI64 416 - '\x01a3'# -> unI64 418 - '\x01a5'# -> unI64 420 - '\x01a8'# -> unI64 423 - '\x01ad'# -> unI64 428 - '\x01b0'# -> unI64 431 - '\x01b4'# -> unI64 435 - '\x01b6'# -> unI64 437 - '\x01b9'# -> unI64 440 - '\x01bd'# -> unI64 444 - '\x01bf'# -> unI64 503 - '\x01c5'# -> unI64 452 - '\x01c6'# -> unI64 452 - '\x01c8'# -> unI64 455 - '\x01c9'# -> unI64 455 - '\x01cb'# -> unI64 458 - '\x01cc'# -> unI64 458 - '\x01ce'# -> unI64 461 - '\x01d0'# -> unI64 463 - '\x01d2'# -> unI64 465 - '\x01d4'# -> unI64 467 - '\x01d6'# -> unI64 469 - '\x01d8'# -> unI64 471 - '\x01da'# -> unI64 473 - '\x01dc'# -> unI64 475 - '\x01dd'# -> unI64 398 - '\x01df'# -> unI64 478 - '\x01e1'# -> unI64 480 - '\x01e3'# -> unI64 482 - '\x01e5'# -> unI64 484 - '\x01e7'# -> unI64 486 - '\x01e9'# -> unI64 488 - '\x01eb'# -> unI64 490 - '\x01ed'# -> unI64 492 - '\x01ef'# -> unI64 494 - '\x01f2'# -> unI64 497 - '\x01f3'# -> unI64 497 - '\x01f5'# -> unI64 500 - '\x01f9'# -> unI64 504 - '\x01fb'# -> unI64 506 - '\x01fd'# -> unI64 508 - '\x01ff'# -> unI64 510 - '\x0201'# -> unI64 512 - '\x0203'# -> unI64 514 - '\x0205'# -> unI64 516 - '\x0207'# -> unI64 518 - '\x0209'# -> unI64 520 - '\x020b'# -> unI64 522 - '\x020d'# -> unI64 524 - '\x020f'# -> unI64 526 - '\x0211'# -> unI64 528 - '\x0213'# -> unI64 530 - '\x0215'# -> unI64 532 - '\x0217'# -> unI64 534 - '\x0219'# -> unI64 536 - '\x021b'# -> unI64 538 - '\x021d'# -> unI64 540 - '\x021f'# -> unI64 542 - '\x0223'# -> unI64 546 - '\x0225'# -> unI64 548 - '\x0227'# -> unI64 550 - '\x0229'# -> unI64 552 - '\x022b'# -> unI64 554 - '\x022d'# -> unI64 556 - '\x022f'# -> unI64 558 - '\x0231'# -> unI64 560 - '\x0233'# -> unI64 562 - '\x023c'# -> unI64 571 - '\x023f'# -> unI64 11390 - '\x0240'# -> unI64 11391 - '\x0242'# -> unI64 577 - '\x0247'# -> unI64 582 - '\x0249'# -> unI64 584 - '\x024b'# -> unI64 586 - '\x024d'# -> unI64 588 - '\x024f'# -> unI64 590 - '\x0250'# -> unI64 11375 - '\x0251'# -> unI64 11373 - '\x0252'# -> unI64 11376 - '\x0253'# -> unI64 385 - '\x0254'# -> unI64 390 - '\x0256'# -> unI64 393 - '\x0257'# -> unI64 394 - '\x0259'# -> unI64 399 - '\x025b'# -> unI64 400 - '\x025c'# -> unI64 42923 - '\x0260'# -> unI64 403 - '\x0261'# -> unI64 42924 - '\x0263'# -> unI64 404 - '\x0265'# -> unI64 42893 - '\x0266'# -> unI64 42922 - '\x0268'# -> unI64 407 - '\x0269'# -> unI64 406 - '\x026a'# -> unI64 42926 - '\x026b'# -> unI64 11362 - '\x026c'# -> unI64 42925 - '\x026f'# -> unI64 412 - '\x0271'# -> unI64 11374 - '\x0272'# -> unI64 413 - '\x0275'# -> unI64 415 - '\x027d'# -> unI64 11364 - '\x0280'# -> unI64 422 - '\x0282'# -> unI64 42949 - '\x0283'# -> unI64 425 - '\x0287'# -> unI64 42929 - '\x0288'# -> unI64 430 - '\x0289'# -> unI64 580 - '\x028a'# -> unI64 433 - '\x028b'# -> unI64 434 - '\x028c'# -> unI64 581 - '\x0292'# -> unI64 439 - '\x029d'# -> unI64 42930 - '\x029e'# -> unI64 42928 - '\x0345'# -> unI64 921 - '\x0371'# -> unI64 880 - '\x0373'# -> unI64 882 - '\x0377'# -> unI64 886 - '\x037b'# -> unI64 1021 - '\x037c'# -> unI64 1022 - '\x037d'# -> unI64 1023 - '\x03ac'# -> unI64 902 - '\x03ad'# -> unI64 904 - '\x03ae'# -> unI64 905 - '\x03af'# -> unI64 906 - '\x03b1'# -> unI64 913 - '\x03b2'# -> unI64 914 - '\x03b3'# -> unI64 915 - '\x03b4'# -> unI64 916 - '\x03b5'# -> unI64 917 - '\x03b6'# -> unI64 918 - '\x03b7'# -> unI64 919 - '\x03b8'# -> unI64 920 - '\x03b9'# -> unI64 921 - '\x03ba'# -> unI64 922 - '\x03bb'# -> unI64 923 - '\x03bc'# -> unI64 924 - '\x03bd'# -> unI64 925 - '\x03be'# -> unI64 926 - '\x03bf'# -> unI64 927 - '\x03c0'# -> unI64 928 - '\x03c1'# -> unI64 929 - '\x03c2'# -> unI64 931 - '\x03c3'# -> unI64 931 - '\x03c4'# -> unI64 932 - '\x03c5'# -> unI64 933 - '\x03c6'# -> unI64 934 - '\x03c7'# -> unI64 935 - '\x03c8'# -> unI64 936 - '\x03c9'# -> unI64 937 - '\x03ca'# -> unI64 938 - '\x03cb'# -> unI64 939 - '\x03cc'# -> unI64 908 - '\x03cd'# -> unI64 910 - '\x03ce'# -> unI64 911 - '\x03d0'# -> unI64 914 - '\x03d1'# -> unI64 920 - '\x03d5'# -> unI64 934 - '\x03d6'# -> unI64 928 - '\x03d7'# -> unI64 975 - '\x03d9'# -> unI64 984 - '\x03db'# -> unI64 986 - '\x03dd'# -> unI64 988 - '\x03df'# -> unI64 990 - '\x03e1'# -> unI64 992 - '\x03e3'# -> unI64 994 - '\x03e5'# -> unI64 996 - '\x03e7'# -> unI64 998 - '\x03e9'# -> unI64 1000 - '\x03eb'# -> unI64 1002 - '\x03ed'# -> unI64 1004 - '\x03ef'# -> unI64 1006 - '\x03f0'# -> unI64 922 - '\x03f1'# -> unI64 929 - '\x03f2'# -> unI64 1017 - '\x03f3'# -> unI64 895 - '\x03f5'# -> unI64 917 - '\x03f8'# -> unI64 1015 - '\x03fb'# -> unI64 1018 - '\x0430'# -> unI64 1040 - '\x0431'# -> unI64 1041 - '\x0432'# -> unI64 1042 - '\x0433'# -> unI64 1043 - '\x0434'# -> unI64 1044 - '\x0435'# -> unI64 1045 - '\x0436'# -> unI64 1046 - '\x0437'# -> unI64 1047 - '\x0438'# -> unI64 1048 - '\x0439'# -> unI64 1049 - '\x043a'# -> unI64 1050 - '\x043b'# -> unI64 1051 - '\x043c'# -> unI64 1052 - '\x043d'# -> unI64 1053 - '\x043e'# -> unI64 1054 - '\x043f'# -> unI64 1055 - '\x0440'# -> unI64 1056 - '\x0441'# -> unI64 1057 - '\x0442'# -> unI64 1058 - '\x0443'# -> unI64 1059 - '\x0444'# -> unI64 1060 - '\x0445'# -> unI64 1061 - '\x0446'# -> unI64 1062 - '\x0447'# -> unI64 1063 - '\x0448'# -> unI64 1064 - '\x0449'# -> unI64 1065 - '\x044a'# -> unI64 1066 - '\x044b'# -> unI64 1067 - '\x044c'# -> unI64 1068 - '\x044d'# -> unI64 1069 - '\x044e'# -> unI64 1070 - '\x044f'# -> unI64 1071 - '\x0450'# -> unI64 1024 - '\x0451'# -> unI64 1025 - '\x0452'# -> unI64 1026 - '\x0453'# -> unI64 1027 - '\x0454'# -> unI64 1028 - '\x0455'# -> unI64 1029 - '\x0456'# -> unI64 1030 - '\x0457'# -> unI64 1031 - '\x0458'# -> unI64 1032 - '\x0459'# -> unI64 1033 - '\x045a'# -> unI64 1034 - '\x045b'# -> unI64 1035 - '\x045c'# -> unI64 1036 - '\x045d'# -> unI64 1037 - '\x045e'# -> unI64 1038 - '\x045f'# -> unI64 1039 - '\x0461'# -> unI64 1120 - '\x0463'# -> unI64 1122 - '\x0465'# -> unI64 1124 - '\x0467'# -> unI64 1126 - '\x0469'# -> unI64 1128 - '\x046b'# -> unI64 1130 - '\x046d'# -> unI64 1132 - '\x046f'# -> unI64 1134 - '\x0471'# -> unI64 1136 - '\x0473'# -> unI64 1138 - '\x0475'# -> unI64 1140 - '\x0477'# -> unI64 1142 - '\x0479'# -> unI64 1144 - '\x047b'# -> unI64 1146 - '\x047d'# -> unI64 1148 - '\x047f'# -> unI64 1150 - '\x0481'# -> unI64 1152 - '\x048b'# -> unI64 1162 - '\x048d'# -> unI64 1164 - '\x048f'# -> unI64 1166 - '\x0491'# -> unI64 1168 - '\x0493'# -> unI64 1170 - '\x0495'# -> unI64 1172 - '\x0497'# -> unI64 1174 - '\x0499'# -> unI64 1176 - '\x049b'# -> unI64 1178 - '\x049d'# -> unI64 1180 - '\x049f'# -> unI64 1182 - '\x04a1'# -> unI64 1184 - '\x04a3'# -> unI64 1186 - '\x04a5'# -> unI64 1188 - '\x04a7'# -> unI64 1190 - '\x04a9'# -> unI64 1192 - '\x04ab'# -> unI64 1194 - '\x04ad'# -> unI64 1196 - '\x04af'# -> unI64 1198 - '\x04b1'# -> unI64 1200 - '\x04b3'# -> unI64 1202 - '\x04b5'# -> unI64 1204 - '\x04b7'# -> unI64 1206 - '\x04b9'# -> unI64 1208 - '\x04bb'# -> unI64 1210 - '\x04bd'# -> unI64 1212 - '\x04bf'# -> unI64 1214 - '\x04c2'# -> unI64 1217 - '\x04c4'# -> unI64 1219 - '\x04c6'# -> unI64 1221 - '\x04c8'# -> unI64 1223 - '\x04ca'# -> unI64 1225 - '\x04cc'# -> unI64 1227 - '\x04ce'# -> unI64 1229 - '\x04cf'# -> unI64 1216 - '\x04d1'# -> unI64 1232 - '\x04d3'# -> unI64 1234 - '\x04d5'# -> unI64 1236 - '\x04d7'# -> unI64 1238 - '\x04d9'# -> unI64 1240 - '\x04db'# -> unI64 1242 - '\x04dd'# -> unI64 1244 - '\x04df'# -> unI64 1246 - '\x04e1'# -> unI64 1248 - '\x04e3'# -> unI64 1250 - '\x04e5'# -> unI64 1252 - '\x04e7'# -> unI64 1254 - '\x04e9'# -> unI64 1256 - '\x04eb'# -> unI64 1258 - '\x04ed'# -> unI64 1260 - '\x04ef'# -> unI64 1262 - '\x04f1'# -> unI64 1264 - '\x04f3'# -> unI64 1266 - '\x04f5'# -> unI64 1268 - '\x04f7'# -> unI64 1270 - '\x04f9'# -> unI64 1272 - '\x04fb'# -> unI64 1274 - '\x04fd'# -> unI64 1276 - '\x04ff'# -> unI64 1278 - '\x0501'# -> unI64 1280 - '\x0503'# -> unI64 1282 - '\x0505'# -> unI64 1284 - '\x0507'# -> unI64 1286 - '\x0509'# -> unI64 1288 - '\x050b'# -> unI64 1290 - '\x050d'# -> unI64 1292 - '\x050f'# -> unI64 1294 - '\x0511'# -> unI64 1296 - '\x0513'# -> unI64 1298 - '\x0515'# -> unI64 1300 - '\x0517'# -> unI64 1302 - '\x0519'# -> unI64 1304 - '\x051b'# -> unI64 1306 - '\x051d'# -> unI64 1308 - '\x051f'# -> unI64 1310 - '\x0521'# -> unI64 1312 - '\x0523'# -> unI64 1314 - '\x0525'# -> unI64 1316 - '\x0527'# -> unI64 1318 - '\x0529'# -> unI64 1320 - '\x052b'# -> unI64 1322 - '\x052d'# -> unI64 1324 - '\x052f'# -> unI64 1326 - '\x0561'# -> unI64 1329 - '\x0562'# -> unI64 1330 - '\x0563'# -> unI64 1331 - '\x0564'# -> unI64 1332 - '\x0565'# -> unI64 1333 - '\x0566'# -> unI64 1334 - '\x0567'# -> unI64 1335 - '\x0568'# -> unI64 1336 - '\x0569'# -> unI64 1337 - '\x056a'# -> unI64 1338 - '\x056b'# -> unI64 1339 - '\x056c'# -> unI64 1340 - '\x056d'# -> unI64 1341 - '\x056e'# -> unI64 1342 - '\x056f'# -> unI64 1343 - '\x0570'# -> unI64 1344 - '\x0571'# -> unI64 1345 - '\x0572'# -> unI64 1346 - '\x0573'# -> unI64 1347 - '\x0574'# -> unI64 1348 - '\x0575'# -> unI64 1349 - '\x0576'# -> unI64 1350 - '\x0577'# -> unI64 1351 - '\x0578'# -> unI64 1352 - '\x0579'# -> unI64 1353 - '\x057a'# -> unI64 1354 - '\x057b'# -> unI64 1355 - '\x057c'# -> unI64 1356 - '\x057d'# -> unI64 1357 - '\x057e'# -> unI64 1358 - '\x057f'# -> unI64 1359 - '\x0580'# -> unI64 1360 - '\x0581'# -> unI64 1361 - '\x0582'# -> unI64 1362 - '\x0583'# -> unI64 1363 - '\x0584'# -> unI64 1364 - '\x0585'# -> unI64 1365 - '\x0586'# -> unI64 1366 - '\x10d0'# -> unI64 7312 - '\x10d1'# -> unI64 7313 - '\x10d2'# -> unI64 7314 - '\x10d3'# -> unI64 7315 - '\x10d4'# -> unI64 7316 - '\x10d5'# -> unI64 7317 - '\x10d6'# -> unI64 7318 - '\x10d7'# -> unI64 7319 - '\x10d8'# -> unI64 7320 - '\x10d9'# -> unI64 7321 - '\x10da'# -> unI64 7322 - '\x10db'# -> unI64 7323 - '\x10dc'# -> unI64 7324 - '\x10dd'# -> unI64 7325 - '\x10de'# -> unI64 7326 - '\x10df'# -> unI64 7327 - '\x10e0'# -> unI64 7328 - '\x10e1'# -> unI64 7329 - '\x10e2'# -> unI64 7330 - '\x10e3'# -> unI64 7331 - '\x10e4'# -> unI64 7332 - '\x10e5'# -> unI64 7333 - '\x10e6'# -> unI64 7334 - '\x10e7'# -> unI64 7335 - '\x10e8'# -> unI64 7336 - '\x10e9'# -> unI64 7337 - '\x10ea'# -> unI64 7338 - '\x10eb'# -> unI64 7339 - '\x10ec'# -> unI64 7340 - '\x10ed'# -> unI64 7341 - '\x10ee'# -> unI64 7342 - '\x10ef'# -> unI64 7343 - '\x10f0'# -> unI64 7344 - '\x10f1'# -> unI64 7345 - '\x10f2'# -> unI64 7346 - '\x10f3'# -> unI64 7347 - '\x10f4'# -> unI64 7348 - '\x10f5'# -> unI64 7349 - '\x10f6'# -> unI64 7350 - '\x10f7'# -> unI64 7351 - '\x10f8'# -> unI64 7352 - '\x10f9'# -> unI64 7353 - '\x10fa'# -> unI64 7354 - '\x10fd'# -> unI64 7357 - '\x10fe'# -> unI64 7358 - '\x10ff'# -> unI64 7359 - '\x13f8'# -> unI64 5104 - '\x13f9'# -> unI64 5105 - '\x13fa'# -> unI64 5106 - '\x13fb'# -> unI64 5107 - '\x13fc'# -> unI64 5108 - '\x13fd'# -> unI64 5109 - '\x1c80'# -> unI64 1042 - '\x1c81'# -> unI64 1044 - '\x1c82'# -> unI64 1054 - '\x1c83'# -> unI64 1057 - '\x1c84'# -> unI64 1058 - '\x1c85'# -> unI64 1058 - '\x1c86'# -> unI64 1066 - '\x1c87'# -> unI64 1122 - '\x1c88'# -> unI64 42570 - '\x1d79'# -> unI64 42877 - '\x1d7d'# -> unI64 11363 - '\x1d8e'# -> unI64 42950 - '\x1e01'# -> unI64 7680 - '\x1e03'# -> unI64 7682 - '\x1e05'# -> unI64 7684 - '\x1e07'# -> unI64 7686 - '\x1e09'# -> unI64 7688 - '\x1e0b'# -> unI64 7690 - '\x1e0d'# -> unI64 7692 - '\x1e0f'# -> unI64 7694 - '\x1e11'# -> unI64 7696 - '\x1e13'# -> unI64 7698 - '\x1e15'# -> unI64 7700 - '\x1e17'# -> unI64 7702 - '\x1e19'# -> unI64 7704 - '\x1e1b'# -> unI64 7706 - '\x1e1d'# -> unI64 7708 - '\x1e1f'# -> unI64 7710 - '\x1e21'# -> unI64 7712 - '\x1e23'# -> unI64 7714 - '\x1e25'# -> unI64 7716 - '\x1e27'# -> unI64 7718 - '\x1e29'# -> unI64 7720 - '\x1e2b'# -> unI64 7722 - '\x1e2d'# -> unI64 7724 - '\x1e2f'# -> unI64 7726 - '\x1e31'# -> unI64 7728 - '\x1e33'# -> unI64 7730 - '\x1e35'# -> unI64 7732 - '\x1e37'# -> unI64 7734 - '\x1e39'# -> unI64 7736 - '\x1e3b'# -> unI64 7738 - '\x1e3d'# -> unI64 7740 - '\x1e3f'# -> unI64 7742 - '\x1e41'# -> unI64 7744 - '\x1e43'# -> unI64 7746 - '\x1e45'# -> unI64 7748 - '\x1e47'# -> unI64 7750 - '\x1e49'# -> unI64 7752 - '\x1e4b'# -> unI64 7754 - '\x1e4d'# -> unI64 7756 - '\x1e4f'# -> unI64 7758 - '\x1e51'# -> unI64 7760 - '\x1e53'# -> unI64 7762 - '\x1e55'# -> unI64 7764 - '\x1e57'# -> unI64 7766 - '\x1e59'# -> unI64 7768 - '\x1e5b'# -> unI64 7770 - '\x1e5d'# -> unI64 7772 - '\x1e5f'# -> unI64 7774 - '\x1e61'# -> unI64 7776 - '\x1e63'# -> unI64 7778 - '\x1e65'# -> unI64 7780 - '\x1e67'# -> unI64 7782 - '\x1e69'# -> unI64 7784 - '\x1e6b'# -> unI64 7786 - '\x1e6d'# -> unI64 7788 - '\x1e6f'# -> unI64 7790 - '\x1e71'# -> unI64 7792 - '\x1e73'# -> unI64 7794 - '\x1e75'# -> unI64 7796 - '\x1e77'# -> unI64 7798 - '\x1e79'# -> unI64 7800 - '\x1e7b'# -> unI64 7802 - '\x1e7d'# -> unI64 7804 - '\x1e7f'# -> unI64 7806 - '\x1e81'# -> unI64 7808 - '\x1e83'# -> unI64 7810 - '\x1e85'# -> unI64 7812 - '\x1e87'# -> unI64 7814 - '\x1e89'# -> unI64 7816 - '\x1e8b'# -> unI64 7818 - '\x1e8d'# -> unI64 7820 - '\x1e8f'# -> unI64 7822 - '\x1e91'# -> unI64 7824 - '\x1e93'# -> unI64 7826 - '\x1e95'# -> unI64 7828 - '\x1e9b'# -> unI64 7776 - '\x1ea1'# -> unI64 7840 - '\x1ea3'# -> unI64 7842 - '\x1ea5'# -> unI64 7844 - '\x1ea7'# -> unI64 7846 - '\x1ea9'# -> unI64 7848 - '\x1eab'# -> unI64 7850 - '\x1ead'# -> unI64 7852 - '\x1eaf'# -> unI64 7854 - '\x1eb1'# -> unI64 7856 - '\x1eb3'# -> unI64 7858 - '\x1eb5'# -> unI64 7860 - '\x1eb7'# -> unI64 7862 - '\x1eb9'# -> unI64 7864 - '\x1ebb'# -> unI64 7866 - '\x1ebd'# -> unI64 7868 - '\x1ebf'# -> unI64 7870 - '\x1ec1'# -> unI64 7872 - '\x1ec3'# -> unI64 7874 - '\x1ec5'# -> unI64 7876 - '\x1ec7'# -> unI64 7878 - '\x1ec9'# -> unI64 7880 - '\x1ecb'# -> unI64 7882 - '\x1ecd'# -> unI64 7884 - '\x1ecf'# -> unI64 7886 - '\x1ed1'# -> unI64 7888 - '\x1ed3'# -> unI64 7890 - '\x1ed5'# -> unI64 7892 - '\x1ed7'# -> unI64 7894 - '\x1ed9'# -> unI64 7896 - '\x1edb'# -> unI64 7898 - '\x1edd'# -> unI64 7900 - '\x1edf'# -> unI64 7902 - '\x1ee1'# -> unI64 7904 - '\x1ee3'# -> unI64 7906 - '\x1ee5'# -> unI64 7908 - '\x1ee7'# -> unI64 7910 - '\x1ee9'# -> unI64 7912 - '\x1eeb'# -> unI64 7914 - '\x1eed'# -> unI64 7916 - '\x1eef'# -> unI64 7918 - '\x1ef1'# -> unI64 7920 - '\x1ef3'# -> unI64 7922 - '\x1ef5'# -> unI64 7924 - '\x1ef7'# -> unI64 7926 - '\x1ef9'# -> unI64 7928 - '\x1efb'# -> unI64 7930 - '\x1efd'# -> unI64 7932 - '\x1eff'# -> unI64 7934 - '\x1f00'# -> unI64 7944 - '\x1f01'# -> unI64 7945 - '\x1f02'# -> unI64 7946 - '\x1f03'# -> unI64 7947 - '\x1f04'# -> unI64 7948 - '\x1f05'# -> unI64 7949 - '\x1f06'# -> unI64 7950 - '\x1f07'# -> unI64 7951 - '\x1f10'# -> unI64 7960 - '\x1f11'# -> unI64 7961 - '\x1f12'# -> unI64 7962 - '\x1f13'# -> unI64 7963 - '\x1f14'# -> unI64 7964 - '\x1f15'# -> unI64 7965 - '\x1f20'# -> unI64 7976 - '\x1f21'# -> unI64 7977 - '\x1f22'# -> unI64 7978 - '\x1f23'# -> unI64 7979 - '\x1f24'# -> unI64 7980 - '\x1f25'# -> unI64 7981 - '\x1f26'# -> unI64 7982 - '\x1f27'# -> unI64 7983 - '\x1f30'# -> unI64 7992 - '\x1f31'# -> unI64 7993 - '\x1f32'# -> unI64 7994 - '\x1f33'# -> unI64 7995 - '\x1f34'# -> unI64 7996 - '\x1f35'# -> unI64 7997 - '\x1f36'# -> unI64 7998 - '\x1f37'# -> unI64 7999 - '\x1f40'# -> unI64 8008 - '\x1f41'# -> unI64 8009 - '\x1f42'# -> unI64 8010 - '\x1f43'# -> unI64 8011 - '\x1f44'# -> unI64 8012 - '\x1f45'# -> unI64 8013 - '\x1f51'# -> unI64 8025 - '\x1f53'# -> unI64 8027 - '\x1f55'# -> unI64 8029 - '\x1f57'# -> unI64 8031 - '\x1f60'# -> unI64 8040 - '\x1f61'# -> unI64 8041 - '\x1f62'# -> unI64 8042 - '\x1f63'# -> unI64 8043 - '\x1f64'# -> unI64 8044 - '\x1f65'# -> unI64 8045 - '\x1f66'# -> unI64 8046 - '\x1f67'# -> unI64 8047 - '\x1f70'# -> unI64 8122 - '\x1f71'# -> unI64 8123 - '\x1f72'# -> unI64 8136 - '\x1f73'# -> unI64 8137 - '\x1f74'# -> unI64 8138 - '\x1f75'# -> unI64 8139 - '\x1f76'# -> unI64 8154 - '\x1f77'# -> unI64 8155 - '\x1f78'# -> unI64 8184 - '\x1f79'# -> unI64 8185 - '\x1f7a'# -> unI64 8170 - '\x1f7b'# -> unI64 8171 - '\x1f7c'# -> unI64 8186 - '\x1f7d'# -> unI64 8187 - '\x1fb0'# -> unI64 8120 - '\x1fb1'# -> unI64 8121 - '\x1fbe'# -> unI64 921 - '\x1fd0'# -> unI64 8152 - '\x1fd1'# -> unI64 8153 - '\x1fe0'# -> unI64 8168 - '\x1fe1'# -> unI64 8169 - '\x1fe5'# -> unI64 8172 - '\x214e'# -> unI64 8498 - '\x2170'# -> unI64 8544 - '\x2171'# -> unI64 8545 - '\x2172'# -> unI64 8546 - '\x2173'# -> unI64 8547 - '\x2174'# -> unI64 8548 - '\x2175'# -> unI64 8549 - '\x2176'# -> unI64 8550 - '\x2177'# -> unI64 8551 - '\x2178'# -> unI64 8552 - '\x2179'# -> unI64 8553 - '\x217a'# -> unI64 8554 - '\x217b'# -> unI64 8555 - '\x217c'# -> unI64 8556 - '\x217d'# -> unI64 8557 - '\x217e'# -> unI64 8558 - '\x217f'# -> unI64 8559 - '\x2184'# -> unI64 8579 - '\x24d0'# -> unI64 9398 - '\x24d1'# -> unI64 9399 - '\x24d2'# -> unI64 9400 - '\x24d3'# -> unI64 9401 - '\x24d4'# -> unI64 9402 - '\x24d5'# -> unI64 9403 - '\x24d6'# -> unI64 9404 - '\x24d7'# -> unI64 9405 - '\x24d8'# -> unI64 9406 - '\x24d9'# -> unI64 9407 - '\x24da'# -> unI64 9408 - '\x24db'# -> unI64 9409 - '\x24dc'# -> unI64 9410 - '\x24dd'# -> unI64 9411 - '\x24de'# -> unI64 9412 - '\x24df'# -> unI64 9413 - '\x24e0'# -> unI64 9414 - '\x24e1'# -> unI64 9415 - '\x24e2'# -> unI64 9416 - '\x24e3'# -> unI64 9417 - '\x24e4'# -> unI64 9418 - '\x24e5'# -> unI64 9419 - '\x24e6'# -> unI64 9420 - '\x24e7'# -> unI64 9421 - '\x24e8'# -> unI64 9422 - '\x24e9'# -> unI64 9423 - '\x2c30'# -> unI64 11264 - '\x2c31'# -> unI64 11265 - '\x2c32'# -> unI64 11266 - '\x2c33'# -> unI64 11267 - '\x2c34'# -> unI64 11268 - '\x2c35'# -> unI64 11269 - '\x2c36'# -> unI64 11270 - '\x2c37'# -> unI64 11271 - '\x2c38'# -> unI64 11272 - '\x2c39'# -> unI64 11273 - '\x2c3a'# -> unI64 11274 - '\x2c3b'# -> unI64 11275 - '\x2c3c'# -> unI64 11276 - '\x2c3d'# -> unI64 11277 - '\x2c3e'# -> unI64 11278 - '\x2c3f'# -> unI64 11279 - '\x2c40'# -> unI64 11280 - '\x2c41'# -> unI64 11281 - '\x2c42'# -> unI64 11282 - '\x2c43'# -> unI64 11283 - '\x2c44'# -> unI64 11284 - '\x2c45'# -> unI64 11285 - '\x2c46'# -> unI64 11286 - '\x2c47'# -> unI64 11287 - '\x2c48'# -> unI64 11288 - '\x2c49'# -> unI64 11289 - '\x2c4a'# -> unI64 11290 - '\x2c4b'# -> unI64 11291 - '\x2c4c'# -> unI64 11292 - '\x2c4d'# -> unI64 11293 - '\x2c4e'# -> unI64 11294 - '\x2c4f'# -> unI64 11295 - '\x2c50'# -> unI64 11296 - '\x2c51'# -> unI64 11297 - '\x2c52'# -> unI64 11298 - '\x2c53'# -> unI64 11299 - '\x2c54'# -> unI64 11300 - '\x2c55'# -> unI64 11301 - '\x2c56'# -> unI64 11302 - '\x2c57'# -> unI64 11303 - '\x2c58'# -> unI64 11304 - '\x2c59'# -> unI64 11305 - '\x2c5a'# -> unI64 11306 - '\x2c5b'# -> unI64 11307 - '\x2c5c'# -> unI64 11308 - '\x2c5d'# -> unI64 11309 - '\x2c5e'# -> unI64 11310 - '\x2c5f'# -> unI64 11311 - '\x2c61'# -> unI64 11360 - '\x2c65'# -> unI64 570 - '\x2c66'# -> unI64 574 - '\x2c68'# -> unI64 11367 - '\x2c6a'# -> unI64 11369 - '\x2c6c'# -> unI64 11371 - '\x2c73'# -> unI64 11378 - '\x2c76'# -> unI64 11381 - '\x2c81'# -> unI64 11392 - '\x2c83'# -> unI64 11394 - '\x2c85'# -> unI64 11396 - '\x2c87'# -> unI64 11398 - '\x2c89'# -> unI64 11400 - '\x2c8b'# -> unI64 11402 - '\x2c8d'# -> unI64 11404 - '\x2c8f'# -> unI64 11406 - '\x2c91'# -> unI64 11408 - '\x2c93'# -> unI64 11410 - '\x2c95'# -> unI64 11412 - '\x2c97'# -> unI64 11414 - '\x2c99'# -> unI64 11416 - '\x2c9b'# -> unI64 11418 - '\x2c9d'# -> unI64 11420 - '\x2c9f'# -> unI64 11422 - '\x2ca1'# -> unI64 11424 - '\x2ca3'# -> unI64 11426 - '\x2ca5'# -> unI64 11428 - '\x2ca7'# -> unI64 11430 - '\x2ca9'# -> unI64 11432 - '\x2cab'# -> unI64 11434 - '\x2cad'# -> unI64 11436 - '\x2caf'# -> unI64 11438 - '\x2cb1'# -> unI64 11440 - '\x2cb3'# -> unI64 11442 - '\x2cb5'# -> unI64 11444 - '\x2cb7'# -> unI64 11446 - '\x2cb9'# -> unI64 11448 - '\x2cbb'# -> unI64 11450 - '\x2cbd'# -> unI64 11452 - '\x2cbf'# -> unI64 11454 - '\x2cc1'# -> unI64 11456 - '\x2cc3'# -> unI64 11458 - '\x2cc5'# -> unI64 11460 - '\x2cc7'# -> unI64 11462 - '\x2cc9'# -> unI64 11464 - '\x2ccb'# -> unI64 11466 - '\x2ccd'# -> unI64 11468 - '\x2ccf'# -> unI64 11470 - '\x2cd1'# -> unI64 11472 - '\x2cd3'# -> unI64 11474 - '\x2cd5'# -> unI64 11476 - '\x2cd7'# -> unI64 11478 - '\x2cd9'# -> unI64 11480 - '\x2cdb'# -> unI64 11482 - '\x2cdd'# -> unI64 11484 - '\x2cdf'# -> unI64 11486 - '\x2ce1'# -> unI64 11488 - '\x2ce3'# -> unI64 11490 - '\x2cec'# -> unI64 11499 - '\x2cee'# -> unI64 11501 - '\x2cf3'# -> unI64 11506 - '\x2d00'# -> unI64 4256 - '\x2d01'# -> unI64 4257 - '\x2d02'# -> unI64 4258 - '\x2d03'# -> unI64 4259 - '\x2d04'# -> unI64 4260 - '\x2d05'# -> unI64 4261 - '\x2d06'# -> unI64 4262 - '\x2d07'# -> unI64 4263 - '\x2d08'# -> unI64 4264 - '\x2d09'# -> unI64 4265 - '\x2d0a'# -> unI64 4266 - '\x2d0b'# -> unI64 4267 - '\x2d0c'# -> unI64 4268 - '\x2d0d'# -> unI64 4269 - '\x2d0e'# -> unI64 4270 - '\x2d0f'# -> unI64 4271 - '\x2d10'# -> unI64 4272 - '\x2d11'# -> unI64 4273 - '\x2d12'# -> unI64 4274 - '\x2d13'# -> unI64 4275 - '\x2d14'# -> unI64 4276 - '\x2d15'# -> unI64 4277 - '\x2d16'# -> unI64 4278 - '\x2d17'# -> unI64 4279 - '\x2d18'# -> unI64 4280 - '\x2d19'# -> unI64 4281 - '\x2d1a'# -> unI64 4282 - '\x2d1b'# -> unI64 4283 - '\x2d1c'# -> unI64 4284 - '\x2d1d'# -> unI64 4285 - '\x2d1e'# -> unI64 4286 - '\x2d1f'# -> unI64 4287 - '\x2d20'# -> unI64 4288 - '\x2d21'# -> unI64 4289 - '\x2d22'# -> unI64 4290 - '\x2d23'# -> unI64 4291 - '\x2d24'# -> unI64 4292 - '\x2d25'# -> unI64 4293 - '\x2d27'# -> unI64 4295 - '\x2d2d'# -> unI64 4301 - '\xa641'# -> unI64 42560 - '\xa643'# -> unI64 42562 - '\xa645'# -> unI64 42564 - '\xa647'# -> unI64 42566 - '\xa649'# -> unI64 42568 - '\xa64b'# -> unI64 42570 - '\xa64d'# -> unI64 42572 - '\xa64f'# -> unI64 42574 - '\xa651'# -> unI64 42576 - '\xa653'# -> unI64 42578 - '\xa655'# -> unI64 42580 - '\xa657'# -> unI64 42582 - '\xa659'# -> unI64 42584 - '\xa65b'# -> unI64 42586 - '\xa65d'# -> unI64 42588 - '\xa65f'# -> unI64 42590 - '\xa661'# -> unI64 42592 - '\xa663'# -> unI64 42594 - '\xa665'# -> unI64 42596 - '\xa667'# -> unI64 42598 - '\xa669'# -> unI64 42600 - '\xa66b'# -> unI64 42602 - '\xa66d'# -> unI64 42604 - '\xa681'# -> unI64 42624 - '\xa683'# -> unI64 42626 - '\xa685'# -> unI64 42628 - '\xa687'# -> unI64 42630 - '\xa689'# -> unI64 42632 - '\xa68b'# -> unI64 42634 - '\xa68d'# -> unI64 42636 - '\xa68f'# -> unI64 42638 - '\xa691'# -> unI64 42640 - '\xa693'# -> unI64 42642 - '\xa695'# -> unI64 42644 - '\xa697'# -> unI64 42646 - '\xa699'# -> unI64 42648 - '\xa69b'# -> unI64 42650 - '\xa723'# -> unI64 42786 - '\xa725'# -> unI64 42788 - '\xa727'# -> unI64 42790 - '\xa729'# -> unI64 42792 - '\xa72b'# -> unI64 42794 - '\xa72d'# -> unI64 42796 - '\xa72f'# -> unI64 42798 - '\xa733'# -> unI64 42802 - '\xa735'# -> unI64 42804 - '\xa737'# -> unI64 42806 - '\xa739'# -> unI64 42808 - '\xa73b'# -> unI64 42810 - '\xa73d'# -> unI64 42812 - '\xa73f'# -> unI64 42814 - '\xa741'# -> unI64 42816 - '\xa743'# -> unI64 42818 - '\xa745'# -> unI64 42820 - '\xa747'# -> unI64 42822 - '\xa749'# -> unI64 42824 - '\xa74b'# -> unI64 42826 - '\xa74d'# -> unI64 42828 - '\xa74f'# -> unI64 42830 - '\xa751'# -> unI64 42832 - '\xa753'# -> unI64 42834 - '\xa755'# -> unI64 42836 - '\xa757'# -> unI64 42838 - '\xa759'# -> unI64 42840 - '\xa75b'# -> unI64 42842 - '\xa75d'# -> unI64 42844 - '\xa75f'# -> unI64 42846 - '\xa761'# -> unI64 42848 - '\xa763'# -> unI64 42850 - '\xa765'# -> unI64 42852 - '\xa767'# -> unI64 42854 - '\xa769'# -> unI64 42856 - '\xa76b'# -> unI64 42858 - '\xa76d'# -> unI64 42860 - '\xa76f'# -> unI64 42862 - '\xa77a'# -> unI64 42873 - '\xa77c'# -> unI64 42875 - '\xa77f'# -> unI64 42878 - '\xa781'# -> unI64 42880 - '\xa783'# -> unI64 42882 - '\xa785'# -> unI64 42884 - '\xa787'# -> unI64 42886 - '\xa78c'# -> unI64 42891 - '\xa791'# -> unI64 42896 - '\xa793'# -> unI64 42898 - '\xa794'# -> unI64 42948 - '\xa797'# -> unI64 42902 - '\xa799'# -> unI64 42904 - '\xa79b'# -> unI64 42906 - '\xa79d'# -> unI64 42908 - '\xa79f'# -> unI64 42910 - '\xa7a1'# -> unI64 42912 - '\xa7a3'# -> unI64 42914 - '\xa7a5'# -> unI64 42916 - '\xa7a7'# -> unI64 42918 - '\xa7a9'# -> unI64 42920 - '\xa7b5'# -> unI64 42932 - '\xa7b7'# -> unI64 42934 - '\xa7b9'# -> unI64 42936 - '\xa7bb'# -> unI64 42938 - '\xa7bd'# -> unI64 42940 - '\xa7bf'# -> unI64 42942 - '\xa7c1'# -> unI64 42944 - '\xa7c3'# -> unI64 42946 - '\xa7c8'# -> unI64 42951 - '\xa7ca'# -> unI64 42953 - '\xa7d1'# -> unI64 42960 - '\xa7d7'# -> unI64 42966 - '\xa7d9'# -> unI64 42968 - '\xa7f6'# -> unI64 42997 - '\xab53'# -> unI64 42931 - '\xab70'# -> unI64 5024 - '\xab71'# -> unI64 5025 - '\xab72'# -> unI64 5026 - '\xab73'# -> unI64 5027 - '\xab74'# -> unI64 5028 - '\xab75'# -> unI64 5029 - '\xab76'# -> unI64 5030 - '\xab77'# -> unI64 5031 - '\xab78'# -> unI64 5032 - '\xab79'# -> unI64 5033 - '\xab7a'# -> unI64 5034 - '\xab7b'# -> unI64 5035 - '\xab7c'# -> unI64 5036 - '\xab7d'# -> unI64 5037 - '\xab7e'# -> unI64 5038 - '\xab7f'# -> unI64 5039 - '\xab80'# -> unI64 5040 - '\xab81'# -> unI64 5041 - '\xab82'# -> unI64 5042 - '\xab83'# -> unI64 5043 - '\xab84'# -> unI64 5044 - '\xab85'# -> unI64 5045 - '\xab86'# -> unI64 5046 - '\xab87'# -> unI64 5047 - '\xab88'# -> unI64 5048 - '\xab89'# -> unI64 5049 - '\xab8a'# -> unI64 5050 - '\xab8b'# -> unI64 5051 - '\xab8c'# -> unI64 5052 - '\xab8d'# -> unI64 5053 - '\xab8e'# -> unI64 5054 - '\xab8f'# -> unI64 5055 - '\xab90'# -> unI64 5056 - '\xab91'# -> unI64 5057 - '\xab92'# -> unI64 5058 - '\xab93'# -> unI64 5059 - '\xab94'# -> unI64 5060 - '\xab95'# -> unI64 5061 - '\xab96'# -> unI64 5062 - '\xab97'# -> unI64 5063 - '\xab98'# -> unI64 5064 - '\xab99'# -> unI64 5065 - '\xab9a'# -> unI64 5066 - '\xab9b'# -> unI64 5067 - '\xab9c'# -> unI64 5068 - '\xab9d'# -> unI64 5069 - '\xab9e'# -> unI64 5070 - '\xab9f'# -> unI64 5071 - '\xaba0'# -> unI64 5072 - '\xaba1'# -> unI64 5073 - '\xaba2'# -> unI64 5074 - '\xaba3'# -> unI64 5075 - '\xaba4'# -> unI64 5076 - '\xaba5'# -> unI64 5077 - '\xaba6'# -> unI64 5078 - '\xaba7'# -> unI64 5079 - '\xaba8'# -> unI64 5080 - '\xaba9'# -> unI64 5081 - '\xabaa'# -> unI64 5082 - '\xabab'# -> unI64 5083 - '\xabac'# -> unI64 5084 - '\xabad'# -> unI64 5085 - '\xabae'# -> unI64 5086 - '\xabaf'# -> unI64 5087 - '\xabb0'# -> unI64 5088 - '\xabb1'# -> unI64 5089 - '\xabb2'# -> unI64 5090 - '\xabb3'# -> unI64 5091 - '\xabb4'# -> unI64 5092 - '\xabb5'# -> unI64 5093 - '\xabb6'# -> unI64 5094 - '\xabb7'# -> unI64 5095 - '\xabb8'# -> unI64 5096 - '\xabb9'# -> unI64 5097 - '\xabba'# -> unI64 5098 - '\xabbb'# -> unI64 5099 - '\xabbc'# -> unI64 5100 - '\xabbd'# -> unI64 5101 - '\xabbe'# -> unI64 5102 - '\xabbf'# -> unI64 5103 - '\xff41'# -> unI64 65313 - '\xff42'# -> unI64 65314 - '\xff43'# -> unI64 65315 - '\xff44'# -> unI64 65316 - '\xff45'# -> unI64 65317 - '\xff46'# -> unI64 65318 - '\xff47'# -> unI64 65319 - '\xff48'# -> unI64 65320 - '\xff49'# -> unI64 65321 - '\xff4a'# -> unI64 65322 - '\xff4b'# -> unI64 65323 - '\xff4c'# -> unI64 65324 - '\xff4d'# -> unI64 65325 - '\xff4e'# -> unI64 65326 - '\xff4f'# -> unI64 65327 - '\xff50'# -> unI64 65328 - '\xff51'# -> unI64 65329 - '\xff52'# -> unI64 65330 - '\xff53'# -> unI64 65331 - '\xff54'# -> unI64 65332 - '\xff55'# -> unI64 65333 - '\xff56'# -> unI64 65334 - '\xff57'# -> unI64 65335 - '\xff58'# -> unI64 65336 - '\xff59'# -> unI64 65337 - '\xff5a'# -> unI64 65338 - '\x10428'# -> unI64 66560 - '\x10429'# -> unI64 66561 - '\x1042a'# -> unI64 66562 - '\x1042b'# -> unI64 66563 - '\x1042c'# -> unI64 66564 - '\x1042d'# -> unI64 66565 - '\x1042e'# -> unI64 66566 - '\x1042f'# -> unI64 66567 - '\x10430'# -> unI64 66568 - '\x10431'# -> unI64 66569 - '\x10432'# -> unI64 66570 - '\x10433'# -> unI64 66571 - '\x10434'# -> unI64 66572 - '\x10435'# -> unI64 66573 - '\x10436'# -> unI64 66574 - '\x10437'# -> unI64 66575 - '\x10438'# -> unI64 66576 - '\x10439'# -> unI64 66577 - '\x1043a'# -> unI64 66578 - '\x1043b'# -> unI64 66579 - '\x1043c'# -> unI64 66580 - '\x1043d'# -> unI64 66581 - '\x1043e'# -> unI64 66582 - '\x1043f'# -> unI64 66583 - '\x10440'# -> unI64 66584 - '\x10441'# -> unI64 66585 - '\x10442'# -> unI64 66586 - '\x10443'# -> unI64 66587 - '\x10444'# -> unI64 66588 - '\x10445'# -> unI64 66589 - '\x10446'# -> unI64 66590 - '\x10447'# -> unI64 66591 - '\x10448'# -> unI64 66592 - '\x10449'# -> unI64 66593 - '\x1044a'# -> unI64 66594 - '\x1044b'# -> unI64 66595 - '\x1044c'# -> unI64 66596 - '\x1044d'# -> unI64 66597 - '\x1044e'# -> unI64 66598 - '\x1044f'# -> unI64 66599 - '\x104d8'# -> unI64 66736 - '\x104d9'# -> unI64 66737 - '\x104da'# -> unI64 66738 - '\x104db'# -> unI64 66739 - '\x104dc'# -> unI64 66740 - '\x104dd'# -> unI64 66741 - '\x104de'# -> unI64 66742 - '\x104df'# -> unI64 66743 - '\x104e0'# -> unI64 66744 - '\x104e1'# -> unI64 66745 - '\x104e2'# -> unI64 66746 - '\x104e3'# -> unI64 66747 - '\x104e4'# -> unI64 66748 - '\x104e5'# -> unI64 66749 - '\x104e6'# -> unI64 66750 - '\x104e7'# -> unI64 66751 - '\x104e8'# -> unI64 66752 - '\x104e9'# -> unI64 66753 - '\x104ea'# -> unI64 66754 - '\x104eb'# -> unI64 66755 - '\x104ec'# -> unI64 66756 - '\x104ed'# -> unI64 66757 - '\x104ee'# -> unI64 66758 - '\x104ef'# -> unI64 66759 - '\x104f0'# -> unI64 66760 - '\x104f1'# -> unI64 66761 - '\x104f2'# -> unI64 66762 - '\x104f3'# -> unI64 66763 - '\x104f4'# -> unI64 66764 - '\x104f5'# -> unI64 66765 - '\x104f6'# -> unI64 66766 - '\x104f7'# -> unI64 66767 - '\x104f8'# -> unI64 66768 - '\x104f9'# -> unI64 66769 - '\x104fa'# -> unI64 66770 - '\x104fb'# -> unI64 66771 - '\x10597'# -> unI64 66928 - '\x10598'# -> unI64 66929 - '\x10599'# -> unI64 66930 - '\x1059a'# -> unI64 66931 - '\x1059b'# -> unI64 66932 - '\x1059c'# -> unI64 66933 - '\x1059d'# -> unI64 66934 - '\x1059e'# -> unI64 66935 - '\x1059f'# -> unI64 66936 - '\x105a0'# -> unI64 66937 - '\x105a1'# -> unI64 66938 - '\x105a3'# -> unI64 66940 - '\x105a4'# -> unI64 66941 - '\x105a5'# -> unI64 66942 - '\x105a6'# -> unI64 66943 - '\x105a7'# -> unI64 66944 - '\x105a8'# -> unI64 66945 - '\x105a9'# -> unI64 66946 - '\x105aa'# -> unI64 66947 - '\x105ab'# -> unI64 66948 - '\x105ac'# -> unI64 66949 - '\x105ad'# -> unI64 66950 - '\x105ae'# -> unI64 66951 - '\x105af'# -> unI64 66952 - '\x105b0'# -> unI64 66953 - '\x105b1'# -> unI64 66954 - '\x105b3'# -> unI64 66956 - '\x105b4'# -> unI64 66957 - '\x105b5'# -> unI64 66958 - '\x105b6'# -> unI64 66959 - '\x105b7'# -> unI64 66960 - '\x105b8'# -> unI64 66961 - '\x105b9'# -> unI64 66962 - '\x105bb'# -> unI64 66964 - '\x105bc'# -> unI64 66965 - '\x10cc0'# -> unI64 68736 - '\x10cc1'# -> unI64 68737 - '\x10cc2'# -> unI64 68738 - '\x10cc3'# -> unI64 68739 - '\x10cc4'# -> unI64 68740 - '\x10cc5'# -> unI64 68741 - '\x10cc6'# -> unI64 68742 - '\x10cc7'# -> unI64 68743 - '\x10cc8'# -> unI64 68744 - '\x10cc9'# -> unI64 68745 - '\x10cca'# -> unI64 68746 - '\x10ccb'# -> unI64 68747 - '\x10ccc'# -> unI64 68748 - '\x10ccd'# -> unI64 68749 - '\x10cce'# -> unI64 68750 - '\x10ccf'# -> unI64 68751 - '\x10cd0'# -> unI64 68752 - '\x10cd1'# -> unI64 68753 - '\x10cd2'# -> unI64 68754 - '\x10cd3'# -> unI64 68755 - '\x10cd4'# -> unI64 68756 - '\x10cd5'# -> unI64 68757 - '\x10cd6'# -> unI64 68758 - '\x10cd7'# -> unI64 68759 - '\x10cd8'# -> unI64 68760 - '\x10cd9'# -> unI64 68761 - '\x10cda'# -> unI64 68762 - '\x10cdb'# -> unI64 68763 - '\x10cdc'# -> unI64 68764 - '\x10cdd'# -> unI64 68765 - '\x10cde'# -> unI64 68766 - '\x10cdf'# -> unI64 68767 - '\x10ce0'# -> unI64 68768 - '\x10ce1'# -> unI64 68769 - '\x10ce2'# -> unI64 68770 - '\x10ce3'# -> unI64 68771 - '\x10ce4'# -> unI64 68772 - '\x10ce5'# -> unI64 68773 - '\x10ce6'# -> unI64 68774 - '\x10ce7'# -> unI64 68775 - '\x10ce8'# -> unI64 68776 - '\x10ce9'# -> unI64 68777 - '\x10cea'# -> unI64 68778 - '\x10ceb'# -> unI64 68779 - '\x10cec'# -> unI64 68780 - '\x10ced'# -> unI64 68781 - '\x10cee'# -> unI64 68782 - '\x10cef'# -> unI64 68783 - '\x10cf0'# -> unI64 68784 - '\x10cf1'# -> unI64 68785 - '\x10cf2'# -> unI64 68786 - '\x118c0'# -> unI64 71840 - '\x118c1'# -> unI64 71841 - '\x118c2'# -> unI64 71842 - '\x118c3'# -> unI64 71843 - '\x118c4'# -> unI64 71844 - '\x118c5'# -> unI64 71845 - '\x118c6'# -> unI64 71846 - '\x118c7'# -> unI64 71847 - '\x118c8'# -> unI64 71848 - '\x118c9'# -> unI64 71849 - '\x118ca'# -> unI64 71850 - '\x118cb'# -> unI64 71851 - '\x118cc'# -> unI64 71852 - '\x118cd'# -> unI64 71853 - '\x118ce'# -> unI64 71854 - '\x118cf'# -> unI64 71855 - '\x118d0'# -> unI64 71856 - '\x118d1'# -> unI64 71857 - '\x118d2'# -> unI64 71858 - '\x118d3'# -> unI64 71859 - '\x118d4'# -> unI64 71860 - '\x118d5'# -> unI64 71861 - '\x118d6'# -> unI64 71862 - '\x118d7'# -> unI64 71863 - '\x118d8'# -> unI64 71864 - '\x118d9'# -> unI64 71865 - '\x118da'# -> unI64 71866 - '\x118db'# -> unI64 71867 - '\x118dc'# -> unI64 71868 - '\x118dd'# -> unI64 71869 - '\x118de'# -> unI64 71870 - '\x118df'# -> unI64 71871 - '\x16e60'# -> unI64 93760 - '\x16e61'# -> unI64 93761 - '\x16e62'# -> unI64 93762 - '\x16e63'# -> unI64 93763 - '\x16e64'# -> unI64 93764 - '\x16e65'# -> unI64 93765 - '\x16e66'# -> unI64 93766 - '\x16e67'# -> unI64 93767 - '\x16e68'# -> unI64 93768 - '\x16e69'# -> unI64 93769 - '\x16e6a'# -> unI64 93770 - '\x16e6b'# -> unI64 93771 - '\x16e6c'# -> unI64 93772 - '\x16e6d'# -> unI64 93773 - '\x16e6e'# -> unI64 93774 - '\x16e6f'# -> unI64 93775 - '\x16e70'# -> unI64 93776 - '\x16e71'# -> unI64 93777 - '\x16e72'# -> unI64 93778 - '\x16e73'# -> unI64 93779 - '\x16e74'# -> unI64 93780 - '\x16e75'# -> unI64 93781 - '\x16e76'# -> unI64 93782 - '\x16e77'# -> unI64 93783 - '\x16e78'# -> unI64 93784 - '\x16e79'# -> unI64 93785 - '\x16e7a'# -> unI64 93786 - '\x16e7b'# -> unI64 93787 - '\x16e7c'# -> unI64 93788 - '\x16e7d'# -> unI64 93789 - '\x16e7e'# -> unI64 93790 - '\x16e7f'# -> unI64 93791 - '\x1e922'# -> unI64 125184 - '\x1e923'# -> unI64 125185 - '\x1e924'# -> unI64 125186 - '\x1e925'# -> unI64 125187 - '\x1e926'# -> unI64 125188 - '\x1e927'# -> unI64 125189 - '\x1e928'# -> unI64 125190 - '\x1e929'# -> unI64 125191 - '\x1e92a'# -> unI64 125192 - '\x1e92b'# -> unI64 125193 - '\x1e92c'# -> unI64 125194 - '\x1e92d'# -> unI64 125195 - '\x1e92e'# -> unI64 125196 - '\x1e92f'# -> unI64 125197 - '\x1e930'# -> unI64 125198 - '\x1e931'# -> unI64 125199 - '\x1e932'# -> unI64 125200 - '\x1e933'# -> unI64 125201 - '\x1e934'# -> unI64 125202 - '\x1e935'# -> unI64 125203 - '\x1e936'# -> unI64 125204 - '\x1e937'# -> unI64 125205 - '\x1e938'# -> unI64 125206 - '\x1e939'# -> unI64 125207 - '\x1e93a'# -> unI64 125208 - '\x1e93b'# -> unI64 125209 - '\x1e93c'# -> unI64 125210 - '\x1e93d'# -> unI64 125211 - '\x1e93e'# -> unI64 125212 - '\x1e93f'# -> unI64 125213 - '\x1e940'# -> unI64 125214 - '\x1e941'# -> unI64 125215 - '\x1e942'# -> unI64 125216 - '\x1e943'# -> unI64 125217 - _ -> unI64 0 -lowerMapping :: Char# -> _ {- unboxed Int64 -} -{-# NOINLINE lowerMapping #-} -lowerMapping = \case - -- LATIN CAPITAL LETTER I WITH DOT ABOVE - '\x0130'# -> unI64 1625292905 - '\x0041'# -> unI64 97 - '\x0042'# -> unI64 98 - '\x0043'# -> unI64 99 - '\x0044'# -> unI64 100 - '\x0045'# -> unI64 101 - '\x0046'# -> unI64 102 - '\x0047'# -> unI64 103 - '\x0048'# -> unI64 104 - '\x0049'# -> unI64 105 - '\x004a'# -> unI64 106 - '\x004b'# -> unI64 107 - '\x004c'# -> unI64 108 - '\x004d'# -> unI64 109 - '\x004e'# -> unI64 110 - '\x004f'# -> unI64 111 - '\x0050'# -> unI64 112 - '\x0051'# -> unI64 113 - '\x0052'# -> unI64 114 - '\x0053'# -> unI64 115 - '\x0054'# -> unI64 116 - '\x0055'# -> unI64 117 - '\x0056'# -> unI64 118 - '\x0057'# -> unI64 119 - '\x0058'# -> unI64 120 - '\x0059'# -> unI64 121 - '\x005a'# -> unI64 122 - '\x00c0'# -> unI64 224 - '\x00c1'# -> unI64 225 - '\x00c2'# -> unI64 226 - '\x00c3'# -> unI64 227 - '\x00c4'# -> unI64 228 - '\x00c5'# -> unI64 229 - '\x00c6'# -> unI64 230 - '\x00c7'# -> unI64 231 - '\x00c8'# -> unI64 232 - '\x00c9'# -> unI64 233 - '\x00ca'# -> unI64 234 - '\x00cb'# -> unI64 235 - '\x00cc'# -> unI64 236 - '\x00cd'# -> unI64 237 - '\x00ce'# -> unI64 238 - '\x00cf'# -> unI64 239 - '\x00d0'# -> unI64 240 - '\x00d1'# -> unI64 241 - '\x00d2'# -> unI64 242 - '\x00d3'# -> unI64 243 - '\x00d4'# -> unI64 244 - '\x00d5'# -> unI64 245 - '\x00d6'# -> unI64 246 - '\x00d8'# -> unI64 248 - '\x00d9'# -> unI64 249 - '\x00da'# -> unI64 250 - '\x00db'# -> unI64 251 - '\x00dc'# -> unI64 252 - '\x00dd'# -> unI64 253 - '\x00de'# -> unI64 254 - '\x0100'# -> unI64 257 - '\x0102'# -> unI64 259 - '\x0104'# -> unI64 261 - '\x0106'# -> unI64 263 - '\x0108'# -> unI64 265 - '\x010a'# -> unI64 267 - '\x010c'# -> unI64 269 - '\x010e'# -> unI64 271 - '\x0110'# -> unI64 273 - '\x0112'# -> unI64 275 - '\x0114'# -> unI64 277 - '\x0116'# -> unI64 279 - '\x0118'# -> unI64 281 - '\x011a'# -> unI64 283 - '\x011c'# -> unI64 285 - '\x011e'# -> unI64 287 - '\x0120'# -> unI64 289 - '\x0122'# -> unI64 291 - '\x0124'# -> unI64 293 - '\x0126'# -> unI64 295 - '\x0128'# -> unI64 297 - '\x012a'# -> unI64 299 - '\x012c'# -> unI64 301 - '\x012e'# -> unI64 303 - '\x0132'# -> unI64 307 - '\x0134'# -> unI64 309 - '\x0136'# -> unI64 311 - '\x0139'# -> unI64 314 - '\x013b'# -> unI64 316 - '\x013d'# -> unI64 318 - '\x013f'# -> unI64 320 - '\x0141'# -> unI64 322 - '\x0143'# -> unI64 324 - '\x0145'# -> unI64 326 - '\x0147'# -> unI64 328 - '\x014a'# -> unI64 331 - '\x014c'# -> unI64 333 - '\x014e'# -> unI64 335 - '\x0150'# -> unI64 337 - '\x0152'# -> unI64 339 - '\x0154'# -> unI64 341 - '\x0156'# -> unI64 343 - '\x0158'# -> unI64 345 - '\x015a'# -> unI64 347 - '\x015c'# -> unI64 349 - '\x015e'# -> unI64 351 - '\x0160'# -> unI64 353 - '\x0162'# -> unI64 355 - '\x0164'# -> unI64 357 - '\x0166'# -> unI64 359 - '\x0168'# -> unI64 361 - '\x016a'# -> unI64 363 - '\x016c'# -> unI64 365 - '\x016e'# -> unI64 367 - '\x0170'# -> unI64 369 - '\x0172'# -> unI64 371 - '\x0174'# -> unI64 373 - '\x0176'# -> unI64 375 - '\x0178'# -> unI64 255 - '\x0179'# -> unI64 378 - '\x017b'# -> unI64 380 - '\x017d'# -> unI64 382 - '\x0181'# -> unI64 595 - '\x0182'# -> unI64 387 - '\x0184'# -> unI64 389 - '\x0186'# -> unI64 596 - '\x0187'# -> unI64 392 - '\x0189'# -> unI64 598 - '\x018a'# -> unI64 599 - '\x018b'# -> unI64 396 - '\x018e'# -> unI64 477 - '\x018f'# -> unI64 601 - '\x0190'# -> unI64 603 - '\x0191'# -> unI64 402 - '\x0193'# -> unI64 608 - '\x0194'# -> unI64 611 - '\x0196'# -> unI64 617 - '\x0197'# -> unI64 616 - '\x0198'# -> unI64 409 - '\x019c'# -> unI64 623 - '\x019d'# -> unI64 626 - '\x019f'# -> unI64 629 - '\x01a0'# -> unI64 417 - '\x01a2'# -> unI64 419 - '\x01a4'# -> unI64 421 - '\x01a6'# -> unI64 640 - '\x01a7'# -> unI64 424 - '\x01a9'# -> unI64 643 - '\x01ac'# -> unI64 429 - '\x01ae'# -> unI64 648 - '\x01af'# -> unI64 432 - '\x01b1'# -> unI64 650 - '\x01b2'# -> unI64 651 - '\x01b3'# -> unI64 436 - '\x01b5'# -> unI64 438 - '\x01b7'# -> unI64 658 - '\x01b8'# -> unI64 441 - '\x01bc'# -> unI64 445 - '\x01c4'# -> unI64 454 - '\x01c5'# -> unI64 454 - '\x01c7'# -> unI64 457 - '\x01c8'# -> unI64 457 - '\x01ca'# -> unI64 460 - '\x01cb'# -> unI64 460 - '\x01cd'# -> unI64 462 - '\x01cf'# -> unI64 464 - '\x01d1'# -> unI64 466 - '\x01d3'# -> unI64 468 - '\x01d5'# -> unI64 470 - '\x01d7'# -> unI64 472 - '\x01d9'# -> unI64 474 - '\x01db'# -> unI64 476 - '\x01de'# -> unI64 479 - '\x01e0'# -> unI64 481 - '\x01e2'# -> unI64 483 - '\x01e4'# -> unI64 485 - '\x01e6'# -> unI64 487 - '\x01e8'# -> unI64 489 - '\x01ea'# -> unI64 491 - '\x01ec'# -> unI64 493 - '\x01ee'# -> unI64 495 - '\x01f1'# -> unI64 499 - '\x01f2'# -> unI64 499 - '\x01f4'# -> unI64 501 - '\x01f6'# -> unI64 405 - '\x01f7'# -> unI64 447 - '\x01f8'# -> unI64 505 - '\x01fa'# -> unI64 507 - '\x01fc'# -> unI64 509 - '\x01fe'# -> unI64 511 - '\x0200'# -> unI64 513 - '\x0202'# -> unI64 515 - '\x0204'# -> unI64 517 - '\x0206'# -> unI64 519 - '\x0208'# -> unI64 521 - '\x020a'# -> unI64 523 - '\x020c'# -> unI64 525 - '\x020e'# -> unI64 527 - '\x0210'# -> unI64 529 - '\x0212'# -> unI64 531 - '\x0214'# -> unI64 533 - '\x0216'# -> unI64 535 - '\x0218'# -> unI64 537 - '\x021a'# -> unI64 539 - '\x021c'# -> unI64 541 - '\x021e'# -> unI64 543 - '\x0220'# -> unI64 414 - '\x0222'# -> unI64 547 - '\x0224'# -> unI64 549 - '\x0226'# -> unI64 551 - '\x0228'# -> unI64 553 - '\x022a'# -> unI64 555 - '\x022c'# -> unI64 557 - '\x022e'# -> unI64 559 - '\x0230'# -> unI64 561 - '\x0232'# -> unI64 563 - '\x023a'# -> unI64 11365 - '\x023b'# -> unI64 572 - '\x023d'# -> unI64 410 - '\x023e'# -> unI64 11366 - '\x0241'# -> unI64 578 - '\x0243'# -> unI64 384 - '\x0244'# -> unI64 649 - '\x0245'# -> unI64 652 - '\x0246'# -> unI64 583 - '\x0248'# -> unI64 585 - '\x024a'# -> unI64 587 - '\x024c'# -> unI64 589 - '\x024e'# -> unI64 591 - '\x0370'# -> unI64 881 - '\x0372'# -> unI64 883 - '\x0376'# -> unI64 887 - '\x037f'# -> unI64 1011 - '\x0386'# -> unI64 940 - '\x0388'# -> unI64 941 - '\x0389'# -> unI64 942 - '\x038a'# -> unI64 943 - '\x038c'# -> unI64 972 - '\x038e'# -> unI64 973 - '\x038f'# -> unI64 974 - '\x0391'# -> unI64 945 - '\x0392'# -> unI64 946 - '\x0393'# -> unI64 947 - '\x0394'# -> unI64 948 - '\x0395'# -> unI64 949 - '\x0396'# -> unI64 950 - '\x0397'# -> unI64 951 - '\x0398'# -> unI64 952 - '\x0399'# -> unI64 953 - '\x039a'# -> unI64 954 - '\x039b'# -> unI64 955 - '\x039c'# -> unI64 956 - '\x039d'# -> unI64 957 - '\x039e'# -> unI64 958 - '\x039f'# -> unI64 959 - '\x03a0'# -> unI64 960 - '\x03a1'# -> unI64 961 - '\x03a3'# -> unI64 963 - '\x03a4'# -> unI64 964 - '\x03a5'# -> unI64 965 - '\x03a6'# -> unI64 966 - '\x03a7'# -> unI64 967 - '\x03a8'# -> unI64 968 - '\x03a9'# -> unI64 969 - '\x03aa'# -> unI64 970 - '\x03ab'# -> unI64 971 - '\x03cf'# -> unI64 983 - '\x03d8'# -> unI64 985 - '\x03da'# -> unI64 987 - '\x03dc'# -> unI64 989 - '\x03de'# -> unI64 991 - '\x03e0'# -> unI64 993 - '\x03e2'# -> unI64 995 - '\x03e4'# -> unI64 997 - '\x03e6'# -> unI64 999 - '\x03e8'# -> unI64 1001 - '\x03ea'# -> unI64 1003 - '\x03ec'# -> unI64 1005 - '\x03ee'# -> unI64 1007 - '\x03f4'# -> unI64 952 - '\x03f7'# -> unI64 1016 - '\x03f9'# -> unI64 1010 - '\x03fa'# -> unI64 1019 - '\x03fd'# -> unI64 891 - '\x03fe'# -> unI64 892 - '\x03ff'# -> unI64 893 - '\x0400'# -> unI64 1104 - '\x0401'# -> unI64 1105 - '\x0402'# -> unI64 1106 - '\x0403'# -> unI64 1107 - '\x0404'# -> unI64 1108 - '\x0405'# -> unI64 1109 - '\x0406'# -> unI64 1110 - '\x0407'# -> unI64 1111 - '\x0408'# -> unI64 1112 - '\x0409'# -> unI64 1113 - '\x040a'# -> unI64 1114 - '\x040b'# -> unI64 1115 - '\x040c'# -> unI64 1116 - '\x040d'# -> unI64 1117 - '\x040e'# -> unI64 1118 - '\x040f'# -> unI64 1119 - '\x0410'# -> unI64 1072 - '\x0411'# -> unI64 1073 - '\x0412'# -> unI64 1074 - '\x0413'# -> unI64 1075 - '\x0414'# -> unI64 1076 - '\x0415'# -> unI64 1077 - '\x0416'# -> unI64 1078 - '\x0417'# -> unI64 1079 - '\x0418'# -> unI64 1080 - '\x0419'# -> unI64 1081 - '\x041a'# -> unI64 1082 - '\x041b'# -> unI64 1083 - '\x041c'# -> unI64 1084 - '\x041d'# -> unI64 1085 - '\x041e'# -> unI64 1086 - '\x041f'# -> unI64 1087 - '\x0420'# -> unI64 1088 - '\x0421'# -> unI64 1089 - '\x0422'# -> unI64 1090 - '\x0423'# -> unI64 1091 - '\x0424'# -> unI64 1092 - '\x0425'# -> unI64 1093 - '\x0426'# -> unI64 1094 - '\x0427'# -> unI64 1095 - '\x0428'# -> unI64 1096 - '\x0429'# -> unI64 1097 - '\x042a'# -> unI64 1098 - '\x042b'# -> unI64 1099 - '\x042c'# -> unI64 1100 - '\x042d'# -> unI64 1101 - '\x042e'# -> unI64 1102 - '\x042f'# -> unI64 1103 - '\x0460'# -> unI64 1121 - '\x0462'# -> unI64 1123 - '\x0464'# -> unI64 1125 - '\x0466'# -> unI64 1127 - '\x0468'# -> unI64 1129 - '\x046a'# -> unI64 1131 - '\x046c'# -> unI64 1133 - '\x046e'# -> unI64 1135 - '\x0470'# -> unI64 1137 - '\x0472'# -> unI64 1139 - '\x0474'# -> unI64 1141 - '\x0476'# -> unI64 1143 - '\x0478'# -> unI64 1145 - '\x047a'# -> unI64 1147 - '\x047c'# -> unI64 1149 - '\x047e'# -> unI64 1151 - '\x0480'# -> unI64 1153 - '\x048a'# -> unI64 1163 - '\x048c'# -> unI64 1165 - '\x048e'# -> unI64 1167 - '\x0490'# -> unI64 1169 - '\x0492'# -> unI64 1171 - '\x0494'# -> unI64 1173 - '\x0496'# -> unI64 1175 - '\x0498'# -> unI64 1177 - '\x049a'# -> unI64 1179 - '\x049c'# -> unI64 1181 - '\x049e'# -> unI64 1183 - '\x04a0'# -> unI64 1185 - '\x04a2'# -> unI64 1187 - '\x04a4'# -> unI64 1189 - '\x04a6'# -> unI64 1191 - '\x04a8'# -> unI64 1193 - '\x04aa'# -> unI64 1195 - '\x04ac'# -> unI64 1197 - '\x04ae'# -> unI64 1199 - '\x04b0'# -> unI64 1201 - '\x04b2'# -> unI64 1203 - '\x04b4'# -> unI64 1205 - '\x04b6'# -> unI64 1207 - '\x04b8'# -> unI64 1209 - '\x04ba'# -> unI64 1211 - '\x04bc'# -> unI64 1213 - '\x04be'# -> unI64 1215 - '\x04c0'# -> unI64 1231 - '\x04c1'# -> unI64 1218 - '\x04c3'# -> unI64 1220 - '\x04c5'# -> unI64 1222 - '\x04c7'# -> unI64 1224 - '\x04c9'# -> unI64 1226 - '\x04cb'# -> unI64 1228 - '\x04cd'# -> unI64 1230 - '\x04d0'# -> unI64 1233 - '\x04d2'# -> unI64 1235 - '\x04d4'# -> unI64 1237 - '\x04d6'# -> unI64 1239 - '\x04d8'# -> unI64 1241 - '\x04da'# -> unI64 1243 - '\x04dc'# -> unI64 1245 - '\x04de'# -> unI64 1247 - '\x04e0'# -> unI64 1249 - '\x04e2'# -> unI64 1251 - '\x04e4'# -> unI64 1253 - '\x04e6'# -> unI64 1255 - '\x04e8'# -> unI64 1257 - '\x04ea'# -> unI64 1259 - '\x04ec'# -> unI64 1261 - '\x04ee'# -> unI64 1263 - '\x04f0'# -> unI64 1265 - '\x04f2'# -> unI64 1267 - '\x04f4'# -> unI64 1269 - '\x04f6'# -> unI64 1271 - '\x04f8'# -> unI64 1273 - '\x04fa'# -> unI64 1275 - '\x04fc'# -> unI64 1277 - '\x04fe'# -> unI64 1279 - '\x0500'# -> unI64 1281 - '\x0502'# -> unI64 1283 - '\x0504'# -> unI64 1285 - '\x0506'# -> unI64 1287 - '\x0508'# -> unI64 1289 - '\x050a'# -> unI64 1291 - '\x050c'# -> unI64 1293 - '\x050e'# -> unI64 1295 - '\x0510'# -> unI64 1297 - '\x0512'# -> unI64 1299 - '\x0514'# -> unI64 1301 - '\x0516'# -> unI64 1303 - '\x0518'# -> unI64 1305 - '\x051a'# -> unI64 1307 - '\x051c'# -> unI64 1309 - '\x051e'# -> unI64 1311 - '\x0520'# -> unI64 1313 - '\x0522'# -> unI64 1315 - '\x0524'# -> unI64 1317 - '\x0526'# -> unI64 1319 - '\x0528'# -> unI64 1321 - '\x052a'# -> unI64 1323 - '\x052c'# -> unI64 1325 - '\x052e'# -> unI64 1327 - '\x0531'# -> unI64 1377 - '\x0532'# -> unI64 1378 - '\x0533'# -> unI64 1379 - '\x0534'# -> unI64 1380 - '\x0535'# -> unI64 1381 - '\x0536'# -> unI64 1382 - '\x0537'# -> unI64 1383 - '\x0538'# -> unI64 1384 - '\x0539'# -> unI64 1385 - '\x053a'# -> unI64 1386 - '\x053b'# -> unI64 1387 - '\x053c'# -> unI64 1388 - '\x053d'# -> unI64 1389 - '\x053e'# -> unI64 1390 - '\x053f'# -> unI64 1391 - '\x0540'# -> unI64 1392 - '\x0541'# -> unI64 1393 - '\x0542'# -> unI64 1394 - '\x0543'# -> unI64 1395 - '\x0544'# -> unI64 1396 - '\x0545'# -> unI64 1397 - '\x0546'# -> unI64 1398 - '\x0547'# -> unI64 1399 - '\x0548'# -> unI64 1400 - '\x0549'# -> unI64 1401 - '\x054a'# -> unI64 1402 - '\x054b'# -> unI64 1403 - '\x054c'# -> unI64 1404 - '\x054d'# -> unI64 1405 - '\x054e'# -> unI64 1406 - '\x054f'# -> unI64 1407 - '\x0550'# -> unI64 1408 - '\x0551'# -> unI64 1409 - '\x0552'# -> unI64 1410 - '\x0553'# -> unI64 1411 - '\x0554'# -> unI64 1412 - '\x0555'# -> unI64 1413 - '\x0556'# -> unI64 1414 - '\x10a0'# -> unI64 11520 - '\x10a1'# -> unI64 11521 - '\x10a2'# -> unI64 11522 - '\x10a3'# -> unI64 11523 - '\x10a4'# -> unI64 11524 - '\x10a5'# -> unI64 11525 - '\x10a6'# -> unI64 11526 - '\x10a7'# -> unI64 11527 - '\x10a8'# -> unI64 11528 - '\x10a9'# -> unI64 11529 - '\x10aa'# -> unI64 11530 - '\x10ab'# -> unI64 11531 - '\x10ac'# -> unI64 11532 - '\x10ad'# -> unI64 11533 - '\x10ae'# -> unI64 11534 - '\x10af'# -> unI64 11535 - '\x10b0'# -> unI64 11536 - '\x10b1'# -> unI64 11537 - '\x10b2'# -> unI64 11538 - '\x10b3'# -> unI64 11539 - '\x10b4'# -> unI64 11540 - '\x10b5'# -> unI64 11541 - '\x10b6'# -> unI64 11542 - '\x10b7'# -> unI64 11543 - '\x10b8'# -> unI64 11544 - '\x10b9'# -> unI64 11545 - '\x10ba'# -> unI64 11546 - '\x10bb'# -> unI64 11547 - '\x10bc'# -> unI64 11548 - '\x10bd'# -> unI64 11549 - '\x10be'# -> unI64 11550 - '\x10bf'# -> unI64 11551 - '\x10c0'# -> unI64 11552 - '\x10c1'# -> unI64 11553 - '\x10c2'# -> unI64 11554 - '\x10c3'# -> unI64 11555 - '\x10c4'# -> unI64 11556 - '\x10c5'# -> unI64 11557 - '\x10c7'# -> unI64 11559 - '\x10cd'# -> unI64 11565 - '\x13a0'# -> unI64 43888 - '\x13a1'# -> unI64 43889 - '\x13a2'# -> unI64 43890 - '\x13a3'# -> unI64 43891 - '\x13a4'# -> unI64 43892 - '\x13a5'# -> unI64 43893 - '\x13a6'# -> unI64 43894 - '\x13a7'# -> unI64 43895 - '\x13a8'# -> unI64 43896 - '\x13a9'# -> unI64 43897 - '\x13aa'# -> unI64 43898 - '\x13ab'# -> unI64 43899 - '\x13ac'# -> unI64 43900 - '\x13ad'# -> unI64 43901 - '\x13ae'# -> unI64 43902 - '\x13af'# -> unI64 43903 - '\x13b0'# -> unI64 43904 - '\x13b1'# -> unI64 43905 - '\x13b2'# -> unI64 43906 - '\x13b3'# -> unI64 43907 - '\x13b4'# -> unI64 43908 - '\x13b5'# -> unI64 43909 - '\x13b6'# -> unI64 43910 - '\x13b7'# -> unI64 43911 - '\x13b8'# -> unI64 43912 - '\x13b9'# -> unI64 43913 - '\x13ba'# -> unI64 43914 - '\x13bb'# -> unI64 43915 - '\x13bc'# -> unI64 43916 - '\x13bd'# -> unI64 43917 - '\x13be'# -> unI64 43918 - '\x13bf'# -> unI64 43919 - '\x13c0'# -> unI64 43920 - '\x13c1'# -> unI64 43921 - '\x13c2'# -> unI64 43922 - '\x13c3'# -> unI64 43923 - '\x13c4'# -> unI64 43924 - '\x13c5'# -> unI64 43925 - '\x13c6'# -> unI64 43926 - '\x13c7'# -> unI64 43927 - '\x13c8'# -> unI64 43928 - '\x13c9'# -> unI64 43929 - '\x13ca'# -> unI64 43930 - '\x13cb'# -> unI64 43931 - '\x13cc'# -> unI64 43932 - '\x13cd'# -> unI64 43933 - '\x13ce'# -> unI64 43934 - '\x13cf'# -> unI64 43935 - '\x13d0'# -> unI64 43936 - '\x13d1'# -> unI64 43937 - '\x13d2'# -> unI64 43938 - '\x13d3'# -> unI64 43939 - '\x13d4'# -> unI64 43940 - '\x13d5'# -> unI64 43941 - '\x13d6'# -> unI64 43942 - '\x13d7'# -> unI64 43943 - '\x13d8'# -> unI64 43944 - '\x13d9'# -> unI64 43945 - '\x13da'# -> unI64 43946 - '\x13db'# -> unI64 43947 - '\x13dc'# -> unI64 43948 - '\x13dd'# -> unI64 43949 - '\x13de'# -> unI64 43950 - '\x13df'# -> unI64 43951 - '\x13e0'# -> unI64 43952 - '\x13e1'# -> unI64 43953 - '\x13e2'# -> unI64 43954 - '\x13e3'# -> unI64 43955 - '\x13e4'# -> unI64 43956 - '\x13e5'# -> unI64 43957 - '\x13e6'# -> unI64 43958 - '\x13e7'# -> unI64 43959 - '\x13e8'# -> unI64 43960 - '\x13e9'# -> unI64 43961 - '\x13ea'# -> unI64 43962 - '\x13eb'# -> unI64 43963 - '\x13ec'# -> unI64 43964 - '\x13ed'# -> unI64 43965 - '\x13ee'# -> unI64 43966 - '\x13ef'# -> unI64 43967 - '\x13f0'# -> unI64 5112 - '\x13f1'# -> unI64 5113 - '\x13f2'# -> unI64 5114 - '\x13f3'# -> unI64 5115 - '\x13f4'# -> unI64 5116 - '\x13f5'# -> unI64 5117 - '\x1c90'# -> unI64 4304 - '\x1c91'# -> unI64 4305 - '\x1c92'# -> unI64 4306 - '\x1c93'# -> unI64 4307 - '\x1c94'# -> unI64 4308 - '\x1c95'# -> unI64 4309 - '\x1c96'# -> unI64 4310 - '\x1c97'# -> unI64 4311 - '\x1c98'# -> unI64 4312 - '\x1c99'# -> unI64 4313 - '\x1c9a'# -> unI64 4314 - '\x1c9b'# -> unI64 4315 - '\x1c9c'# -> unI64 4316 - '\x1c9d'# -> unI64 4317 - '\x1c9e'# -> unI64 4318 - '\x1c9f'# -> unI64 4319 - '\x1ca0'# -> unI64 4320 - '\x1ca1'# -> unI64 4321 - '\x1ca2'# -> unI64 4322 - '\x1ca3'# -> unI64 4323 - '\x1ca4'# -> unI64 4324 - '\x1ca5'# -> unI64 4325 - '\x1ca6'# -> unI64 4326 - '\x1ca7'# -> unI64 4327 - '\x1ca8'# -> unI64 4328 - '\x1ca9'# -> unI64 4329 - '\x1caa'# -> unI64 4330 - '\x1cab'# -> unI64 4331 - '\x1cac'# -> unI64 4332 - '\x1cad'# -> unI64 4333 - '\x1cae'# -> unI64 4334 - '\x1caf'# -> unI64 4335 - '\x1cb0'# -> unI64 4336 - '\x1cb1'# -> unI64 4337 - '\x1cb2'# -> unI64 4338 - '\x1cb3'# -> unI64 4339 - '\x1cb4'# -> unI64 4340 - '\x1cb5'# -> unI64 4341 - '\x1cb6'# -> unI64 4342 - '\x1cb7'# -> unI64 4343 - '\x1cb8'# -> unI64 4344 - '\x1cb9'# -> unI64 4345 - '\x1cba'# -> unI64 4346 - '\x1cbd'# -> unI64 4349 - '\x1cbe'# -> unI64 4350 - '\x1cbf'# -> unI64 4351 - '\x1e00'# -> unI64 7681 - '\x1e02'# -> unI64 7683 - '\x1e04'# -> unI64 7685 - '\x1e06'# -> unI64 7687 - '\x1e08'# -> unI64 7689 - '\x1e0a'# -> unI64 7691 - '\x1e0c'# -> unI64 7693 - '\x1e0e'# -> unI64 7695 - '\x1e10'# -> unI64 7697 - '\x1e12'# -> unI64 7699 - '\x1e14'# -> unI64 7701 - '\x1e16'# -> unI64 7703 - '\x1e18'# -> unI64 7705 - '\x1e1a'# -> unI64 7707 - '\x1e1c'# -> unI64 7709 - '\x1e1e'# -> unI64 7711 - '\x1e20'# -> unI64 7713 - '\x1e22'# -> unI64 7715 - '\x1e24'# -> unI64 7717 - '\x1e26'# -> unI64 7719 - '\x1e28'# -> unI64 7721 - '\x1e2a'# -> unI64 7723 - '\x1e2c'# -> unI64 7725 - '\x1e2e'# -> unI64 7727 - '\x1e30'# -> unI64 7729 - '\x1e32'# -> unI64 7731 - '\x1e34'# -> unI64 7733 - '\x1e36'# -> unI64 7735 - '\x1e38'# -> unI64 7737 - '\x1e3a'# -> unI64 7739 - '\x1e3c'# -> unI64 7741 - '\x1e3e'# -> unI64 7743 - '\x1e40'# -> unI64 7745 - '\x1e42'# -> unI64 7747 - '\x1e44'# -> unI64 7749 - '\x1e46'# -> unI64 7751 - '\x1e48'# -> unI64 7753 - '\x1e4a'# -> unI64 7755 - '\x1e4c'# -> unI64 7757 - '\x1e4e'# -> unI64 7759 - '\x1e50'# -> unI64 7761 - '\x1e52'# -> unI64 7763 - '\x1e54'# -> unI64 7765 - '\x1e56'# -> unI64 7767 - '\x1e58'# -> unI64 7769 - '\x1e5a'# -> unI64 7771 - '\x1e5c'# -> unI64 7773 - '\x1e5e'# -> unI64 7775 - '\x1e60'# -> unI64 7777 - '\x1e62'# -> unI64 7779 - '\x1e64'# -> unI64 7781 - '\x1e66'# -> unI64 7783 - '\x1e68'# -> unI64 7785 - '\x1e6a'# -> unI64 7787 - '\x1e6c'# -> unI64 7789 - '\x1e6e'# -> unI64 7791 - '\x1e70'# -> unI64 7793 - '\x1e72'# -> unI64 7795 - '\x1e74'# -> unI64 7797 - '\x1e76'# -> unI64 7799 - '\x1e78'# -> unI64 7801 - '\x1e7a'# -> unI64 7803 - '\x1e7c'# -> unI64 7805 - '\x1e7e'# -> unI64 7807 - '\x1e80'# -> unI64 7809 - '\x1e82'# -> unI64 7811 - '\x1e84'# -> unI64 7813 - '\x1e86'# -> unI64 7815 - '\x1e88'# -> unI64 7817 - '\x1e8a'# -> unI64 7819 - '\x1e8c'# -> unI64 7821 - '\x1e8e'# -> unI64 7823 - '\x1e90'# -> unI64 7825 - '\x1e92'# -> unI64 7827 - '\x1e94'# -> unI64 7829 - '\x1e9e'# -> unI64 223 - '\x1ea0'# -> unI64 7841 - '\x1ea2'# -> unI64 7843 - '\x1ea4'# -> unI64 7845 - '\x1ea6'# -> unI64 7847 - '\x1ea8'# -> unI64 7849 - '\x1eaa'# -> unI64 7851 - '\x1eac'# -> unI64 7853 - '\x1eae'# -> unI64 7855 - '\x1eb0'# -> unI64 7857 - '\x1eb2'# -> unI64 7859 - '\x1eb4'# -> unI64 7861 - '\x1eb6'# -> unI64 7863 - '\x1eb8'# -> unI64 7865 - '\x1eba'# -> unI64 7867 - '\x1ebc'# -> unI64 7869 - '\x1ebe'# -> unI64 7871 - '\x1ec0'# -> unI64 7873 - '\x1ec2'# -> unI64 7875 - '\x1ec4'# -> unI64 7877 - '\x1ec6'# -> unI64 7879 - '\x1ec8'# -> unI64 7881 - '\x1eca'# -> unI64 7883 - '\x1ecc'# -> unI64 7885 - '\x1ece'# -> unI64 7887 - '\x1ed0'# -> unI64 7889 - '\x1ed2'# -> unI64 7891 - '\x1ed4'# -> unI64 7893 - '\x1ed6'# -> unI64 7895 - '\x1ed8'# -> unI64 7897 - '\x1eda'# -> unI64 7899 - '\x1edc'# -> unI64 7901 - '\x1ede'# -> unI64 7903 - '\x1ee0'# -> unI64 7905 - '\x1ee2'# -> unI64 7907 - '\x1ee4'# -> unI64 7909 - '\x1ee6'# -> unI64 7911 - '\x1ee8'# -> unI64 7913 - '\x1eea'# -> unI64 7915 - '\x1eec'# -> unI64 7917 - '\x1eee'# -> unI64 7919 - '\x1ef0'# -> unI64 7921 - '\x1ef2'# -> unI64 7923 - '\x1ef4'# -> unI64 7925 - '\x1ef6'# -> unI64 7927 - '\x1ef8'# -> unI64 7929 - '\x1efa'# -> unI64 7931 - '\x1efc'# -> unI64 7933 - '\x1efe'# -> unI64 7935 - '\x1f08'# -> unI64 7936 - '\x1f09'# -> unI64 7937 - '\x1f0a'# -> unI64 7938 - '\x1f0b'# -> unI64 7939 - '\x1f0c'# -> unI64 7940 - '\x1f0d'# -> unI64 7941 - '\x1f0e'# -> unI64 7942 - '\x1f0f'# -> unI64 7943 - '\x1f18'# -> unI64 7952 - '\x1f19'# -> unI64 7953 - '\x1f1a'# -> unI64 7954 - '\x1f1b'# -> unI64 7955 - '\x1f1c'# -> unI64 7956 - '\x1f1d'# -> unI64 7957 - '\x1f28'# -> unI64 7968 - '\x1f29'# -> unI64 7969 - '\x1f2a'# -> unI64 7970 - '\x1f2b'# -> unI64 7971 - '\x1f2c'# -> unI64 7972 - '\x1f2d'# -> unI64 7973 - '\x1f2e'# -> unI64 7974 - '\x1f2f'# -> unI64 7975 - '\x1f38'# -> unI64 7984 - '\x1f39'# -> unI64 7985 - '\x1f3a'# -> unI64 7986 - '\x1f3b'# -> unI64 7987 - '\x1f3c'# -> unI64 7988 - '\x1f3d'# -> unI64 7989 - '\x1f3e'# -> unI64 7990 - '\x1f3f'# -> unI64 7991 - '\x1f48'# -> unI64 8000 - '\x1f49'# -> unI64 8001 - '\x1f4a'# -> unI64 8002 - '\x1f4b'# -> unI64 8003 - '\x1f4c'# -> unI64 8004 - '\x1f4d'# -> unI64 8005 - '\x1f59'# -> unI64 8017 - '\x1f5b'# -> unI64 8019 - '\x1f5d'# -> unI64 8021 - '\x1f5f'# -> unI64 8023 - '\x1f68'# -> unI64 8032 - '\x1f69'# -> unI64 8033 - '\x1f6a'# -> unI64 8034 - '\x1f6b'# -> unI64 8035 - '\x1f6c'# -> unI64 8036 - '\x1f6d'# -> unI64 8037 - '\x1f6e'# -> unI64 8038 - '\x1f6f'# -> unI64 8039 - '\x1f88'# -> unI64 8064 - '\x1f89'# -> unI64 8065 - '\x1f8a'# -> unI64 8066 - '\x1f8b'# -> unI64 8067 - '\x1f8c'# -> unI64 8068 - '\x1f8d'# -> unI64 8069 - '\x1f8e'# -> unI64 8070 - '\x1f8f'# -> unI64 8071 - '\x1f98'# -> unI64 8080 - '\x1f99'# -> unI64 8081 - '\x1f9a'# -> unI64 8082 - '\x1f9b'# -> unI64 8083 - '\x1f9c'# -> unI64 8084 - '\x1f9d'# -> unI64 8085 - '\x1f9e'# -> unI64 8086 - '\x1f9f'# -> unI64 8087 - '\x1fa8'# -> unI64 8096 - '\x1fa9'# -> unI64 8097 - '\x1faa'# -> unI64 8098 - '\x1fab'# -> unI64 8099 - '\x1fac'# -> unI64 8100 - '\x1fad'# -> unI64 8101 - '\x1fae'# -> unI64 8102 - '\x1faf'# -> unI64 8103 - '\x1fb8'# -> unI64 8112 - '\x1fb9'# -> unI64 8113 - '\x1fba'# -> unI64 8048 - '\x1fbb'# -> unI64 8049 - '\x1fbc'# -> unI64 8115 - '\x1fc8'# -> unI64 8050 - '\x1fc9'# -> unI64 8051 - '\x1fca'# -> unI64 8052 - '\x1fcb'# -> unI64 8053 - '\x1fcc'# -> unI64 8131 - '\x1fd8'# -> unI64 8144 - '\x1fd9'# -> unI64 8145 - '\x1fda'# -> unI64 8054 - '\x1fdb'# -> unI64 8055 - '\x1fe8'# -> unI64 8160 - '\x1fe9'# -> unI64 8161 - '\x1fea'# -> unI64 8058 - '\x1feb'# -> unI64 8059 - '\x1fec'# -> unI64 8165 - '\x1ff8'# -> unI64 8056 - '\x1ff9'# -> unI64 8057 - '\x1ffa'# -> unI64 8060 - '\x1ffb'# -> unI64 8061 - '\x1ffc'# -> unI64 8179 - '\x2126'# -> unI64 969 - '\x212a'# -> unI64 107 - '\x212b'# -> unI64 229 - '\x2132'# -> unI64 8526 - '\x2160'# -> unI64 8560 - '\x2161'# -> unI64 8561 - '\x2162'# -> unI64 8562 - '\x2163'# -> unI64 8563 - '\x2164'# -> unI64 8564 - '\x2165'# -> unI64 8565 - '\x2166'# -> unI64 8566 - '\x2167'# -> unI64 8567 - '\x2168'# -> unI64 8568 - '\x2169'# -> unI64 8569 - '\x216a'# -> unI64 8570 - '\x216b'# -> unI64 8571 - '\x216c'# -> unI64 8572 - '\x216d'# -> unI64 8573 - '\x216e'# -> unI64 8574 - '\x216f'# -> unI64 8575 - '\x2183'# -> unI64 8580 - '\x24b6'# -> unI64 9424 - '\x24b7'# -> unI64 9425 - '\x24b8'# -> unI64 9426 - '\x24b9'# -> unI64 9427 - '\x24ba'# -> unI64 9428 - '\x24bb'# -> unI64 9429 - '\x24bc'# -> unI64 9430 - '\x24bd'# -> unI64 9431 - '\x24be'# -> unI64 9432 - '\x24bf'# -> unI64 9433 - '\x24c0'# -> unI64 9434 - '\x24c1'# -> unI64 9435 - '\x24c2'# -> unI64 9436 - '\x24c3'# -> unI64 9437 - '\x24c4'# -> unI64 9438 - '\x24c5'# -> unI64 9439 - '\x24c6'# -> unI64 9440 - '\x24c7'# -> unI64 9441 - '\x24c8'# -> unI64 9442 - '\x24c9'# -> unI64 9443 - '\x24ca'# -> unI64 9444 - '\x24cb'# -> unI64 9445 - '\x24cc'# -> unI64 9446 - '\x24cd'# -> unI64 9447 - '\x24ce'# -> unI64 9448 - '\x24cf'# -> unI64 9449 - '\x2c00'# -> unI64 11312 - '\x2c01'# -> unI64 11313 - '\x2c02'# -> unI64 11314 - '\x2c03'# -> unI64 11315 - '\x2c04'# -> unI64 11316 - '\x2c05'# -> unI64 11317 - '\x2c06'# -> unI64 11318 - '\x2c07'# -> unI64 11319 - '\x2c08'# -> unI64 11320 - '\x2c09'# -> unI64 11321 - '\x2c0a'# -> unI64 11322 - '\x2c0b'# -> unI64 11323 - '\x2c0c'# -> unI64 11324 - '\x2c0d'# -> unI64 11325 - '\x2c0e'# -> unI64 11326 - '\x2c0f'# -> unI64 11327 - '\x2c10'# -> unI64 11328 - '\x2c11'# -> unI64 11329 - '\x2c12'# -> unI64 11330 - '\x2c13'# -> unI64 11331 - '\x2c14'# -> unI64 11332 - '\x2c15'# -> unI64 11333 - '\x2c16'# -> unI64 11334 - '\x2c17'# -> unI64 11335 - '\x2c18'# -> unI64 11336 - '\x2c19'# -> unI64 11337 - '\x2c1a'# -> unI64 11338 - '\x2c1b'# -> unI64 11339 - '\x2c1c'# -> unI64 11340 - '\x2c1d'# -> unI64 11341 - '\x2c1e'# -> unI64 11342 - '\x2c1f'# -> unI64 11343 - '\x2c20'# -> unI64 11344 - '\x2c21'# -> unI64 11345 - '\x2c22'# -> unI64 11346 - '\x2c23'# -> unI64 11347 - '\x2c24'# -> unI64 11348 - '\x2c25'# -> unI64 11349 - '\x2c26'# -> unI64 11350 - '\x2c27'# -> unI64 11351 - '\x2c28'# -> unI64 11352 - '\x2c29'# -> unI64 11353 - '\x2c2a'# -> unI64 11354 - '\x2c2b'# -> unI64 11355 - '\x2c2c'# -> unI64 11356 - '\x2c2d'# -> unI64 11357 - '\x2c2e'# -> unI64 11358 - '\x2c2f'# -> unI64 11359 - '\x2c60'# -> unI64 11361 - '\x2c62'# -> unI64 619 - '\x2c63'# -> unI64 7549 - '\x2c64'# -> unI64 637 - '\x2c67'# -> unI64 11368 - '\x2c69'# -> unI64 11370 - '\x2c6b'# -> unI64 11372 - '\x2c6d'# -> unI64 593 - '\x2c6e'# -> unI64 625 - '\x2c6f'# -> unI64 592 - '\x2c70'# -> unI64 594 - '\x2c72'# -> unI64 11379 - '\x2c75'# -> unI64 11382 - '\x2c7e'# -> unI64 575 - '\x2c7f'# -> unI64 576 - '\x2c80'# -> unI64 11393 - '\x2c82'# -> unI64 11395 - '\x2c84'# -> unI64 11397 - '\x2c86'# -> unI64 11399 - '\x2c88'# -> unI64 11401 - '\x2c8a'# -> unI64 11403 - '\x2c8c'# -> unI64 11405 - '\x2c8e'# -> unI64 11407 - '\x2c90'# -> unI64 11409 - '\x2c92'# -> unI64 11411 - '\x2c94'# -> unI64 11413 - '\x2c96'# -> unI64 11415 - '\x2c98'# -> unI64 11417 - '\x2c9a'# -> unI64 11419 - '\x2c9c'# -> unI64 11421 - '\x2c9e'# -> unI64 11423 - '\x2ca0'# -> unI64 11425 - '\x2ca2'# -> unI64 11427 - '\x2ca4'# -> unI64 11429 - '\x2ca6'# -> unI64 11431 - '\x2ca8'# -> unI64 11433 - '\x2caa'# -> unI64 11435 - '\x2cac'# -> unI64 11437 - '\x2cae'# -> unI64 11439 - '\x2cb0'# -> unI64 11441 - '\x2cb2'# -> unI64 11443 - '\x2cb4'# -> unI64 11445 - '\x2cb6'# -> unI64 11447 - '\x2cb8'# -> unI64 11449 - '\x2cba'# -> unI64 11451 - '\x2cbc'# -> unI64 11453 - '\x2cbe'# -> unI64 11455 - '\x2cc0'# -> unI64 11457 - '\x2cc2'# -> unI64 11459 - '\x2cc4'# -> unI64 11461 - '\x2cc6'# -> unI64 11463 - '\x2cc8'# -> unI64 11465 - '\x2cca'# -> unI64 11467 - '\x2ccc'# -> unI64 11469 - '\x2cce'# -> unI64 11471 - '\x2cd0'# -> unI64 11473 - '\x2cd2'# -> unI64 11475 - '\x2cd4'# -> unI64 11477 - '\x2cd6'# -> unI64 11479 - '\x2cd8'# -> unI64 11481 - '\x2cda'# -> unI64 11483 - '\x2cdc'# -> unI64 11485 - '\x2cde'# -> unI64 11487 - '\x2ce0'# -> unI64 11489 - '\x2ce2'# -> unI64 11491 - '\x2ceb'# -> unI64 11500 - '\x2ced'# -> unI64 11502 - '\x2cf2'# -> unI64 11507 - '\xa640'# -> unI64 42561 - '\xa642'# -> unI64 42563 - '\xa644'# -> unI64 42565 - '\xa646'# -> unI64 42567 - '\xa648'# -> unI64 42569 - '\xa64a'# -> unI64 42571 - '\xa64c'# -> unI64 42573 - '\xa64e'# -> unI64 42575 - '\xa650'# -> unI64 42577 - '\xa652'# -> unI64 42579 - '\xa654'# -> unI64 42581 - '\xa656'# -> unI64 42583 - '\xa658'# -> unI64 42585 - '\xa65a'# -> unI64 42587 - '\xa65c'# -> unI64 42589 - '\xa65e'# -> unI64 42591 - '\xa660'# -> unI64 42593 - '\xa662'# -> unI64 42595 - '\xa664'# -> unI64 42597 - '\xa666'# -> unI64 42599 - '\xa668'# -> unI64 42601 - '\xa66a'# -> unI64 42603 - '\xa66c'# -> unI64 42605 - '\xa680'# -> unI64 42625 - '\xa682'# -> unI64 42627 - '\xa684'# -> unI64 42629 - '\xa686'# -> unI64 42631 - '\xa688'# -> unI64 42633 - '\xa68a'# -> unI64 42635 - '\xa68c'# -> unI64 42637 - '\xa68e'# -> unI64 42639 - '\xa690'# -> unI64 42641 - '\xa692'# -> unI64 42643 - '\xa694'# -> unI64 42645 - '\xa696'# -> unI64 42647 - '\xa698'# -> unI64 42649 - '\xa69a'# -> unI64 42651 - '\xa722'# -> unI64 42787 - '\xa724'# -> unI64 42789 - '\xa726'# -> unI64 42791 - '\xa728'# -> unI64 42793 - '\xa72a'# -> unI64 42795 - '\xa72c'# -> unI64 42797 - '\xa72e'# -> unI64 42799 - '\xa732'# -> unI64 42803 - '\xa734'# -> unI64 42805 - '\xa736'# -> unI64 42807 - '\xa738'# -> unI64 42809 - '\xa73a'# -> unI64 42811 - '\xa73c'# -> unI64 42813 - '\xa73e'# -> unI64 42815 - '\xa740'# -> unI64 42817 - '\xa742'# -> unI64 42819 - '\xa744'# -> unI64 42821 - '\xa746'# -> unI64 42823 - '\xa748'# -> unI64 42825 - '\xa74a'# -> unI64 42827 - '\xa74c'# -> unI64 42829 - '\xa74e'# -> unI64 42831 - '\xa750'# -> unI64 42833 - '\xa752'# -> unI64 42835 - '\xa754'# -> unI64 42837 - '\xa756'# -> unI64 42839 - '\xa758'# -> unI64 42841 - '\xa75a'# -> unI64 42843 - '\xa75c'# -> unI64 42845 - '\xa75e'# -> unI64 42847 - '\xa760'# -> unI64 42849 - '\xa762'# -> unI64 42851 - '\xa764'# -> unI64 42853 - '\xa766'# -> unI64 42855 - '\xa768'# -> unI64 42857 - '\xa76a'# -> unI64 42859 - '\xa76c'# -> unI64 42861 - '\xa76e'# -> unI64 42863 - '\xa779'# -> unI64 42874 - '\xa77b'# -> unI64 42876 - '\xa77d'# -> unI64 7545 - '\xa77e'# -> unI64 42879 - '\xa780'# -> unI64 42881 - '\xa782'# -> unI64 42883 - '\xa784'# -> unI64 42885 - '\xa786'# -> unI64 42887 - '\xa78b'# -> unI64 42892 - '\xa78d'# -> unI64 613 - '\xa790'# -> unI64 42897 - '\xa792'# -> unI64 42899 - '\xa796'# -> unI64 42903 - '\xa798'# -> unI64 42905 - '\xa79a'# -> unI64 42907 - '\xa79c'# -> unI64 42909 - '\xa79e'# -> unI64 42911 - '\xa7a0'# -> unI64 42913 - '\xa7a2'# -> unI64 42915 - '\xa7a4'# -> unI64 42917 - '\xa7a6'# -> unI64 42919 - '\xa7a8'# -> unI64 42921 - '\xa7aa'# -> unI64 614 - '\xa7ab'# -> unI64 604 - '\xa7ac'# -> unI64 609 - '\xa7ad'# -> unI64 620 - '\xa7ae'# -> unI64 618 - '\xa7b0'# -> unI64 670 - '\xa7b1'# -> unI64 647 - '\xa7b2'# -> unI64 669 - '\xa7b3'# -> unI64 43859 - '\xa7b4'# -> unI64 42933 - '\xa7b6'# -> unI64 42935 - '\xa7b8'# -> unI64 42937 - '\xa7ba'# -> unI64 42939 - '\xa7bc'# -> unI64 42941 - '\xa7be'# -> unI64 42943 - '\xa7c0'# -> unI64 42945 - '\xa7c2'# -> unI64 42947 - '\xa7c4'# -> unI64 42900 - '\xa7c5'# -> unI64 642 - '\xa7c6'# -> unI64 7566 - '\xa7c7'# -> unI64 42952 - '\xa7c9'# -> unI64 42954 - '\xa7d0'# -> unI64 42961 - '\xa7d6'# -> unI64 42967 - '\xa7d8'# -> unI64 42969 - '\xa7f5'# -> unI64 42998 - '\xff21'# -> unI64 65345 - '\xff22'# -> unI64 65346 - '\xff23'# -> unI64 65347 - '\xff24'# -> unI64 65348 - '\xff25'# -> unI64 65349 - '\xff26'# -> unI64 65350 - '\xff27'# -> unI64 65351 - '\xff28'# -> unI64 65352 - '\xff29'# -> unI64 65353 - '\xff2a'# -> unI64 65354 - '\xff2b'# -> unI64 65355 - '\xff2c'# -> unI64 65356 - '\xff2d'# -> unI64 65357 - '\xff2e'# -> unI64 65358 - '\xff2f'# -> unI64 65359 - '\xff30'# -> unI64 65360 - '\xff31'# -> unI64 65361 - '\xff32'# -> unI64 65362 - '\xff33'# -> unI64 65363 - '\xff34'# -> unI64 65364 - '\xff35'# -> unI64 65365 - '\xff36'# -> unI64 65366 - '\xff37'# -> unI64 65367 - '\xff38'# -> unI64 65368 - '\xff39'# -> unI64 65369 - '\xff3a'# -> unI64 65370 - '\x10400'# -> unI64 66600 - '\x10401'# -> unI64 66601 - '\x10402'# -> unI64 66602 - '\x10403'# -> unI64 66603 - '\x10404'# -> unI64 66604 - '\x10405'# -> unI64 66605 - '\x10406'# -> unI64 66606 - '\x10407'# -> unI64 66607 - '\x10408'# -> unI64 66608 - '\x10409'# -> unI64 66609 - '\x1040a'# -> unI64 66610 - '\x1040b'# -> unI64 66611 - '\x1040c'# -> unI64 66612 - '\x1040d'# -> unI64 66613 - '\x1040e'# -> unI64 66614 - '\x1040f'# -> unI64 66615 - '\x10410'# -> unI64 66616 - '\x10411'# -> unI64 66617 - '\x10412'# -> unI64 66618 - '\x10413'# -> unI64 66619 - '\x10414'# -> unI64 66620 - '\x10415'# -> unI64 66621 - '\x10416'# -> unI64 66622 - '\x10417'# -> unI64 66623 - '\x10418'# -> unI64 66624 - '\x10419'# -> unI64 66625 - '\x1041a'# -> unI64 66626 - '\x1041b'# -> unI64 66627 - '\x1041c'# -> unI64 66628 - '\x1041d'# -> unI64 66629 - '\x1041e'# -> unI64 66630 - '\x1041f'# -> unI64 66631 - '\x10420'# -> unI64 66632 - '\x10421'# -> unI64 66633 - '\x10422'# -> unI64 66634 - '\x10423'# -> unI64 66635 - '\x10424'# -> unI64 66636 - '\x10425'# -> unI64 66637 - '\x10426'# -> unI64 66638 - '\x10427'# -> unI64 66639 - '\x104b0'# -> unI64 66776 - '\x104b1'# -> unI64 66777 - '\x104b2'# -> unI64 66778 - '\x104b3'# -> unI64 66779 - '\x104b4'# -> unI64 66780 - '\x104b5'# -> unI64 66781 - '\x104b6'# -> unI64 66782 - '\x104b7'# -> unI64 66783 - '\x104b8'# -> unI64 66784 - '\x104b9'# -> unI64 66785 - '\x104ba'# -> unI64 66786 - '\x104bb'# -> unI64 66787 - '\x104bc'# -> unI64 66788 - '\x104bd'# -> unI64 66789 - '\x104be'# -> unI64 66790 - '\x104bf'# -> unI64 66791 - '\x104c0'# -> unI64 66792 - '\x104c1'# -> unI64 66793 - '\x104c2'# -> unI64 66794 - '\x104c3'# -> unI64 66795 - '\x104c4'# -> unI64 66796 - '\x104c5'# -> unI64 66797 - '\x104c6'# -> unI64 66798 - '\x104c7'# -> unI64 66799 - '\x104c8'# -> unI64 66800 - '\x104c9'# -> unI64 66801 - '\x104ca'# -> unI64 66802 - '\x104cb'# -> unI64 66803 - '\x104cc'# -> unI64 66804 - '\x104cd'# -> unI64 66805 - '\x104ce'# -> unI64 66806 - '\x104cf'# -> unI64 66807 - '\x104d0'# -> unI64 66808 - '\x104d1'# -> unI64 66809 - '\x104d2'# -> unI64 66810 - '\x104d3'# -> unI64 66811 - '\x10570'# -> unI64 66967 - '\x10571'# -> unI64 66968 - '\x10572'# -> unI64 66969 - '\x10573'# -> unI64 66970 - '\x10574'# -> unI64 66971 - '\x10575'# -> unI64 66972 - '\x10576'# -> unI64 66973 - '\x10577'# -> unI64 66974 - '\x10578'# -> unI64 66975 - '\x10579'# -> unI64 66976 - '\x1057a'# -> unI64 66977 - '\x1057c'# -> unI64 66979 - '\x1057d'# -> unI64 66980 - '\x1057e'# -> unI64 66981 - '\x1057f'# -> unI64 66982 - '\x10580'# -> unI64 66983 - '\x10581'# -> unI64 66984 - '\x10582'# -> unI64 66985 - '\x10583'# -> unI64 66986 - '\x10584'# -> unI64 66987 - '\x10585'# -> unI64 66988 - '\x10586'# -> unI64 66989 - '\x10587'# -> unI64 66990 - '\x10588'# -> unI64 66991 - '\x10589'# -> unI64 66992 - '\x1058a'# -> unI64 66993 - '\x1058c'# -> unI64 66995 - '\x1058d'# -> unI64 66996 - '\x1058e'# -> unI64 66997 - '\x1058f'# -> unI64 66998 - '\x10590'# -> unI64 66999 - '\x10591'# -> unI64 67000 - '\x10592'# -> unI64 67001 - '\x10594'# -> unI64 67003 - '\x10595'# -> unI64 67004 - '\x10c80'# -> unI64 68800 - '\x10c81'# -> unI64 68801 - '\x10c82'# -> unI64 68802 - '\x10c83'# -> unI64 68803 - '\x10c84'# -> unI64 68804 - '\x10c85'# -> unI64 68805 - '\x10c86'# -> unI64 68806 - '\x10c87'# -> unI64 68807 - '\x10c88'# -> unI64 68808 - '\x10c89'# -> unI64 68809 - '\x10c8a'# -> unI64 68810 - '\x10c8b'# -> unI64 68811 - '\x10c8c'# -> unI64 68812 - '\x10c8d'# -> unI64 68813 - '\x10c8e'# -> unI64 68814 - '\x10c8f'# -> unI64 68815 - '\x10c90'# -> unI64 68816 - '\x10c91'# -> unI64 68817 - '\x10c92'# -> unI64 68818 - '\x10c93'# -> unI64 68819 - '\x10c94'# -> unI64 68820 - '\x10c95'# -> unI64 68821 - '\x10c96'# -> unI64 68822 - '\x10c97'# -> unI64 68823 - '\x10c98'# -> unI64 68824 - '\x10c99'# -> unI64 68825 - '\x10c9a'# -> unI64 68826 - '\x10c9b'# -> unI64 68827 - '\x10c9c'# -> unI64 68828 - '\x10c9d'# -> unI64 68829 - '\x10c9e'# -> unI64 68830 - '\x10c9f'# -> unI64 68831 - '\x10ca0'# -> unI64 68832 - '\x10ca1'# -> unI64 68833 - '\x10ca2'# -> unI64 68834 - '\x10ca3'# -> unI64 68835 - '\x10ca4'# -> unI64 68836 - '\x10ca5'# -> unI64 68837 - '\x10ca6'# -> unI64 68838 - '\x10ca7'# -> unI64 68839 - '\x10ca8'# -> unI64 68840 - '\x10ca9'# -> unI64 68841 - '\x10caa'# -> unI64 68842 - '\x10cab'# -> unI64 68843 - '\x10cac'# -> unI64 68844 - '\x10cad'# -> unI64 68845 - '\x10cae'# -> unI64 68846 - '\x10caf'# -> unI64 68847 - '\x10cb0'# -> unI64 68848 - '\x10cb1'# -> unI64 68849 - '\x10cb2'# -> unI64 68850 - '\x118a0'# -> unI64 71872 - '\x118a1'# -> unI64 71873 - '\x118a2'# -> unI64 71874 - '\x118a3'# -> unI64 71875 - '\x118a4'# -> unI64 71876 - '\x118a5'# -> unI64 71877 - '\x118a6'# -> unI64 71878 - '\x118a7'# -> unI64 71879 - '\x118a8'# -> unI64 71880 - '\x118a9'# -> unI64 71881 - '\x118aa'# -> unI64 71882 - '\x118ab'# -> unI64 71883 - '\x118ac'# -> unI64 71884 - '\x118ad'# -> unI64 71885 - '\x118ae'# -> unI64 71886 - '\x118af'# -> unI64 71887 - '\x118b0'# -> unI64 71888 - '\x118b1'# -> unI64 71889 - '\x118b2'# -> unI64 71890 - '\x118b3'# -> unI64 71891 - '\x118b4'# -> unI64 71892 - '\x118b5'# -> unI64 71893 - '\x118b6'# -> unI64 71894 - '\x118b7'# -> unI64 71895 - '\x118b8'# -> unI64 71896 - '\x118b9'# -> unI64 71897 - '\x118ba'# -> unI64 71898 - '\x118bb'# -> unI64 71899 - '\x118bc'# -> unI64 71900 - '\x118bd'# -> unI64 71901 - '\x118be'# -> unI64 71902 - '\x118bf'# -> unI64 71903 - '\x16e40'# -> unI64 93792 - '\x16e41'# -> unI64 93793 - '\x16e42'# -> unI64 93794 - '\x16e43'# -> unI64 93795 - '\x16e44'# -> unI64 93796 - '\x16e45'# -> unI64 93797 - '\x16e46'# -> unI64 93798 - '\x16e47'# -> unI64 93799 - '\x16e48'# -> unI64 93800 - '\x16e49'# -> unI64 93801 - '\x16e4a'# -> unI64 93802 - '\x16e4b'# -> unI64 93803 - '\x16e4c'# -> unI64 93804 - '\x16e4d'# -> unI64 93805 - '\x16e4e'# -> unI64 93806 - '\x16e4f'# -> unI64 93807 - '\x16e50'# -> unI64 93808 - '\x16e51'# -> unI64 93809 - '\x16e52'# -> unI64 93810 - '\x16e53'# -> unI64 93811 - '\x16e54'# -> unI64 93812 - '\x16e55'# -> unI64 93813 - '\x16e56'# -> unI64 93814 - '\x16e57'# -> unI64 93815 - '\x16e58'# -> unI64 93816 - '\x16e59'# -> unI64 93817 - '\x16e5a'# -> unI64 93818 - '\x16e5b'# -> unI64 93819 - '\x16e5c'# -> unI64 93820 - '\x16e5d'# -> unI64 93821 - '\x16e5e'# -> unI64 93822 - '\x16e5f'# -> unI64 93823 - '\x1e900'# -> unI64 125218 - '\x1e901'# -> unI64 125219 - '\x1e902'# -> unI64 125220 - '\x1e903'# -> unI64 125221 - '\x1e904'# -> unI64 125222 - '\x1e905'# -> unI64 125223 - '\x1e906'# -> unI64 125224 - '\x1e907'# -> unI64 125225 - '\x1e908'# -> unI64 125226 - '\x1e909'# -> unI64 125227 - '\x1e90a'# -> unI64 125228 - '\x1e90b'# -> unI64 125229 - '\x1e90c'# -> unI64 125230 - '\x1e90d'# -> unI64 125231 - '\x1e90e'# -> unI64 125232 - '\x1e90f'# -> unI64 125233 - '\x1e910'# -> unI64 125234 - '\x1e911'# -> unI64 125235 - '\x1e912'# -> unI64 125236 - '\x1e913'# -> unI64 125237 - '\x1e914'# -> unI64 125238 - '\x1e915'# -> unI64 125239 - '\x1e916'# -> unI64 125240 - '\x1e917'# -> unI64 125241 - '\x1e918'# -> unI64 125242 - '\x1e919'# -> unI64 125243 - '\x1e91a'# -> unI64 125244 - '\x1e91b'# -> unI64 125245 - '\x1e91c'# -> unI64 125246 - '\x1e91d'# -> unI64 125247 - '\x1e91e'# -> unI64 125248 - '\x1e91f'# -> unI64 125249 - '\x1e920'# -> unI64 125250 - '\x1e921'# -> unI64 125251 - _ -> unI64 0 -titleMapping :: Char# -> _ {- unboxed Int64 -} -{-# NOINLINE titleMapping #-} -titleMapping = \case - -- LATIN SMALL LETTER SHARP S - '\x00df'# -> unI64 241172563 - -- LATIN SMALL LIGATURE FF - '\xfb00'# -> unI64 213909574 - -- LATIN SMALL LIGATURE FI - '\xfb01'# -> unI64 220201030 - -- LATIN SMALL LIGATURE FL - '\xfb02'# -> unI64 226492486 - -- LATIN SMALL LIGATURE FFI - '\xfb03'# -> unI64 461795097575494 - -- LATIN SMALL LIGATURE FFL - '\xfb04'# -> unI64 474989237108806 - -- LATIN SMALL LIGATURE LONG S T - '\xfb05'# -> unI64 243269715 - -- LATIN SMALL LIGATURE ST - '\xfb06'# -> unI64 243269715 - -- ARMENIAN SMALL LIGATURE ECH YIWN - '\x0587'# -> unI64 2956985653 - -- ARMENIAN SMALL LIGATURE MEN NOW - '\xfb13'# -> unI64 2931819844 - -- ARMENIAN SMALL LIGATURE MEN ECH - '\xfb14'# -> unI64 2896168260 - -- ARMENIAN SMALL LIGATURE MEN INI - '\xfb15'# -> unI64 2908751172 - -- ARMENIAN SMALL LIGATURE VEW NOW - '\xfb16'# -> unI64 2931819854 - -- ARMENIAN SMALL LIGATURE MEN XEH - '\xfb17'# -> unI64 2912945476 - -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE - '\x0149'# -> unI64 163578556 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS - '\x0390'# -> unI64 3382099394429849 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS - '\x03b0'# -> unI64 3382099394429861 - -- LATIN SMALL LETTER J WITH CARON - '\x01f0'# -> unI64 1635778634 - -- LATIN SMALL LETTER H WITH LINE BELOW - '\x1e96'# -> unI64 1713373256 - -- LATIN SMALL LETTER T WITH DIAERESIS - '\x1e97'# -> unI64 1627390036 - -- LATIN SMALL LETTER W WITH RING ABOVE - '\x1e98'# -> unI64 1631584343 - -- LATIN SMALL LETTER Y WITH RING ABOVE - '\x1e99'# -> unI64 1631584345 - -- LATIN SMALL LETTER A WITH RIGHT HALF RING - '\x1e9a'# -> unI64 1472200769 - -- GREEK SMALL LETTER UPSILON WITH PSILI - '\x1f50'# -> unI64 1650459557 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA - '\x1f52'# -> unI64 3377701370987429 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA - '\x1f54'# -> unI64 3382099417498533 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI - '\x1f56'# -> unI64 3667972440720293 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI - '\x1fb6'# -> unI64 1749025681 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI - '\x1fc6'# -> unI64 1749025687 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA - '\x1fd2'# -> unI64 3377701347918745 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA - '\x1fd3'# -> unI64 3382099394429849 - -- GREEK SMALL LETTER IOTA WITH PERISPOMENI - '\x1fd6'# -> unI64 1749025689 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI - '\x1fd7'# -> unI64 3667972417651609 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA - '\x1fe2'# -> unI64 3377701347918757 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA - '\x1fe3'# -> unI64 3382099394429861 - -- GREEK SMALL LETTER RHO WITH PSILI - '\x1fe4'# -> unI64 1650459553 - -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI - '\x1fe6'# -> unI64 1749025701 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI - '\x1fe7'# -> unI64 3667972417651621 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI - '\x1ff6'# -> unI64 1749025705 - -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI - '\x1fb2'# -> unI64 1755324346 - -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI - '\x1fb4'# -> unI64 1755317126 - -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI - '\x1fc2'# -> unI64 1755324362 - -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI - '\x1fc4'# -> unI64 1755317129 - -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI - '\x1ff2'# -> unI64 1755324410 - -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI - '\x1ff4'# -> unI64 1755317135 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fb7'# -> unI64 3681166678819729 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fc7'# -> unI64 3681166678819735 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1ff7'# -> unI64 3681166678819753 - '\x0061'# -> unI64 65 - '\x0062'# -> unI64 66 - '\x0063'# -> unI64 67 - '\x0064'# -> unI64 68 - '\x0065'# -> unI64 69 - '\x0066'# -> unI64 70 - '\x0067'# -> unI64 71 - '\x0068'# -> unI64 72 - '\x0069'# -> unI64 73 - '\x006a'# -> unI64 74 - '\x006b'# -> unI64 75 - '\x006c'# -> unI64 76 - '\x006d'# -> unI64 77 - '\x006e'# -> unI64 78 - '\x006f'# -> unI64 79 - '\x0070'# -> unI64 80 - '\x0071'# -> unI64 81 - '\x0072'# -> unI64 82 - '\x0073'# -> unI64 83 - '\x0074'# -> unI64 84 - '\x0075'# -> unI64 85 - '\x0076'# -> unI64 86 - '\x0077'# -> unI64 87 - '\x0078'# -> unI64 88 - '\x0079'# -> unI64 89 - '\x007a'# -> unI64 90 - '\x00b5'# -> unI64 924 - '\x00e0'# -> unI64 192 - '\x00e1'# -> unI64 193 - '\x00e2'# -> unI64 194 - '\x00e3'# -> unI64 195 - '\x00e4'# -> unI64 196 - '\x00e5'# -> unI64 197 - '\x00e6'# -> unI64 198 - '\x00e7'# -> unI64 199 - '\x00e8'# -> unI64 200 - '\x00e9'# -> unI64 201 - '\x00ea'# -> unI64 202 - '\x00eb'# -> unI64 203 - '\x00ec'# -> unI64 204 - '\x00ed'# -> unI64 205 - '\x00ee'# -> unI64 206 - '\x00ef'# -> unI64 207 - '\x00f0'# -> unI64 208 - '\x00f1'# -> unI64 209 - '\x00f2'# -> unI64 210 - '\x00f3'# -> unI64 211 - '\x00f4'# -> unI64 212 - '\x00f5'# -> unI64 213 - '\x00f6'# -> unI64 214 - '\x00f8'# -> unI64 216 - '\x00f9'# -> unI64 217 - '\x00fa'# -> unI64 218 - '\x00fb'# -> unI64 219 - '\x00fc'# -> unI64 220 - '\x00fd'# -> unI64 221 - '\x00fe'# -> unI64 222 - '\x00ff'# -> unI64 376 - '\x0101'# -> unI64 256 - '\x0103'# -> unI64 258 - '\x0105'# -> unI64 260 - '\x0107'# -> unI64 262 - '\x0109'# -> unI64 264 - '\x010b'# -> unI64 266 - '\x010d'# -> unI64 268 - '\x010f'# -> unI64 270 - '\x0111'# -> unI64 272 - '\x0113'# -> unI64 274 - '\x0115'# -> unI64 276 - '\x0117'# -> unI64 278 - '\x0119'# -> unI64 280 - '\x011b'# -> unI64 282 - '\x011d'# -> unI64 284 - '\x011f'# -> unI64 286 - '\x0121'# -> unI64 288 - '\x0123'# -> unI64 290 - '\x0125'# -> unI64 292 - '\x0127'# -> unI64 294 - '\x0129'# -> unI64 296 - '\x012b'# -> unI64 298 - '\x012d'# -> unI64 300 - '\x012f'# -> unI64 302 - '\x0131'# -> unI64 73 - '\x0133'# -> unI64 306 - '\x0135'# -> unI64 308 - '\x0137'# -> unI64 310 - '\x013a'# -> unI64 313 - '\x013c'# -> unI64 315 - '\x013e'# -> unI64 317 - '\x0140'# -> unI64 319 - '\x0142'# -> unI64 321 - '\x0144'# -> unI64 323 - '\x0146'# -> unI64 325 - '\x0148'# -> unI64 327 - '\x014b'# -> unI64 330 - '\x014d'# -> unI64 332 - '\x014f'# -> unI64 334 - '\x0151'# -> unI64 336 - '\x0153'# -> unI64 338 - '\x0155'# -> unI64 340 - '\x0157'# -> unI64 342 - '\x0159'# -> unI64 344 - '\x015b'# -> unI64 346 - '\x015d'# -> unI64 348 - '\x015f'# -> unI64 350 - '\x0161'# -> unI64 352 - '\x0163'# -> unI64 354 - '\x0165'# -> unI64 356 - '\x0167'# -> unI64 358 - '\x0169'# -> unI64 360 - '\x016b'# -> unI64 362 - '\x016d'# -> unI64 364 - '\x016f'# -> unI64 366 - '\x0171'# -> unI64 368 - '\x0173'# -> unI64 370 - '\x0175'# -> unI64 372 - '\x0177'# -> unI64 374 - '\x017a'# -> unI64 377 - '\x017c'# -> unI64 379 - '\x017e'# -> unI64 381 - '\x017f'# -> unI64 83 - '\x0180'# -> unI64 579 - '\x0183'# -> unI64 386 - '\x0185'# -> unI64 388 - '\x0188'# -> unI64 391 - '\x018c'# -> unI64 395 - '\x0192'# -> unI64 401 - '\x0195'# -> unI64 502 - '\x0199'# -> unI64 408 - '\x019a'# -> unI64 573 - '\x019e'# -> unI64 544 - '\x01a1'# -> unI64 416 - '\x01a3'# -> unI64 418 - '\x01a5'# -> unI64 420 - '\x01a8'# -> unI64 423 - '\x01ad'# -> unI64 428 - '\x01b0'# -> unI64 431 - '\x01b4'# -> unI64 435 - '\x01b6'# -> unI64 437 - '\x01b9'# -> unI64 440 - '\x01bd'# -> unI64 444 - '\x01bf'# -> unI64 503 - '\x01c4'# -> unI64 453 - '\x01c6'# -> unI64 453 - '\x01c7'# -> unI64 456 - '\x01c9'# -> unI64 456 - '\x01ca'# -> unI64 459 - '\x01cc'# -> unI64 459 - '\x01ce'# -> unI64 461 - '\x01d0'# -> unI64 463 - '\x01d2'# -> unI64 465 - '\x01d4'# -> unI64 467 - '\x01d6'# -> unI64 469 - '\x01d8'# -> unI64 471 - '\x01da'# -> unI64 473 - '\x01dc'# -> unI64 475 - '\x01dd'# -> unI64 398 - '\x01df'# -> unI64 478 - '\x01e1'# -> unI64 480 - '\x01e3'# -> unI64 482 - '\x01e5'# -> unI64 484 - '\x01e7'# -> unI64 486 - '\x01e9'# -> unI64 488 - '\x01eb'# -> unI64 490 - '\x01ed'# -> unI64 492 - '\x01ef'# -> unI64 494 - '\x01f1'# -> unI64 498 - '\x01f3'# -> unI64 498 - '\x01f5'# -> unI64 500 - '\x01f9'# -> unI64 504 - '\x01fb'# -> unI64 506 - '\x01fd'# -> unI64 508 - '\x01ff'# -> unI64 510 - '\x0201'# -> unI64 512 - '\x0203'# -> unI64 514 - '\x0205'# -> unI64 516 - '\x0207'# -> unI64 518 - '\x0209'# -> unI64 520 - '\x020b'# -> unI64 522 - '\x020d'# -> unI64 524 - '\x020f'# -> unI64 526 - '\x0211'# -> unI64 528 - '\x0213'# -> unI64 530 - '\x0215'# -> unI64 532 - '\x0217'# -> unI64 534 - '\x0219'# -> unI64 536 - '\x021b'# -> unI64 538 - '\x021d'# -> unI64 540 - '\x021f'# -> unI64 542 - '\x0223'# -> unI64 546 - '\x0225'# -> unI64 548 - '\x0227'# -> unI64 550 - '\x0229'# -> unI64 552 - '\x022b'# -> unI64 554 - '\x022d'# -> unI64 556 - '\x022f'# -> unI64 558 - '\x0231'# -> unI64 560 - '\x0233'# -> unI64 562 - '\x023c'# -> unI64 571 - '\x023f'# -> unI64 11390 - '\x0240'# -> unI64 11391 - '\x0242'# -> unI64 577 - '\x0247'# -> unI64 582 - '\x0249'# -> unI64 584 - '\x024b'# -> unI64 586 - '\x024d'# -> unI64 588 - '\x024f'# -> unI64 590 - '\x0250'# -> unI64 11375 - '\x0251'# -> unI64 11373 - '\x0252'# -> unI64 11376 - '\x0253'# -> unI64 385 - '\x0254'# -> unI64 390 - '\x0256'# -> unI64 393 - '\x0257'# -> unI64 394 - '\x0259'# -> unI64 399 - '\x025b'# -> unI64 400 - '\x025c'# -> unI64 42923 - '\x0260'# -> unI64 403 - '\x0261'# -> unI64 42924 - '\x0263'# -> unI64 404 - '\x0265'# -> unI64 42893 - '\x0266'# -> unI64 42922 - '\x0268'# -> unI64 407 - '\x0269'# -> unI64 406 - '\x026a'# -> unI64 42926 - '\x026b'# -> unI64 11362 - '\x026c'# -> unI64 42925 - '\x026f'# -> unI64 412 - '\x0271'# -> unI64 11374 - '\x0272'# -> unI64 413 - '\x0275'# -> unI64 415 - '\x027d'# -> unI64 11364 - '\x0280'# -> unI64 422 - '\x0282'# -> unI64 42949 - '\x0283'# -> unI64 425 - '\x0287'# -> unI64 42929 - '\x0288'# -> unI64 430 - '\x0289'# -> unI64 580 - '\x028a'# -> unI64 433 - '\x028b'# -> unI64 434 - '\x028c'# -> unI64 581 - '\x0292'# -> unI64 439 - '\x029d'# -> unI64 42930 - '\x029e'# -> unI64 42928 - '\x0345'# -> unI64 921 - '\x0371'# -> unI64 880 - '\x0373'# -> unI64 882 - '\x0377'# -> unI64 886 - '\x037b'# -> unI64 1021 - '\x037c'# -> unI64 1022 - '\x037d'# -> unI64 1023 - '\x03ac'# -> unI64 902 - '\x03ad'# -> unI64 904 - '\x03ae'# -> unI64 905 - '\x03af'# -> unI64 906 - '\x03b1'# -> unI64 913 - '\x03b2'# -> unI64 914 - '\x03b3'# -> unI64 915 - '\x03b4'# -> unI64 916 - '\x03b5'# -> unI64 917 - '\x03b6'# -> unI64 918 - '\x03b7'# -> unI64 919 - '\x03b8'# -> unI64 920 - '\x03b9'# -> unI64 921 - '\x03ba'# -> unI64 922 - '\x03bb'# -> unI64 923 - '\x03bc'# -> unI64 924 - '\x03bd'# -> unI64 925 - '\x03be'# -> unI64 926 - '\x03bf'# -> unI64 927 - '\x03c0'# -> unI64 928 - '\x03c1'# -> unI64 929 - '\x03c2'# -> unI64 931 - '\x03c3'# -> unI64 931 - '\x03c4'# -> unI64 932 - '\x03c5'# -> unI64 933 - '\x03c6'# -> unI64 934 - '\x03c7'# -> unI64 935 - '\x03c8'# -> unI64 936 - '\x03c9'# -> unI64 937 - '\x03ca'# -> unI64 938 - '\x03cb'# -> unI64 939 - '\x03cc'# -> unI64 908 - '\x03cd'# -> unI64 910 - '\x03ce'# -> unI64 911 - '\x03d0'# -> unI64 914 - '\x03d1'# -> unI64 920 - '\x03d5'# -> unI64 934 - '\x03d6'# -> unI64 928 - '\x03d7'# -> unI64 975 - '\x03d9'# -> unI64 984 - '\x03db'# -> unI64 986 - '\x03dd'# -> unI64 988 - '\x03df'# -> unI64 990 - '\x03e1'# -> unI64 992 - '\x03e3'# -> unI64 994 - '\x03e5'# -> unI64 996 - '\x03e7'# -> unI64 998 - '\x03e9'# -> unI64 1000 - '\x03eb'# -> unI64 1002 - '\x03ed'# -> unI64 1004 - '\x03ef'# -> unI64 1006 - '\x03f0'# -> unI64 922 - '\x03f1'# -> unI64 929 - '\x03f2'# -> unI64 1017 - '\x03f3'# -> unI64 895 - '\x03f5'# -> unI64 917 - '\x03f8'# -> unI64 1015 - '\x03fb'# -> unI64 1018 - '\x0430'# -> unI64 1040 - '\x0431'# -> unI64 1041 - '\x0432'# -> unI64 1042 - '\x0433'# -> unI64 1043 - '\x0434'# -> unI64 1044 - '\x0435'# -> unI64 1045 - '\x0436'# -> unI64 1046 - '\x0437'# -> unI64 1047 - '\x0438'# -> unI64 1048 - '\x0439'# -> unI64 1049 - '\x043a'# -> unI64 1050 - '\x043b'# -> unI64 1051 - '\x043c'# -> unI64 1052 - '\x043d'# -> unI64 1053 - '\x043e'# -> unI64 1054 - '\x043f'# -> unI64 1055 - '\x0440'# -> unI64 1056 - '\x0441'# -> unI64 1057 - '\x0442'# -> unI64 1058 - '\x0443'# -> unI64 1059 - '\x0444'# -> unI64 1060 - '\x0445'# -> unI64 1061 - '\x0446'# -> unI64 1062 - '\x0447'# -> unI64 1063 - '\x0448'# -> unI64 1064 - '\x0449'# -> unI64 1065 - '\x044a'# -> unI64 1066 - '\x044b'# -> unI64 1067 - '\x044c'# -> unI64 1068 - '\x044d'# -> unI64 1069 - '\x044e'# -> unI64 1070 - '\x044f'# -> unI64 1071 - '\x0450'# -> unI64 1024 - '\x0451'# -> unI64 1025 - '\x0452'# -> unI64 1026 - '\x0453'# -> unI64 1027 - '\x0454'# -> unI64 1028 - '\x0455'# -> unI64 1029 - '\x0456'# -> unI64 1030 - '\x0457'# -> unI64 1031 - '\x0458'# -> unI64 1032 - '\x0459'# -> unI64 1033 - '\x045a'# -> unI64 1034 - '\x045b'# -> unI64 1035 - '\x045c'# -> unI64 1036 - '\x045d'# -> unI64 1037 - '\x045e'# -> unI64 1038 - '\x045f'# -> unI64 1039 - '\x0461'# -> unI64 1120 - '\x0463'# -> unI64 1122 - '\x0465'# -> unI64 1124 - '\x0467'# -> unI64 1126 - '\x0469'# -> unI64 1128 - '\x046b'# -> unI64 1130 - '\x046d'# -> unI64 1132 - '\x046f'# -> unI64 1134 - '\x0471'# -> unI64 1136 - '\x0473'# -> unI64 1138 - '\x0475'# -> unI64 1140 - '\x0477'# -> unI64 1142 - '\x0479'# -> unI64 1144 - '\x047b'# -> unI64 1146 - '\x047d'# -> unI64 1148 - '\x047f'# -> unI64 1150 - '\x0481'# -> unI64 1152 - '\x048b'# -> unI64 1162 - '\x048d'# -> unI64 1164 - '\x048f'# -> unI64 1166 - '\x0491'# -> unI64 1168 - '\x0493'# -> unI64 1170 - '\x0495'# -> unI64 1172 - '\x0497'# -> unI64 1174 - '\x0499'# -> unI64 1176 - '\x049b'# -> unI64 1178 - '\x049d'# -> unI64 1180 - '\x049f'# -> unI64 1182 - '\x04a1'# -> unI64 1184 - '\x04a3'# -> unI64 1186 - '\x04a5'# -> unI64 1188 - '\x04a7'# -> unI64 1190 - '\x04a9'# -> unI64 1192 - '\x04ab'# -> unI64 1194 - '\x04ad'# -> unI64 1196 - '\x04af'# -> unI64 1198 - '\x04b1'# -> unI64 1200 - '\x04b3'# -> unI64 1202 - '\x04b5'# -> unI64 1204 - '\x04b7'# -> unI64 1206 - '\x04b9'# -> unI64 1208 - '\x04bb'# -> unI64 1210 - '\x04bd'# -> unI64 1212 - '\x04bf'# -> unI64 1214 - '\x04c2'# -> unI64 1217 - '\x04c4'# -> unI64 1219 - '\x04c6'# -> unI64 1221 - '\x04c8'# -> unI64 1223 - '\x04ca'# -> unI64 1225 - '\x04cc'# -> unI64 1227 - '\x04ce'# -> unI64 1229 - '\x04cf'# -> unI64 1216 - '\x04d1'# -> unI64 1232 - '\x04d3'# -> unI64 1234 - '\x04d5'# -> unI64 1236 - '\x04d7'# -> unI64 1238 - '\x04d9'# -> unI64 1240 - '\x04db'# -> unI64 1242 - '\x04dd'# -> unI64 1244 - '\x04df'# -> unI64 1246 - '\x04e1'# -> unI64 1248 - '\x04e3'# -> unI64 1250 - '\x04e5'# -> unI64 1252 - '\x04e7'# -> unI64 1254 - '\x04e9'# -> unI64 1256 - '\x04eb'# -> unI64 1258 - '\x04ed'# -> unI64 1260 - '\x04ef'# -> unI64 1262 - '\x04f1'# -> unI64 1264 - '\x04f3'# -> unI64 1266 - '\x04f5'# -> unI64 1268 - '\x04f7'# -> unI64 1270 - '\x04f9'# -> unI64 1272 - '\x04fb'# -> unI64 1274 - '\x04fd'# -> unI64 1276 - '\x04ff'# -> unI64 1278 - '\x0501'# -> unI64 1280 - '\x0503'# -> unI64 1282 - '\x0505'# -> unI64 1284 - '\x0507'# -> unI64 1286 - '\x0509'# -> unI64 1288 - '\x050b'# -> unI64 1290 - '\x050d'# -> unI64 1292 - '\x050f'# -> unI64 1294 - '\x0511'# -> unI64 1296 - '\x0513'# -> unI64 1298 - '\x0515'# -> unI64 1300 - '\x0517'# -> unI64 1302 - '\x0519'# -> unI64 1304 - '\x051b'# -> unI64 1306 - '\x051d'# -> unI64 1308 - '\x051f'# -> unI64 1310 - '\x0521'# -> unI64 1312 - '\x0523'# -> unI64 1314 - '\x0525'# -> unI64 1316 - '\x0527'# -> unI64 1318 - '\x0529'# -> unI64 1320 - '\x052b'# -> unI64 1322 - '\x052d'# -> unI64 1324 - '\x052f'# -> unI64 1326 - '\x0561'# -> unI64 1329 - '\x0562'# -> unI64 1330 - '\x0563'# -> unI64 1331 - '\x0564'# -> unI64 1332 - '\x0565'# -> unI64 1333 - '\x0566'# -> unI64 1334 - '\x0567'# -> unI64 1335 - '\x0568'# -> unI64 1336 - '\x0569'# -> unI64 1337 - '\x056a'# -> unI64 1338 - '\x056b'# -> unI64 1339 - '\x056c'# -> unI64 1340 - '\x056d'# -> unI64 1341 - '\x056e'# -> unI64 1342 - '\x056f'# -> unI64 1343 - '\x0570'# -> unI64 1344 - '\x0571'# -> unI64 1345 - '\x0572'# -> unI64 1346 - '\x0573'# -> unI64 1347 - '\x0574'# -> unI64 1348 - '\x0575'# -> unI64 1349 - '\x0576'# -> unI64 1350 - '\x0577'# -> unI64 1351 - '\x0578'# -> unI64 1352 - '\x0579'# -> unI64 1353 - '\x057a'# -> unI64 1354 - '\x057b'# -> unI64 1355 - '\x057c'# -> unI64 1356 - '\x057d'# -> unI64 1357 - '\x057e'# -> unI64 1358 - '\x057f'# -> unI64 1359 - '\x0580'# -> unI64 1360 - '\x0581'# -> unI64 1361 - '\x0582'# -> unI64 1362 - '\x0583'# -> unI64 1363 - '\x0584'# -> unI64 1364 - '\x0585'# -> unI64 1365 - '\x0586'# -> unI64 1366 - '\x13f8'# -> unI64 5104 - '\x13f9'# -> unI64 5105 - '\x13fa'# -> unI64 5106 - '\x13fb'# -> unI64 5107 - '\x13fc'# -> unI64 5108 - '\x13fd'# -> unI64 5109 - '\x1c80'# -> unI64 1042 - '\x1c81'# -> unI64 1044 - '\x1c82'# -> unI64 1054 - '\x1c83'# -> unI64 1057 - '\x1c84'# -> unI64 1058 - '\x1c85'# -> unI64 1058 - '\x1c86'# -> unI64 1066 - '\x1c87'# -> unI64 1122 - '\x1c88'# -> unI64 42570 - '\x1d79'# -> unI64 42877 - '\x1d7d'# -> unI64 11363 - '\x1d8e'# -> unI64 42950 - '\x1e01'# -> unI64 7680 - '\x1e03'# -> unI64 7682 - '\x1e05'# -> unI64 7684 - '\x1e07'# -> unI64 7686 - '\x1e09'# -> unI64 7688 - '\x1e0b'# -> unI64 7690 - '\x1e0d'# -> unI64 7692 - '\x1e0f'# -> unI64 7694 - '\x1e11'# -> unI64 7696 - '\x1e13'# -> unI64 7698 - '\x1e15'# -> unI64 7700 - '\x1e17'# -> unI64 7702 - '\x1e19'# -> unI64 7704 - '\x1e1b'# -> unI64 7706 - '\x1e1d'# -> unI64 7708 - '\x1e1f'# -> unI64 7710 - '\x1e21'# -> unI64 7712 - '\x1e23'# -> unI64 7714 - '\x1e25'# -> unI64 7716 - '\x1e27'# -> unI64 7718 - '\x1e29'# -> unI64 7720 - '\x1e2b'# -> unI64 7722 - '\x1e2d'# -> unI64 7724 - '\x1e2f'# -> unI64 7726 - '\x1e31'# -> unI64 7728 - '\x1e33'# -> unI64 7730 - '\x1e35'# -> unI64 7732 - '\x1e37'# -> unI64 7734 - '\x1e39'# -> unI64 7736 - '\x1e3b'# -> unI64 7738 - '\x1e3d'# -> unI64 7740 - '\x1e3f'# -> unI64 7742 - '\x1e41'# -> unI64 7744 - '\x1e43'# -> unI64 7746 - '\x1e45'# -> unI64 7748 - '\x1e47'# -> unI64 7750 - '\x1e49'# -> unI64 7752 - '\x1e4b'# -> unI64 7754 - '\x1e4d'# -> unI64 7756 - '\x1e4f'# -> unI64 7758 - '\x1e51'# -> unI64 7760 - '\x1e53'# -> unI64 7762 - '\x1e55'# -> unI64 7764 - '\x1e57'# -> unI64 7766 - '\x1e59'# -> unI64 7768 - '\x1e5b'# -> unI64 7770 - '\x1e5d'# -> unI64 7772 - '\x1e5f'# -> unI64 7774 - '\x1e61'# -> unI64 7776 - '\x1e63'# -> unI64 7778 - '\x1e65'# -> unI64 7780 - '\x1e67'# -> unI64 7782 - '\x1e69'# -> unI64 7784 - '\x1e6b'# -> unI64 7786 - '\x1e6d'# -> unI64 7788 - '\x1e6f'# -> unI64 7790 - '\x1e71'# -> unI64 7792 - '\x1e73'# -> unI64 7794 - '\x1e75'# -> unI64 7796 - '\x1e77'# -> unI64 7798 - '\x1e79'# -> unI64 7800 - '\x1e7b'# -> unI64 7802 - '\x1e7d'# -> unI64 7804 - '\x1e7f'# -> unI64 7806 - '\x1e81'# -> unI64 7808 - '\x1e83'# -> unI64 7810 - '\x1e85'# -> unI64 7812 - '\x1e87'# -> unI64 7814 - '\x1e89'# -> unI64 7816 - '\x1e8b'# -> unI64 7818 - '\x1e8d'# -> unI64 7820 - '\x1e8f'# -> unI64 7822 - '\x1e91'# -> unI64 7824 - '\x1e93'# -> unI64 7826 - '\x1e95'# -> unI64 7828 - '\x1e9b'# -> unI64 7776 - '\x1ea1'# -> unI64 7840 - '\x1ea3'# -> unI64 7842 - '\x1ea5'# -> unI64 7844 - '\x1ea7'# -> unI64 7846 - '\x1ea9'# -> unI64 7848 - '\x1eab'# -> unI64 7850 - '\x1ead'# -> unI64 7852 - '\x1eaf'# -> unI64 7854 - '\x1eb1'# -> unI64 7856 - '\x1eb3'# -> unI64 7858 - '\x1eb5'# -> unI64 7860 - '\x1eb7'# -> unI64 7862 - '\x1eb9'# -> unI64 7864 - '\x1ebb'# -> unI64 7866 - '\x1ebd'# -> unI64 7868 - '\x1ebf'# -> unI64 7870 - '\x1ec1'# -> unI64 7872 - '\x1ec3'# -> unI64 7874 - '\x1ec5'# -> unI64 7876 - '\x1ec7'# -> unI64 7878 - '\x1ec9'# -> unI64 7880 - '\x1ecb'# -> unI64 7882 - '\x1ecd'# -> unI64 7884 - '\x1ecf'# -> unI64 7886 - '\x1ed1'# -> unI64 7888 - '\x1ed3'# -> unI64 7890 - '\x1ed5'# -> unI64 7892 - '\x1ed7'# -> unI64 7894 - '\x1ed9'# -> unI64 7896 - '\x1edb'# -> unI64 7898 - '\x1edd'# -> unI64 7900 - '\x1edf'# -> unI64 7902 - '\x1ee1'# -> unI64 7904 - '\x1ee3'# -> unI64 7906 - '\x1ee5'# -> unI64 7908 - '\x1ee7'# -> unI64 7910 - '\x1ee9'# -> unI64 7912 - '\x1eeb'# -> unI64 7914 - '\x1eed'# -> unI64 7916 - '\x1eef'# -> unI64 7918 - '\x1ef1'# -> unI64 7920 - '\x1ef3'# -> unI64 7922 - '\x1ef5'# -> unI64 7924 - '\x1ef7'# -> unI64 7926 - '\x1ef9'# -> unI64 7928 - '\x1efb'# -> unI64 7930 - '\x1efd'# -> unI64 7932 - '\x1eff'# -> unI64 7934 - '\x1f00'# -> unI64 7944 - '\x1f01'# -> unI64 7945 - '\x1f02'# -> unI64 7946 - '\x1f03'# -> unI64 7947 - '\x1f04'# -> unI64 7948 - '\x1f05'# -> unI64 7949 - '\x1f06'# -> unI64 7950 - '\x1f07'# -> unI64 7951 - '\x1f10'# -> unI64 7960 - '\x1f11'# -> unI64 7961 - '\x1f12'# -> unI64 7962 - '\x1f13'# -> unI64 7963 - '\x1f14'# -> unI64 7964 - '\x1f15'# -> unI64 7965 - '\x1f20'# -> unI64 7976 - '\x1f21'# -> unI64 7977 - '\x1f22'# -> unI64 7978 - '\x1f23'# -> unI64 7979 - '\x1f24'# -> unI64 7980 - '\x1f25'# -> unI64 7981 - '\x1f26'# -> unI64 7982 - '\x1f27'# -> unI64 7983 - '\x1f30'# -> unI64 7992 - '\x1f31'# -> unI64 7993 - '\x1f32'# -> unI64 7994 - '\x1f33'# -> unI64 7995 - '\x1f34'# -> unI64 7996 - '\x1f35'# -> unI64 7997 - '\x1f36'# -> unI64 7998 - '\x1f37'# -> unI64 7999 - '\x1f40'# -> unI64 8008 - '\x1f41'# -> unI64 8009 - '\x1f42'# -> unI64 8010 - '\x1f43'# -> unI64 8011 - '\x1f44'# -> unI64 8012 - '\x1f45'# -> unI64 8013 - '\x1f51'# -> unI64 8025 - '\x1f53'# -> unI64 8027 - '\x1f55'# -> unI64 8029 - '\x1f57'# -> unI64 8031 - '\x1f60'# -> unI64 8040 - '\x1f61'# -> unI64 8041 - '\x1f62'# -> unI64 8042 - '\x1f63'# -> unI64 8043 - '\x1f64'# -> unI64 8044 - '\x1f65'# -> unI64 8045 - '\x1f66'# -> unI64 8046 - '\x1f67'# -> unI64 8047 - '\x1f70'# -> unI64 8122 - '\x1f71'# -> unI64 8123 - '\x1f72'# -> unI64 8136 - '\x1f73'# -> unI64 8137 - '\x1f74'# -> unI64 8138 - '\x1f75'# -> unI64 8139 - '\x1f76'# -> unI64 8154 - '\x1f77'# -> unI64 8155 - '\x1f78'# -> unI64 8184 - '\x1f79'# -> unI64 8185 - '\x1f7a'# -> unI64 8170 - '\x1f7b'# -> unI64 8171 - '\x1f7c'# -> unI64 8186 - '\x1f7d'# -> unI64 8187 - '\x1f80'# -> unI64 8072 - '\x1f81'# -> unI64 8073 - '\x1f82'# -> unI64 8074 - '\x1f83'# -> unI64 8075 - '\x1f84'# -> unI64 8076 - '\x1f85'# -> unI64 8077 - '\x1f86'# -> unI64 8078 - '\x1f87'# -> unI64 8079 - '\x1f90'# -> unI64 8088 - '\x1f91'# -> unI64 8089 - '\x1f92'# -> unI64 8090 - '\x1f93'# -> unI64 8091 - '\x1f94'# -> unI64 8092 - '\x1f95'# -> unI64 8093 - '\x1f96'# -> unI64 8094 - '\x1f97'# -> unI64 8095 - '\x1fa0'# -> unI64 8104 - '\x1fa1'# -> unI64 8105 - '\x1fa2'# -> unI64 8106 - '\x1fa3'# -> unI64 8107 - '\x1fa4'# -> unI64 8108 - '\x1fa5'# -> unI64 8109 - '\x1fa6'# -> unI64 8110 - '\x1fa7'# -> unI64 8111 - '\x1fb0'# -> unI64 8120 - '\x1fb1'# -> unI64 8121 - '\x1fb3'# -> unI64 8124 - '\x1fbe'# -> unI64 921 - '\x1fc3'# -> unI64 8140 - '\x1fd0'# -> unI64 8152 - '\x1fd1'# -> unI64 8153 - '\x1fe0'# -> unI64 8168 - '\x1fe1'# -> unI64 8169 - '\x1fe5'# -> unI64 8172 - '\x1ff3'# -> unI64 8188 - '\x214e'# -> unI64 8498 - '\x2170'# -> unI64 8544 - '\x2171'# -> unI64 8545 - '\x2172'# -> unI64 8546 - '\x2173'# -> unI64 8547 - '\x2174'# -> unI64 8548 - '\x2175'# -> unI64 8549 - '\x2176'# -> unI64 8550 - '\x2177'# -> unI64 8551 - '\x2178'# -> unI64 8552 - '\x2179'# -> unI64 8553 - '\x217a'# -> unI64 8554 - '\x217b'# -> unI64 8555 - '\x217c'# -> unI64 8556 - '\x217d'# -> unI64 8557 - '\x217e'# -> unI64 8558 - '\x217f'# -> unI64 8559 - '\x2184'# -> unI64 8579 - '\x24d0'# -> unI64 9398 - '\x24d1'# -> unI64 9399 - '\x24d2'# -> unI64 9400 - '\x24d3'# -> unI64 9401 - '\x24d4'# -> unI64 9402 - '\x24d5'# -> unI64 9403 - '\x24d6'# -> unI64 9404 - '\x24d7'# -> unI64 9405 - '\x24d8'# -> unI64 9406 - '\x24d9'# -> unI64 9407 - '\x24da'# -> unI64 9408 - '\x24db'# -> unI64 9409 - '\x24dc'# -> unI64 9410 - '\x24dd'# -> unI64 9411 - '\x24de'# -> unI64 9412 - '\x24df'# -> unI64 9413 - '\x24e0'# -> unI64 9414 - '\x24e1'# -> unI64 9415 - '\x24e2'# -> unI64 9416 - '\x24e3'# -> unI64 9417 - '\x24e4'# -> unI64 9418 - '\x24e5'# -> unI64 9419 - '\x24e6'# -> unI64 9420 - '\x24e7'# -> unI64 9421 - '\x24e8'# -> unI64 9422 - '\x24e9'# -> unI64 9423 - '\x2c30'# -> unI64 11264 - '\x2c31'# -> unI64 11265 - '\x2c32'# -> unI64 11266 - '\x2c33'# -> unI64 11267 - '\x2c34'# -> unI64 11268 - '\x2c35'# -> unI64 11269 - '\x2c36'# -> unI64 11270 - '\x2c37'# -> unI64 11271 - '\x2c38'# -> unI64 11272 - '\x2c39'# -> unI64 11273 - '\x2c3a'# -> unI64 11274 - '\x2c3b'# -> unI64 11275 - '\x2c3c'# -> unI64 11276 - '\x2c3d'# -> unI64 11277 - '\x2c3e'# -> unI64 11278 - '\x2c3f'# -> unI64 11279 - '\x2c40'# -> unI64 11280 - '\x2c41'# -> unI64 11281 - '\x2c42'# -> unI64 11282 - '\x2c43'# -> unI64 11283 - '\x2c44'# -> unI64 11284 - '\x2c45'# -> unI64 11285 - '\x2c46'# -> unI64 11286 - '\x2c47'# -> unI64 11287 - '\x2c48'# -> unI64 11288 - '\x2c49'# -> unI64 11289 - '\x2c4a'# -> unI64 11290 - '\x2c4b'# -> unI64 11291 - '\x2c4c'# -> unI64 11292 - '\x2c4d'# -> unI64 11293 - '\x2c4e'# -> unI64 11294 - '\x2c4f'# -> unI64 11295 - '\x2c50'# -> unI64 11296 - '\x2c51'# -> unI64 11297 - '\x2c52'# -> unI64 11298 - '\x2c53'# -> unI64 11299 - '\x2c54'# -> unI64 11300 - '\x2c55'# -> unI64 11301 - '\x2c56'# -> unI64 11302 - '\x2c57'# -> unI64 11303 - '\x2c58'# -> unI64 11304 - '\x2c59'# -> unI64 11305 - '\x2c5a'# -> unI64 11306 - '\x2c5b'# -> unI64 11307 - '\x2c5c'# -> unI64 11308 - '\x2c5d'# -> unI64 11309 - '\x2c5e'# -> unI64 11310 - '\x2c5f'# -> unI64 11311 - '\x2c61'# -> unI64 11360 - '\x2c65'# -> unI64 570 - '\x2c66'# -> unI64 574 - '\x2c68'# -> unI64 11367 - '\x2c6a'# -> unI64 11369 - '\x2c6c'# -> unI64 11371 - '\x2c73'# -> unI64 11378 - '\x2c76'# -> unI64 11381 - '\x2c81'# -> unI64 11392 - '\x2c83'# -> unI64 11394 - '\x2c85'# -> unI64 11396 - '\x2c87'# -> unI64 11398 - '\x2c89'# -> unI64 11400 - '\x2c8b'# -> unI64 11402 - '\x2c8d'# -> unI64 11404 - '\x2c8f'# -> unI64 11406 - '\x2c91'# -> unI64 11408 - '\x2c93'# -> unI64 11410 - '\x2c95'# -> unI64 11412 - '\x2c97'# -> unI64 11414 - '\x2c99'# -> unI64 11416 - '\x2c9b'# -> unI64 11418 - '\x2c9d'# -> unI64 11420 - '\x2c9f'# -> unI64 11422 - '\x2ca1'# -> unI64 11424 - '\x2ca3'# -> unI64 11426 - '\x2ca5'# -> unI64 11428 - '\x2ca7'# -> unI64 11430 - '\x2ca9'# -> unI64 11432 - '\x2cab'# -> unI64 11434 - '\x2cad'# -> unI64 11436 - '\x2caf'# -> unI64 11438 - '\x2cb1'# -> unI64 11440 - '\x2cb3'# -> unI64 11442 - '\x2cb5'# -> unI64 11444 - '\x2cb7'# -> unI64 11446 - '\x2cb9'# -> unI64 11448 - '\x2cbb'# -> unI64 11450 - '\x2cbd'# -> unI64 11452 - '\x2cbf'# -> unI64 11454 - '\x2cc1'# -> unI64 11456 - '\x2cc3'# -> unI64 11458 - '\x2cc5'# -> unI64 11460 - '\x2cc7'# -> unI64 11462 - '\x2cc9'# -> unI64 11464 - '\x2ccb'# -> unI64 11466 - '\x2ccd'# -> unI64 11468 - '\x2ccf'# -> unI64 11470 - '\x2cd1'# -> unI64 11472 - '\x2cd3'# -> unI64 11474 - '\x2cd5'# -> unI64 11476 - '\x2cd7'# -> unI64 11478 - '\x2cd9'# -> unI64 11480 - '\x2cdb'# -> unI64 11482 - '\x2cdd'# -> unI64 11484 - '\x2cdf'# -> unI64 11486 - '\x2ce1'# -> unI64 11488 - '\x2ce3'# -> unI64 11490 - '\x2cec'# -> unI64 11499 - '\x2cee'# -> unI64 11501 - '\x2cf3'# -> unI64 11506 - '\x2d00'# -> unI64 4256 - '\x2d01'# -> unI64 4257 - '\x2d02'# -> unI64 4258 - '\x2d03'# -> unI64 4259 - '\x2d04'# -> unI64 4260 - '\x2d05'# -> unI64 4261 - '\x2d06'# -> unI64 4262 - '\x2d07'# -> unI64 4263 - '\x2d08'# -> unI64 4264 - '\x2d09'# -> unI64 4265 - '\x2d0a'# -> unI64 4266 - '\x2d0b'# -> unI64 4267 - '\x2d0c'# -> unI64 4268 - '\x2d0d'# -> unI64 4269 - '\x2d0e'# -> unI64 4270 - '\x2d0f'# -> unI64 4271 - '\x2d10'# -> unI64 4272 - '\x2d11'# -> unI64 4273 - '\x2d12'# -> unI64 4274 - '\x2d13'# -> unI64 4275 - '\x2d14'# -> unI64 4276 - '\x2d15'# -> unI64 4277 - '\x2d16'# -> unI64 4278 - '\x2d17'# -> unI64 4279 - '\x2d18'# -> unI64 4280 - '\x2d19'# -> unI64 4281 - '\x2d1a'# -> unI64 4282 - '\x2d1b'# -> unI64 4283 - '\x2d1c'# -> unI64 4284 - '\x2d1d'# -> unI64 4285 - '\x2d1e'# -> unI64 4286 - '\x2d1f'# -> unI64 4287 - '\x2d20'# -> unI64 4288 - '\x2d21'# -> unI64 4289 - '\x2d22'# -> unI64 4290 - '\x2d23'# -> unI64 4291 - '\x2d24'# -> unI64 4292 - '\x2d25'# -> unI64 4293 - '\x2d27'# -> unI64 4295 - '\x2d2d'# -> unI64 4301 - '\xa641'# -> unI64 42560 - '\xa643'# -> unI64 42562 - '\xa645'# -> unI64 42564 - '\xa647'# -> unI64 42566 - '\xa649'# -> unI64 42568 - '\xa64b'# -> unI64 42570 - '\xa64d'# -> unI64 42572 - '\xa64f'# -> unI64 42574 - '\xa651'# -> unI64 42576 - '\xa653'# -> unI64 42578 - '\xa655'# -> unI64 42580 - '\xa657'# -> unI64 42582 - '\xa659'# -> unI64 42584 - '\xa65b'# -> unI64 42586 - '\xa65d'# -> unI64 42588 - '\xa65f'# -> unI64 42590 - '\xa661'# -> unI64 42592 - '\xa663'# -> unI64 42594 - '\xa665'# -> unI64 42596 - '\xa667'# -> unI64 42598 - '\xa669'# -> unI64 42600 - '\xa66b'# -> unI64 42602 - '\xa66d'# -> unI64 42604 - '\xa681'# -> unI64 42624 - '\xa683'# -> unI64 42626 - '\xa685'# -> unI64 42628 - '\xa687'# -> unI64 42630 - '\xa689'# -> unI64 42632 - '\xa68b'# -> unI64 42634 - '\xa68d'# -> unI64 42636 - '\xa68f'# -> unI64 42638 - '\xa691'# -> unI64 42640 - '\xa693'# -> unI64 42642 - '\xa695'# -> unI64 42644 - '\xa697'# -> unI64 42646 - '\xa699'# -> unI64 42648 - '\xa69b'# -> unI64 42650 - '\xa723'# -> unI64 42786 - '\xa725'# -> unI64 42788 - '\xa727'# -> unI64 42790 - '\xa729'# -> unI64 42792 - '\xa72b'# -> unI64 42794 - '\xa72d'# -> unI64 42796 - '\xa72f'# -> unI64 42798 - '\xa733'# -> unI64 42802 - '\xa735'# -> unI64 42804 - '\xa737'# -> unI64 42806 - '\xa739'# -> unI64 42808 - '\xa73b'# -> unI64 42810 - '\xa73d'# -> unI64 42812 - '\xa73f'# -> unI64 42814 - '\xa741'# -> unI64 42816 - '\xa743'# -> unI64 42818 - '\xa745'# -> unI64 42820 - '\xa747'# -> unI64 42822 - '\xa749'# -> unI64 42824 - '\xa74b'# -> unI64 42826 - '\xa74d'# -> unI64 42828 - '\xa74f'# -> unI64 42830 - '\xa751'# -> unI64 42832 - '\xa753'# -> unI64 42834 - '\xa755'# -> unI64 42836 - '\xa757'# -> unI64 42838 - '\xa759'# -> unI64 42840 - '\xa75b'# -> unI64 42842 - '\xa75d'# -> unI64 42844 - '\xa75f'# -> unI64 42846 - '\xa761'# -> unI64 42848 - '\xa763'# -> unI64 42850 - '\xa765'# -> unI64 42852 - '\xa767'# -> unI64 42854 - '\xa769'# -> unI64 42856 - '\xa76b'# -> unI64 42858 - '\xa76d'# -> unI64 42860 - '\xa76f'# -> unI64 42862 - '\xa77a'# -> unI64 42873 - '\xa77c'# -> unI64 42875 - '\xa77f'# -> unI64 42878 - '\xa781'# -> unI64 42880 - '\xa783'# -> unI64 42882 - '\xa785'# -> unI64 42884 - '\xa787'# -> unI64 42886 - '\xa78c'# -> unI64 42891 - '\xa791'# -> unI64 42896 - '\xa793'# -> unI64 42898 - '\xa794'# -> unI64 42948 - '\xa797'# -> unI64 42902 - '\xa799'# -> unI64 42904 - '\xa79b'# -> unI64 42906 - '\xa79d'# -> unI64 42908 - '\xa79f'# -> unI64 42910 - '\xa7a1'# -> unI64 42912 - '\xa7a3'# -> unI64 42914 - '\xa7a5'# -> unI64 42916 - '\xa7a7'# -> unI64 42918 - '\xa7a9'# -> unI64 42920 - '\xa7b5'# -> unI64 42932 - '\xa7b7'# -> unI64 42934 - '\xa7b9'# -> unI64 42936 - '\xa7bb'# -> unI64 42938 - '\xa7bd'# -> unI64 42940 - '\xa7bf'# -> unI64 42942 - '\xa7c1'# -> unI64 42944 - '\xa7c3'# -> unI64 42946 - '\xa7c8'# -> unI64 42951 - '\xa7ca'# -> unI64 42953 - '\xa7d1'# -> unI64 42960 - '\xa7d7'# -> unI64 42966 - '\xa7d9'# -> unI64 42968 - '\xa7f6'# -> unI64 42997 - '\xab53'# -> unI64 42931 - '\xab70'# -> unI64 5024 - '\xab71'# -> unI64 5025 - '\xab72'# -> unI64 5026 - '\xab73'# -> unI64 5027 - '\xab74'# -> unI64 5028 - '\xab75'# -> unI64 5029 - '\xab76'# -> unI64 5030 - '\xab77'# -> unI64 5031 - '\xab78'# -> unI64 5032 - '\xab79'# -> unI64 5033 - '\xab7a'# -> unI64 5034 - '\xab7b'# -> unI64 5035 - '\xab7c'# -> unI64 5036 - '\xab7d'# -> unI64 5037 - '\xab7e'# -> unI64 5038 - '\xab7f'# -> unI64 5039 - '\xab80'# -> unI64 5040 - '\xab81'# -> unI64 5041 - '\xab82'# -> unI64 5042 - '\xab83'# -> unI64 5043 - '\xab84'# -> unI64 5044 - '\xab85'# -> unI64 5045 - '\xab86'# -> unI64 5046 - '\xab87'# -> unI64 5047 - '\xab88'# -> unI64 5048 - '\xab89'# -> unI64 5049 - '\xab8a'# -> unI64 5050 - '\xab8b'# -> unI64 5051 - '\xab8c'# -> unI64 5052 - '\xab8d'# -> unI64 5053 - '\xab8e'# -> unI64 5054 - '\xab8f'# -> unI64 5055 - '\xab90'# -> unI64 5056 - '\xab91'# -> unI64 5057 - '\xab92'# -> unI64 5058 - '\xab93'# -> unI64 5059 - '\xab94'# -> unI64 5060 - '\xab95'# -> unI64 5061 - '\xab96'# -> unI64 5062 - '\xab97'# -> unI64 5063 - '\xab98'# -> unI64 5064 - '\xab99'# -> unI64 5065 - '\xab9a'# -> unI64 5066 - '\xab9b'# -> unI64 5067 - '\xab9c'# -> unI64 5068 - '\xab9d'# -> unI64 5069 - '\xab9e'# -> unI64 5070 - '\xab9f'# -> unI64 5071 - '\xaba0'# -> unI64 5072 - '\xaba1'# -> unI64 5073 - '\xaba2'# -> unI64 5074 - '\xaba3'# -> unI64 5075 - '\xaba4'# -> unI64 5076 - '\xaba5'# -> unI64 5077 - '\xaba6'# -> unI64 5078 - '\xaba7'# -> unI64 5079 - '\xaba8'# -> unI64 5080 - '\xaba9'# -> unI64 5081 - '\xabaa'# -> unI64 5082 - '\xabab'# -> unI64 5083 - '\xabac'# -> unI64 5084 - '\xabad'# -> unI64 5085 - '\xabae'# -> unI64 5086 - '\xabaf'# -> unI64 5087 - '\xabb0'# -> unI64 5088 - '\xabb1'# -> unI64 5089 - '\xabb2'# -> unI64 5090 - '\xabb3'# -> unI64 5091 - '\xabb4'# -> unI64 5092 - '\xabb5'# -> unI64 5093 - '\xabb6'# -> unI64 5094 - '\xabb7'# -> unI64 5095 - '\xabb8'# -> unI64 5096 - '\xabb9'# -> unI64 5097 - '\xabba'# -> unI64 5098 - '\xabbb'# -> unI64 5099 - '\xabbc'# -> unI64 5100 - '\xabbd'# -> unI64 5101 - '\xabbe'# -> unI64 5102 - '\xabbf'# -> unI64 5103 - '\xff41'# -> unI64 65313 - '\xff42'# -> unI64 65314 - '\xff43'# -> unI64 65315 - '\xff44'# -> unI64 65316 - '\xff45'# -> unI64 65317 - '\xff46'# -> unI64 65318 - '\xff47'# -> unI64 65319 - '\xff48'# -> unI64 65320 - '\xff49'# -> unI64 65321 - '\xff4a'# -> unI64 65322 - '\xff4b'# -> unI64 65323 - '\xff4c'# -> unI64 65324 - '\xff4d'# -> unI64 65325 - '\xff4e'# -> unI64 65326 - '\xff4f'# -> unI64 65327 - '\xff50'# -> unI64 65328 - '\xff51'# -> unI64 65329 - '\xff52'# -> unI64 65330 - '\xff53'# -> unI64 65331 - '\xff54'# -> unI64 65332 - '\xff55'# -> unI64 65333 - '\xff56'# -> unI64 65334 - '\xff57'# -> unI64 65335 - '\xff58'# -> unI64 65336 - '\xff59'# -> unI64 65337 - '\xff5a'# -> unI64 65338 - '\x10428'# -> unI64 66560 - '\x10429'# -> unI64 66561 - '\x1042a'# -> unI64 66562 - '\x1042b'# -> unI64 66563 - '\x1042c'# -> unI64 66564 - '\x1042d'# -> unI64 66565 - '\x1042e'# -> unI64 66566 - '\x1042f'# -> unI64 66567 - '\x10430'# -> unI64 66568 - '\x10431'# -> unI64 66569 - '\x10432'# -> unI64 66570 - '\x10433'# -> unI64 66571 - '\x10434'# -> unI64 66572 - '\x10435'# -> unI64 66573 - '\x10436'# -> unI64 66574 - '\x10437'# -> unI64 66575 - '\x10438'# -> unI64 66576 - '\x10439'# -> unI64 66577 - '\x1043a'# -> unI64 66578 - '\x1043b'# -> unI64 66579 - '\x1043c'# -> unI64 66580 - '\x1043d'# -> unI64 66581 - '\x1043e'# -> unI64 66582 - '\x1043f'# -> unI64 66583 - '\x10440'# -> unI64 66584 - '\x10441'# -> unI64 66585 - '\x10442'# -> unI64 66586 - '\x10443'# -> unI64 66587 - '\x10444'# -> unI64 66588 - '\x10445'# -> unI64 66589 - '\x10446'# -> unI64 66590 - '\x10447'# -> unI64 66591 - '\x10448'# -> unI64 66592 - '\x10449'# -> unI64 66593 - '\x1044a'# -> unI64 66594 - '\x1044b'# -> unI64 66595 - '\x1044c'# -> unI64 66596 - '\x1044d'# -> unI64 66597 - '\x1044e'# -> unI64 66598 - '\x1044f'# -> unI64 66599 - '\x104d8'# -> unI64 66736 - '\x104d9'# -> unI64 66737 - '\x104da'# -> unI64 66738 - '\x104db'# -> unI64 66739 - '\x104dc'# -> unI64 66740 - '\x104dd'# -> unI64 66741 - '\x104de'# -> unI64 66742 - '\x104df'# -> unI64 66743 - '\x104e0'# -> unI64 66744 - '\x104e1'# -> unI64 66745 - '\x104e2'# -> unI64 66746 - '\x104e3'# -> unI64 66747 - '\x104e4'# -> unI64 66748 - '\x104e5'# -> unI64 66749 - '\x104e6'# -> unI64 66750 - '\x104e7'# -> unI64 66751 - '\x104e8'# -> unI64 66752 - '\x104e9'# -> unI64 66753 - '\x104ea'# -> unI64 66754 - '\x104eb'# -> unI64 66755 - '\x104ec'# -> unI64 66756 - '\x104ed'# -> unI64 66757 - '\x104ee'# -> unI64 66758 - '\x104ef'# -> unI64 66759 - '\x104f0'# -> unI64 66760 - '\x104f1'# -> unI64 66761 - '\x104f2'# -> unI64 66762 - '\x104f3'# -> unI64 66763 - '\x104f4'# -> unI64 66764 - '\x104f5'# -> unI64 66765 - '\x104f6'# -> unI64 66766 - '\x104f7'# -> unI64 66767 - '\x104f8'# -> unI64 66768 - '\x104f9'# -> unI64 66769 - '\x104fa'# -> unI64 66770 - '\x104fb'# -> unI64 66771 - '\x10597'# -> unI64 66928 - '\x10598'# -> unI64 66929 - '\x10599'# -> unI64 66930 - '\x1059a'# -> unI64 66931 - '\x1059b'# -> unI64 66932 - '\x1059c'# -> unI64 66933 - '\x1059d'# -> unI64 66934 - '\x1059e'# -> unI64 66935 - '\x1059f'# -> unI64 66936 - '\x105a0'# -> unI64 66937 - '\x105a1'# -> unI64 66938 - '\x105a3'# -> unI64 66940 - '\x105a4'# -> unI64 66941 - '\x105a5'# -> unI64 66942 - '\x105a6'# -> unI64 66943 - '\x105a7'# -> unI64 66944 - '\x105a8'# -> unI64 66945 - '\x105a9'# -> unI64 66946 - '\x105aa'# -> unI64 66947 - '\x105ab'# -> unI64 66948 - '\x105ac'# -> unI64 66949 - '\x105ad'# -> unI64 66950 - '\x105ae'# -> unI64 66951 - '\x105af'# -> unI64 66952 - '\x105b0'# -> unI64 66953 - '\x105b1'# -> unI64 66954 - '\x105b3'# -> unI64 66956 - '\x105b4'# -> unI64 66957 - '\x105b5'# -> unI64 66958 - '\x105b6'# -> unI64 66959 - '\x105b7'# -> unI64 66960 - '\x105b8'# -> unI64 66961 - '\x105b9'# -> unI64 66962 - '\x105bb'# -> unI64 66964 - '\x105bc'# -> unI64 66965 - '\x10cc0'# -> unI64 68736 - '\x10cc1'# -> unI64 68737 - '\x10cc2'# -> unI64 68738 - '\x10cc3'# -> unI64 68739 - '\x10cc4'# -> unI64 68740 - '\x10cc5'# -> unI64 68741 - '\x10cc6'# -> unI64 68742 - '\x10cc7'# -> unI64 68743 - '\x10cc8'# -> unI64 68744 - '\x10cc9'# -> unI64 68745 - '\x10cca'# -> unI64 68746 - '\x10ccb'# -> unI64 68747 - '\x10ccc'# -> unI64 68748 - '\x10ccd'# -> unI64 68749 - '\x10cce'# -> unI64 68750 - '\x10ccf'# -> unI64 68751 - '\x10cd0'# -> unI64 68752 - '\x10cd1'# -> unI64 68753 - '\x10cd2'# -> unI64 68754 - '\x10cd3'# -> unI64 68755 - '\x10cd4'# -> unI64 68756 - '\x10cd5'# -> unI64 68757 - '\x10cd6'# -> unI64 68758 - '\x10cd7'# -> unI64 68759 - '\x10cd8'# -> unI64 68760 - '\x10cd9'# -> unI64 68761 - '\x10cda'# -> unI64 68762 - '\x10cdb'# -> unI64 68763 - '\x10cdc'# -> unI64 68764 - '\x10cdd'# -> unI64 68765 - '\x10cde'# -> unI64 68766 - '\x10cdf'# -> unI64 68767 - '\x10ce0'# -> unI64 68768 - '\x10ce1'# -> unI64 68769 - '\x10ce2'# -> unI64 68770 - '\x10ce3'# -> unI64 68771 - '\x10ce4'# -> unI64 68772 - '\x10ce5'# -> unI64 68773 - '\x10ce6'# -> unI64 68774 - '\x10ce7'# -> unI64 68775 - '\x10ce8'# -> unI64 68776 - '\x10ce9'# -> unI64 68777 - '\x10cea'# -> unI64 68778 - '\x10ceb'# -> unI64 68779 - '\x10cec'# -> unI64 68780 - '\x10ced'# -> unI64 68781 - '\x10cee'# -> unI64 68782 - '\x10cef'# -> unI64 68783 - '\x10cf0'# -> unI64 68784 - '\x10cf1'# -> unI64 68785 - '\x10cf2'# -> unI64 68786 - '\x118c0'# -> unI64 71840 - '\x118c1'# -> unI64 71841 - '\x118c2'# -> unI64 71842 - '\x118c3'# -> unI64 71843 - '\x118c4'# -> unI64 71844 - '\x118c5'# -> unI64 71845 - '\x118c6'# -> unI64 71846 - '\x118c7'# -> unI64 71847 - '\x118c8'# -> unI64 71848 - '\x118c9'# -> unI64 71849 - '\x118ca'# -> unI64 71850 - '\x118cb'# -> unI64 71851 - '\x118cc'# -> unI64 71852 - '\x118cd'# -> unI64 71853 - '\x118ce'# -> unI64 71854 - '\x118cf'# -> unI64 71855 - '\x118d0'# -> unI64 71856 - '\x118d1'# -> unI64 71857 - '\x118d2'# -> unI64 71858 - '\x118d3'# -> unI64 71859 - '\x118d4'# -> unI64 71860 - '\x118d5'# -> unI64 71861 - '\x118d6'# -> unI64 71862 - '\x118d7'# -> unI64 71863 - '\x118d8'# -> unI64 71864 - '\x118d9'# -> unI64 71865 - '\x118da'# -> unI64 71866 - '\x118db'# -> unI64 71867 - '\x118dc'# -> unI64 71868 - '\x118dd'# -> unI64 71869 - '\x118de'# -> unI64 71870 - '\x118df'# -> unI64 71871 - '\x16e60'# -> unI64 93760 - '\x16e61'# -> unI64 93761 - '\x16e62'# -> unI64 93762 - '\x16e63'# -> unI64 93763 - '\x16e64'# -> unI64 93764 - '\x16e65'# -> unI64 93765 - '\x16e66'# -> unI64 93766 - '\x16e67'# -> unI64 93767 - '\x16e68'# -> unI64 93768 - '\x16e69'# -> unI64 93769 - '\x16e6a'# -> unI64 93770 - '\x16e6b'# -> unI64 93771 - '\x16e6c'# -> unI64 93772 - '\x16e6d'# -> unI64 93773 - '\x16e6e'# -> unI64 93774 - '\x16e6f'# -> unI64 93775 - '\x16e70'# -> unI64 93776 - '\x16e71'# -> unI64 93777 - '\x16e72'# -> unI64 93778 - '\x16e73'# -> unI64 93779 - '\x16e74'# -> unI64 93780 - '\x16e75'# -> unI64 93781 - '\x16e76'# -> unI64 93782 - '\x16e77'# -> unI64 93783 - '\x16e78'# -> unI64 93784 - '\x16e79'# -> unI64 93785 - '\x16e7a'# -> unI64 93786 - '\x16e7b'# -> unI64 93787 - '\x16e7c'# -> unI64 93788 - '\x16e7d'# -> unI64 93789 - '\x16e7e'# -> unI64 93790 - '\x16e7f'# -> unI64 93791 - '\x1e922'# -> unI64 125184 - '\x1e923'# -> unI64 125185 - '\x1e924'# -> unI64 125186 - '\x1e925'# -> unI64 125187 - '\x1e926'# -> unI64 125188 - '\x1e927'# -> unI64 125189 - '\x1e928'# -> unI64 125190 - '\x1e929'# -> unI64 125191 - '\x1e92a'# -> unI64 125192 - '\x1e92b'# -> unI64 125193 - '\x1e92c'# -> unI64 125194 - '\x1e92d'# -> unI64 125195 - '\x1e92e'# -> unI64 125196 - '\x1e92f'# -> unI64 125197 - '\x1e930'# -> unI64 125198 - '\x1e931'# -> unI64 125199 - '\x1e932'# -> unI64 125200 - '\x1e933'# -> unI64 125201 - '\x1e934'# -> unI64 125202 - '\x1e935'# -> unI64 125203 - '\x1e936'# -> unI64 125204 - '\x1e937'# -> unI64 125205 - '\x1e938'# -> unI64 125206 - '\x1e939'# -> unI64 125207 - '\x1e93a'# -> unI64 125208 - '\x1e93b'# -> unI64 125209 - '\x1e93c'# -> unI64 125210 - '\x1e93d'# -> unI64 125211 - '\x1e93e'# -> unI64 125212 - '\x1e93f'# -> unI64 125213 - '\x1e940'# -> unI64 125214 - '\x1e941'# -> unI64 125215 - '\x1e942'# -> unI64 125216 - '\x1e943'# -> unI64 125217 - _ -> unI64 0 -foldMapping :: Char# -> _ {- unboxed Int64 -} -{-# NOINLINE foldMapping #-} -foldMapping = \case - -- LATIN CAPITAL LETTER A - '\x0041'# -> unI64 97 - -- LATIN CAPITAL LETTER B - '\x0042'# -> unI64 98 - -- LATIN CAPITAL LETTER C - '\x0043'# -> unI64 99 - -- LATIN CAPITAL LETTER D - '\x0044'# -> unI64 100 - -- LATIN CAPITAL LETTER E - '\x0045'# -> unI64 101 - -- LATIN CAPITAL LETTER F - '\x0046'# -> unI64 102 - -- LATIN CAPITAL LETTER G - '\x0047'# -> unI64 103 - -- LATIN CAPITAL LETTER H - '\x0048'# -> unI64 104 - -- LATIN CAPITAL LETTER I - '\x0049'# -> unI64 105 - -- LATIN CAPITAL LETTER J - '\x004a'# -> unI64 106 - -- LATIN CAPITAL LETTER K - '\x004b'# -> unI64 107 - -- LATIN CAPITAL LETTER L - '\x004c'# -> unI64 108 - -- LATIN CAPITAL LETTER M - '\x004d'# -> unI64 109 - -- LATIN CAPITAL LETTER N - '\x004e'# -> unI64 110 - -- LATIN CAPITAL LETTER O - '\x004f'# -> unI64 111 - -- LATIN CAPITAL LETTER P - '\x0050'# -> unI64 112 - -- LATIN CAPITAL LETTER Q - '\x0051'# -> unI64 113 - -- LATIN CAPITAL LETTER R - '\x0052'# -> unI64 114 - -- LATIN CAPITAL LETTER S - '\x0053'# -> unI64 115 - -- LATIN CAPITAL LETTER T - '\x0054'# -> unI64 116 - -- LATIN CAPITAL LETTER U - '\x0055'# -> unI64 117 - -- LATIN CAPITAL LETTER V - '\x0056'# -> unI64 118 - -- LATIN CAPITAL LETTER W - '\x0057'# -> unI64 119 - -- LATIN CAPITAL LETTER X - '\x0058'# -> unI64 120 - -- LATIN CAPITAL LETTER Y - '\x0059'# -> unI64 121 - -- LATIN CAPITAL LETTER Z - '\x005a'# -> unI64 122 - -- MICRO SIGN - '\x00b5'# -> unI64 956 - -- LATIN CAPITAL LETTER A WITH GRAVE - '\x00c0'# -> unI64 224 - -- LATIN CAPITAL LETTER A WITH ACUTE - '\x00c1'# -> unI64 225 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX - '\x00c2'# -> unI64 226 - -- LATIN CAPITAL LETTER A WITH TILDE - '\x00c3'# -> unI64 227 - -- LATIN CAPITAL LETTER A WITH DIAERESIS - '\x00c4'# -> unI64 228 - -- LATIN CAPITAL LETTER A WITH RING ABOVE - '\x00c5'# -> unI64 229 - -- LATIN CAPITAL LETTER AE - '\x00c6'# -> unI64 230 - -- LATIN CAPITAL LETTER C WITH CEDILLA - '\x00c7'# -> unI64 231 - -- LATIN CAPITAL LETTER E WITH GRAVE - '\x00c8'# -> unI64 232 - -- LATIN CAPITAL LETTER E WITH ACUTE - '\x00c9'# -> unI64 233 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX - '\x00ca'# -> unI64 234 - -- LATIN CAPITAL LETTER E WITH DIAERESIS - '\x00cb'# -> unI64 235 - -- LATIN CAPITAL LETTER I WITH GRAVE - '\x00cc'# -> unI64 236 - -- LATIN CAPITAL LETTER I WITH ACUTE - '\x00cd'# -> unI64 237 - -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX - '\x00ce'# -> unI64 238 - -- LATIN CAPITAL LETTER I WITH DIAERESIS - '\x00cf'# -> unI64 239 - -- LATIN CAPITAL LETTER ETH - '\x00d0'# -> unI64 240 - -- LATIN CAPITAL LETTER N WITH TILDE - '\x00d1'# -> unI64 241 - -- LATIN CAPITAL LETTER O WITH GRAVE - '\x00d2'# -> unI64 242 - -- LATIN CAPITAL LETTER O WITH ACUTE - '\x00d3'# -> unI64 243 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\x00d4'# -> unI64 244 - -- LATIN CAPITAL LETTER O WITH TILDE - '\x00d5'# -> unI64 245 - -- LATIN CAPITAL LETTER O WITH DIAERESIS - '\x00d6'# -> unI64 246 - -- LATIN CAPITAL LETTER O WITH STROKE - '\x00d8'# -> unI64 248 - -- LATIN CAPITAL LETTER U WITH GRAVE - '\x00d9'# -> unI64 249 - -- LATIN CAPITAL LETTER U WITH ACUTE - '\x00da'# -> unI64 250 - -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX - '\x00db'# -> unI64 251 - -- LATIN CAPITAL LETTER U WITH DIAERESIS - '\x00dc'# -> unI64 252 - -- LATIN CAPITAL LETTER Y WITH ACUTE - '\x00dd'# -> unI64 253 - -- LATIN CAPITAL LETTER THORN - '\x00de'# -> unI64 254 - -- LATIN SMALL LETTER SHARP S - '\x00df'# -> unI64 241172595 - -- LATIN CAPITAL LETTER A WITH MACRON - '\x0100'# -> unI64 257 - -- LATIN CAPITAL LETTER A WITH BREVE - '\x0102'# -> unI64 259 - -- LATIN CAPITAL LETTER A WITH OGONEK - '\x0104'# -> unI64 261 - -- LATIN CAPITAL LETTER C WITH ACUTE - '\x0106'# -> unI64 263 - -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX - '\x0108'# -> unI64 265 - -- LATIN CAPITAL LETTER C WITH DOT ABOVE - '\x010a'# -> unI64 267 - -- LATIN CAPITAL LETTER C WITH CARON - '\x010c'# -> unI64 269 - -- LATIN CAPITAL LETTER D WITH CARON - '\x010e'# -> unI64 271 - -- LATIN CAPITAL LETTER D WITH STROKE - '\x0110'# -> unI64 273 - -- LATIN CAPITAL LETTER E WITH MACRON - '\x0112'# -> unI64 275 - -- LATIN CAPITAL LETTER E WITH BREVE - '\x0114'# -> unI64 277 - -- LATIN CAPITAL LETTER E WITH DOT ABOVE - '\x0116'# -> unI64 279 - -- LATIN CAPITAL LETTER E WITH OGONEK - '\x0118'# -> unI64 281 - -- LATIN CAPITAL LETTER E WITH CARON - '\x011a'# -> unI64 283 - -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX - '\x011c'# -> unI64 285 - -- LATIN CAPITAL LETTER G WITH BREVE - '\x011e'# -> unI64 287 - -- LATIN CAPITAL LETTER G WITH DOT ABOVE - '\x0120'# -> unI64 289 - -- LATIN CAPITAL LETTER G WITH CEDILLA - '\x0122'# -> unI64 291 - -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX - '\x0124'# -> unI64 293 - -- LATIN CAPITAL LETTER H WITH STROKE - '\x0126'# -> unI64 295 - -- LATIN CAPITAL LETTER I WITH TILDE - '\x0128'# -> unI64 297 - -- LATIN CAPITAL LETTER I WITH MACRON - '\x012a'# -> unI64 299 - -- LATIN CAPITAL LETTER I WITH BREVE - '\x012c'# -> unI64 301 - -- LATIN CAPITAL LETTER I WITH OGONEK - '\x012e'# -> unI64 303 - -- LATIN CAPITAL LETTER I WITH DOT ABOVE - '\x0130'# -> unI64 1625292905 - -- LATIN CAPITAL LIGATURE IJ - '\x0132'# -> unI64 307 - -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX - '\x0134'# -> unI64 309 - -- LATIN CAPITAL LETTER K WITH CEDILLA - '\x0136'# -> unI64 311 - -- LATIN CAPITAL LETTER L WITH ACUTE - '\x0139'# -> unI64 314 - -- LATIN CAPITAL LETTER L WITH CEDILLA - '\x013b'# -> unI64 316 - -- LATIN CAPITAL LETTER L WITH CARON - '\x013d'# -> unI64 318 - -- LATIN CAPITAL LETTER L WITH MIDDLE DOT - '\x013f'# -> unI64 320 - -- LATIN CAPITAL LETTER L WITH STROKE - '\x0141'# -> unI64 322 - -- LATIN CAPITAL LETTER N WITH ACUTE - '\x0143'# -> unI64 324 - -- LATIN CAPITAL LETTER N WITH CEDILLA - '\x0145'# -> unI64 326 - -- LATIN CAPITAL LETTER N WITH CARON - '\x0147'# -> unI64 328 - -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE - '\x0149'# -> unI64 230687420 - -- LATIN CAPITAL LETTER ENG - '\x014a'# -> unI64 331 - -- LATIN CAPITAL LETTER O WITH MACRON - '\x014c'# -> unI64 333 - -- LATIN CAPITAL LETTER O WITH BREVE - '\x014e'# -> unI64 335 - -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE - '\x0150'# -> unI64 337 - -- LATIN CAPITAL LIGATURE OE - '\x0152'# -> unI64 339 - -- LATIN CAPITAL LETTER R WITH ACUTE - '\x0154'# -> unI64 341 - -- LATIN CAPITAL LETTER R WITH CEDILLA - '\x0156'# -> unI64 343 - -- LATIN CAPITAL LETTER R WITH CARON - '\x0158'# -> unI64 345 - -- LATIN CAPITAL LETTER S WITH ACUTE - '\x015a'# -> unI64 347 - -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX - '\x015c'# -> unI64 349 - -- LATIN CAPITAL LETTER S WITH CEDILLA - '\x015e'# -> unI64 351 - -- LATIN CAPITAL LETTER S WITH CARON - '\x0160'# -> unI64 353 - -- LATIN CAPITAL LETTER T WITH CEDILLA - '\x0162'# -> unI64 355 - -- LATIN CAPITAL LETTER T WITH CARON - '\x0164'# -> unI64 357 - -- LATIN CAPITAL LETTER T WITH STROKE - '\x0166'# -> unI64 359 - -- LATIN CAPITAL LETTER U WITH TILDE - '\x0168'# -> unI64 361 - -- LATIN CAPITAL LETTER U WITH MACRON - '\x016a'# -> unI64 363 - -- LATIN CAPITAL LETTER U WITH BREVE - '\x016c'# -> unI64 365 - -- LATIN CAPITAL LETTER U WITH RING ABOVE - '\x016e'# -> unI64 367 - -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE - '\x0170'# -> unI64 369 - -- LATIN CAPITAL LETTER U WITH OGONEK - '\x0172'# -> unI64 371 - -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX - '\x0174'# -> unI64 373 - -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX - '\x0176'# -> unI64 375 - -- LATIN CAPITAL LETTER Y WITH DIAERESIS - '\x0178'# -> unI64 255 - -- LATIN CAPITAL LETTER Z WITH ACUTE - '\x0179'# -> unI64 378 - -- LATIN CAPITAL LETTER Z WITH DOT ABOVE - '\x017b'# -> unI64 380 - -- LATIN CAPITAL LETTER Z WITH CARON - '\x017d'# -> unI64 382 - -- LATIN SMALL LETTER LONG S - '\x017f'# -> unI64 115 - -- LATIN CAPITAL LETTER B WITH HOOK - '\x0181'# -> unI64 595 - -- LATIN CAPITAL LETTER B WITH TOPBAR - '\x0182'# -> unI64 387 - -- LATIN CAPITAL LETTER TONE SIX - '\x0184'# -> unI64 389 - -- LATIN CAPITAL LETTER OPEN O - '\x0186'# -> unI64 596 - -- LATIN CAPITAL LETTER C WITH HOOK - '\x0187'# -> unI64 392 - -- LATIN CAPITAL LETTER AFRICAN D - '\x0189'# -> unI64 598 - -- LATIN CAPITAL LETTER D WITH HOOK - '\x018a'# -> unI64 599 - -- LATIN CAPITAL LETTER D WITH TOPBAR - '\x018b'# -> unI64 396 - -- LATIN CAPITAL LETTER REVERSED E - '\x018e'# -> unI64 477 - -- LATIN CAPITAL LETTER SCHWA - '\x018f'# -> unI64 601 - -- LATIN CAPITAL LETTER OPEN E - '\x0190'# -> unI64 603 - -- LATIN CAPITAL LETTER F WITH HOOK - '\x0191'# -> unI64 402 - -- LATIN CAPITAL LETTER G WITH HOOK - '\x0193'# -> unI64 608 - -- LATIN CAPITAL LETTER GAMMA - '\x0194'# -> unI64 611 - -- LATIN CAPITAL LETTER IOTA - '\x0196'# -> unI64 617 - -- LATIN CAPITAL LETTER I WITH STROKE - '\x0197'# -> unI64 616 - -- LATIN CAPITAL LETTER K WITH HOOK - '\x0198'# -> unI64 409 - -- LATIN CAPITAL LETTER TURNED M - '\x019c'# -> unI64 623 - -- LATIN CAPITAL LETTER N WITH LEFT HOOK - '\x019d'# -> unI64 626 - -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE - '\x019f'# -> unI64 629 - -- LATIN CAPITAL LETTER O WITH HORN - '\x01a0'# -> unI64 417 - -- LATIN CAPITAL LETTER OI - '\x01a2'# -> unI64 419 - -- LATIN CAPITAL LETTER P WITH HOOK - '\x01a4'# -> unI64 421 - -- LATIN LETTER YR - '\x01a6'# -> unI64 640 - -- LATIN CAPITAL LETTER TONE TWO - '\x01a7'# -> unI64 424 - -- LATIN CAPITAL LETTER ESH - '\x01a9'# -> unI64 643 - -- LATIN CAPITAL LETTER T WITH HOOK - '\x01ac'# -> unI64 429 - -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK - '\x01ae'# -> unI64 648 - -- LATIN CAPITAL LETTER U WITH HORN - '\x01af'# -> unI64 432 - -- LATIN CAPITAL LETTER UPSILON - '\x01b1'# -> unI64 650 - -- LATIN CAPITAL LETTER V WITH HOOK - '\x01b2'# -> unI64 651 - -- LATIN CAPITAL LETTER Y WITH HOOK - '\x01b3'# -> unI64 436 - -- LATIN CAPITAL LETTER Z WITH STROKE - '\x01b5'# -> unI64 438 - -- LATIN CAPITAL LETTER EZH - '\x01b7'# -> unI64 658 - -- LATIN CAPITAL LETTER EZH REVERSED - '\x01b8'# -> unI64 441 - -- LATIN CAPITAL LETTER TONE FIVE - '\x01bc'# -> unI64 445 - -- LATIN CAPITAL LETTER DZ WITH CARON - '\x01c4'# -> unI64 454 - -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON - '\x01c5'# -> unI64 454 - -- LATIN CAPITAL LETTER LJ - '\x01c7'# -> unI64 457 - -- LATIN CAPITAL LETTER L WITH SMALL LETTER J - '\x01c8'# -> unI64 457 - -- LATIN CAPITAL LETTER NJ - '\x01ca'# -> unI64 460 - -- LATIN CAPITAL LETTER N WITH SMALL LETTER J - '\x01cb'# -> unI64 460 - -- LATIN CAPITAL LETTER A WITH CARON - '\x01cd'# -> unI64 462 - -- LATIN CAPITAL LETTER I WITH CARON - '\x01cf'# -> unI64 464 - -- LATIN CAPITAL LETTER O WITH CARON - '\x01d1'# -> unI64 466 - -- LATIN CAPITAL LETTER U WITH CARON - '\x01d3'# -> unI64 468 - -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON - '\x01d5'# -> unI64 470 - -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE - '\x01d7'# -> unI64 472 - -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON - '\x01d9'# -> unI64 474 - -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE - '\x01db'# -> unI64 476 - -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON - '\x01de'# -> unI64 479 - -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON - '\x01e0'# -> unI64 481 - -- LATIN CAPITAL LETTER AE WITH MACRON - '\x01e2'# -> unI64 483 - -- LATIN CAPITAL LETTER G WITH STROKE - '\x01e4'# -> unI64 485 - -- LATIN CAPITAL LETTER G WITH CARON - '\x01e6'# -> unI64 487 - -- LATIN CAPITAL LETTER K WITH CARON - '\x01e8'# -> unI64 489 - -- LATIN CAPITAL LETTER O WITH OGONEK - '\x01ea'# -> unI64 491 - -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON - '\x01ec'# -> unI64 493 - -- LATIN CAPITAL LETTER EZH WITH CARON - '\x01ee'# -> unI64 495 - -- LATIN SMALL LETTER J WITH CARON - '\x01f0'# -> unI64 1635778666 - -- LATIN CAPITAL LETTER DZ - '\x01f1'# -> unI64 499 - -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z - '\x01f2'# -> unI64 499 - -- LATIN CAPITAL LETTER G WITH ACUTE - '\x01f4'# -> unI64 501 - -- LATIN CAPITAL LETTER HWAIR - '\x01f6'# -> unI64 405 - -- LATIN CAPITAL LETTER WYNN - '\x01f7'# -> unI64 447 - -- LATIN CAPITAL LETTER N WITH GRAVE - '\x01f8'# -> unI64 505 - -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE - '\x01fa'# -> unI64 507 - -- LATIN CAPITAL LETTER AE WITH ACUTE - '\x01fc'# -> unI64 509 - -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE - '\x01fe'# -> unI64 511 - -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE - '\x0200'# -> unI64 513 - -- LATIN CAPITAL LETTER A WITH INVERTED BREVE - '\x0202'# -> unI64 515 - -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE - '\x0204'# -> unI64 517 - -- LATIN CAPITAL LETTER E WITH INVERTED BREVE - '\x0206'# -> unI64 519 - -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE - '\x0208'# -> unI64 521 - -- LATIN CAPITAL LETTER I WITH INVERTED BREVE - '\x020a'# -> unI64 523 - -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE - '\x020c'# -> unI64 525 - -- LATIN CAPITAL LETTER O WITH INVERTED BREVE - '\x020e'# -> unI64 527 - -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE - '\x0210'# -> unI64 529 - -- LATIN CAPITAL LETTER R WITH INVERTED BREVE - '\x0212'# -> unI64 531 - -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE - '\x0214'# -> unI64 533 - -- LATIN CAPITAL LETTER U WITH INVERTED BREVE - '\x0216'# -> unI64 535 - -- LATIN CAPITAL LETTER S WITH COMMA BELOW - '\x0218'# -> unI64 537 - -- LATIN CAPITAL LETTER T WITH COMMA BELOW - '\x021a'# -> unI64 539 - -- LATIN CAPITAL LETTER YOGH - '\x021c'# -> unI64 541 - -- LATIN CAPITAL LETTER H WITH CARON - '\x021e'# -> unI64 543 - -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG - '\x0220'# -> unI64 414 - -- LATIN CAPITAL LETTER OU - '\x0222'# -> unI64 547 - -- LATIN CAPITAL LETTER Z WITH HOOK - '\x0224'# -> unI64 549 - -- LATIN CAPITAL LETTER A WITH DOT ABOVE - '\x0226'# -> unI64 551 - -- LATIN CAPITAL LETTER E WITH CEDILLA - '\x0228'# -> unI64 553 - -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON - '\x022a'# -> unI64 555 - -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON - '\x022c'# -> unI64 557 - -- LATIN CAPITAL LETTER O WITH DOT ABOVE - '\x022e'# -> unI64 559 - -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON - '\x0230'# -> unI64 561 - -- LATIN CAPITAL LETTER Y WITH MACRON - '\x0232'# -> unI64 563 - -- LATIN CAPITAL LETTER A WITH STROKE - '\x023a'# -> unI64 11365 - -- LATIN CAPITAL LETTER C WITH STROKE - '\x023b'# -> unI64 572 - -- LATIN CAPITAL LETTER L WITH BAR - '\x023d'# -> unI64 410 - -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE - '\x023e'# -> unI64 11366 - -- LATIN CAPITAL LETTER GLOTTAL STOP - '\x0241'# -> unI64 578 - -- LATIN CAPITAL LETTER B WITH STROKE - '\x0243'# -> unI64 384 - -- LATIN CAPITAL LETTER U BAR - '\x0244'# -> unI64 649 - -- LATIN CAPITAL LETTER TURNED V - '\x0245'# -> unI64 652 - -- LATIN CAPITAL LETTER E WITH STROKE - '\x0246'# -> unI64 583 - -- LATIN CAPITAL LETTER J WITH STROKE - '\x0248'# -> unI64 585 - -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL - '\x024a'# -> unI64 587 - -- LATIN CAPITAL LETTER R WITH STROKE - '\x024c'# -> unI64 589 - -- LATIN CAPITAL LETTER Y WITH STROKE - '\x024e'# -> unI64 591 - -- COMBINING GREEK YPOGEGRAMMENI - '\x0345'# -> unI64 953 - -- GREEK CAPITAL LETTER HETA - '\x0370'# -> unI64 881 - -- GREEK CAPITAL LETTER ARCHAIC SAMPI - '\x0372'# -> unI64 883 - -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA - '\x0376'# -> unI64 887 - -- GREEK CAPITAL LETTER YOT - '\x037f'# -> unI64 1011 - -- GREEK CAPITAL LETTER ALPHA WITH TONOS - '\x0386'# -> unI64 940 - -- GREEK CAPITAL LETTER EPSILON WITH TONOS - '\x0388'# -> unI64 941 - -- GREEK CAPITAL LETTER ETA WITH TONOS - '\x0389'# -> unI64 942 - -- GREEK CAPITAL LETTER IOTA WITH TONOS - '\x038a'# -> unI64 943 - -- GREEK CAPITAL LETTER OMICRON WITH TONOS - '\x038c'# -> unI64 972 - -- GREEK CAPITAL LETTER UPSILON WITH TONOS - '\x038e'# -> unI64 973 - -- GREEK CAPITAL LETTER OMEGA WITH TONOS - '\x038f'# -> unI64 974 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS - '\x0390'# -> unI64 3382099394429881 - -- GREEK CAPITAL LETTER ALPHA - '\x0391'# -> unI64 945 - -- GREEK CAPITAL LETTER BETA - '\x0392'# -> unI64 946 - -- GREEK CAPITAL LETTER GAMMA - '\x0393'# -> unI64 947 - -- GREEK CAPITAL LETTER DELTA - '\x0394'# -> unI64 948 - -- GREEK CAPITAL LETTER EPSILON - '\x0395'# -> unI64 949 - -- GREEK CAPITAL LETTER ZETA - '\x0396'# -> unI64 950 - -- GREEK CAPITAL LETTER ETA - '\x0397'# -> unI64 951 - -- GREEK CAPITAL LETTER THETA - '\x0398'# -> unI64 952 - -- GREEK CAPITAL LETTER IOTA - '\x0399'# -> unI64 953 - -- GREEK CAPITAL LETTER KAPPA - '\x039a'# -> unI64 954 - -- GREEK CAPITAL LETTER LAMDA - '\x039b'# -> unI64 955 - -- GREEK CAPITAL LETTER MU - '\x039c'# -> unI64 956 - -- GREEK CAPITAL LETTER NU - '\x039d'# -> unI64 957 - -- GREEK CAPITAL LETTER XI - '\x039e'# -> unI64 958 - -- GREEK CAPITAL LETTER OMICRON - '\x039f'# -> unI64 959 - -- GREEK CAPITAL LETTER PI - '\x03a0'# -> unI64 960 - -- GREEK CAPITAL LETTER RHO - '\x03a1'# -> unI64 961 - -- GREEK CAPITAL LETTER SIGMA - '\x03a3'# -> unI64 963 - -- GREEK CAPITAL LETTER TAU - '\x03a4'# -> unI64 964 - -- GREEK CAPITAL LETTER UPSILON - '\x03a5'# -> unI64 965 - -- GREEK CAPITAL LETTER PHI - '\x03a6'# -> unI64 966 - -- GREEK CAPITAL LETTER CHI - '\x03a7'# -> unI64 967 - -- GREEK CAPITAL LETTER PSI - '\x03a8'# -> unI64 968 - -- GREEK CAPITAL LETTER OMEGA - '\x03a9'# -> unI64 969 - -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA - '\x03aa'# -> unI64 970 - -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA - '\x03ab'# -> unI64 971 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS - '\x03b0'# -> unI64 3382099394429893 - -- GREEK SMALL LETTER FINAL SIGMA - '\x03c2'# -> unI64 963 - -- GREEK CAPITAL KAI SYMBOL - '\x03cf'# -> unI64 983 - -- GREEK BETA SYMBOL - '\x03d0'# -> unI64 946 - -- GREEK THETA SYMBOL - '\x03d1'# -> unI64 952 - -- GREEK PHI SYMBOL - '\x03d5'# -> unI64 966 - -- GREEK PI SYMBOL - '\x03d6'# -> unI64 960 - -- GREEK LETTER ARCHAIC KOPPA - '\x03d8'# -> unI64 985 - -- GREEK LETTER STIGMA - '\x03da'# -> unI64 987 - -- GREEK LETTER DIGAMMA - '\x03dc'# -> unI64 989 - -- GREEK LETTER KOPPA - '\x03de'# -> unI64 991 - -- GREEK LETTER SAMPI - '\x03e0'# -> unI64 993 - -- COPTIC CAPITAL LETTER SHEI - '\x03e2'# -> unI64 995 - -- COPTIC CAPITAL LETTER FEI - '\x03e4'# -> unI64 997 - -- COPTIC CAPITAL LETTER KHEI - '\x03e6'# -> unI64 999 - -- COPTIC CAPITAL LETTER HORI - '\x03e8'# -> unI64 1001 - -- COPTIC CAPITAL LETTER GANGIA - '\x03ea'# -> unI64 1003 - -- COPTIC CAPITAL LETTER SHIMA - '\x03ec'# -> unI64 1005 - -- COPTIC CAPITAL LETTER DEI - '\x03ee'# -> unI64 1007 - -- GREEK KAPPA SYMBOL - '\x03f0'# -> unI64 954 - -- GREEK RHO SYMBOL - '\x03f1'# -> unI64 961 - -- GREEK CAPITAL THETA SYMBOL - '\x03f4'# -> unI64 952 - -- GREEK LUNATE EPSILON SYMBOL - '\x03f5'# -> unI64 949 - -- GREEK CAPITAL LETTER SHO - '\x03f7'# -> unI64 1016 - -- GREEK CAPITAL LUNATE SIGMA SYMBOL - '\x03f9'# -> unI64 1010 - -- GREEK CAPITAL LETTER SAN - '\x03fa'# -> unI64 1019 - -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL - '\x03fd'# -> unI64 891 - -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL - '\x03fe'# -> unI64 892 - -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL - '\x03ff'# -> unI64 893 - -- CYRILLIC CAPITAL LETTER IE WITH GRAVE - '\x0400'# -> unI64 1104 - -- CYRILLIC CAPITAL LETTER IO - '\x0401'# -> unI64 1105 - -- CYRILLIC CAPITAL LETTER DJE - '\x0402'# -> unI64 1106 - -- CYRILLIC CAPITAL LETTER GJE - '\x0403'# -> unI64 1107 - -- CYRILLIC CAPITAL LETTER UKRAINIAN IE - '\x0404'# -> unI64 1108 - -- CYRILLIC CAPITAL LETTER DZE - '\x0405'# -> unI64 1109 - -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - '\x0406'# -> unI64 1110 - -- CYRILLIC CAPITAL LETTER YI - '\x0407'# -> unI64 1111 - -- CYRILLIC CAPITAL LETTER JE - '\x0408'# -> unI64 1112 - -- CYRILLIC CAPITAL LETTER LJE - '\x0409'# -> unI64 1113 - -- CYRILLIC CAPITAL LETTER NJE - '\x040a'# -> unI64 1114 - -- CYRILLIC CAPITAL LETTER TSHE - '\x040b'# -> unI64 1115 - -- CYRILLIC CAPITAL LETTER KJE - '\x040c'# -> unI64 1116 - -- CYRILLIC CAPITAL LETTER I WITH GRAVE - '\x040d'# -> unI64 1117 - -- CYRILLIC CAPITAL LETTER SHORT U - '\x040e'# -> unI64 1118 - -- CYRILLIC CAPITAL LETTER DZHE - '\x040f'# -> unI64 1119 - -- CYRILLIC CAPITAL LETTER A - '\x0410'# -> unI64 1072 - -- CYRILLIC CAPITAL LETTER BE - '\x0411'# -> unI64 1073 - -- CYRILLIC CAPITAL LETTER VE - '\x0412'# -> unI64 1074 - -- CYRILLIC CAPITAL LETTER GHE - '\x0413'# -> unI64 1075 - -- CYRILLIC CAPITAL LETTER DE - '\x0414'# -> unI64 1076 - -- CYRILLIC CAPITAL LETTER IE - '\x0415'# -> unI64 1077 - -- CYRILLIC CAPITAL LETTER ZHE - '\x0416'# -> unI64 1078 - -- CYRILLIC CAPITAL LETTER ZE - '\x0417'# -> unI64 1079 - -- CYRILLIC CAPITAL LETTER I - '\x0418'# -> unI64 1080 - -- CYRILLIC CAPITAL LETTER SHORT I - '\x0419'# -> unI64 1081 - -- CYRILLIC CAPITAL LETTER KA - '\x041a'# -> unI64 1082 - -- CYRILLIC CAPITAL LETTER EL - '\x041b'# -> unI64 1083 - -- CYRILLIC CAPITAL LETTER EM - '\x041c'# -> unI64 1084 - -- CYRILLIC CAPITAL LETTER EN - '\x041d'# -> unI64 1085 - -- CYRILLIC CAPITAL LETTER O - '\x041e'# -> unI64 1086 - -- CYRILLIC CAPITAL LETTER PE - '\x041f'# -> unI64 1087 - -- CYRILLIC CAPITAL LETTER ER - '\x0420'# -> unI64 1088 - -- CYRILLIC CAPITAL LETTER ES - '\x0421'# -> unI64 1089 - -- CYRILLIC CAPITAL LETTER TE - '\x0422'# -> unI64 1090 - -- CYRILLIC CAPITAL LETTER U - '\x0423'# -> unI64 1091 - -- CYRILLIC CAPITAL LETTER EF - '\x0424'# -> unI64 1092 - -- CYRILLIC CAPITAL LETTER HA - '\x0425'# -> unI64 1093 - -- CYRILLIC CAPITAL LETTER TSE - '\x0426'# -> unI64 1094 - -- CYRILLIC CAPITAL LETTER CHE - '\x0427'# -> unI64 1095 - -- CYRILLIC CAPITAL LETTER SHA - '\x0428'# -> unI64 1096 - -- CYRILLIC CAPITAL LETTER SHCHA - '\x0429'# -> unI64 1097 - -- CYRILLIC CAPITAL LETTER HARD SIGN - '\x042a'# -> unI64 1098 - -- CYRILLIC CAPITAL LETTER YERU - '\x042b'# -> unI64 1099 - -- CYRILLIC CAPITAL LETTER SOFT SIGN - '\x042c'# -> unI64 1100 - -- CYRILLIC CAPITAL LETTER E - '\x042d'# -> unI64 1101 - -- CYRILLIC CAPITAL LETTER YU - '\x042e'# -> unI64 1102 - -- CYRILLIC CAPITAL LETTER YA - '\x042f'# -> unI64 1103 - -- CYRILLIC CAPITAL LETTER OMEGA - '\x0460'# -> unI64 1121 - -- CYRILLIC CAPITAL LETTER YAT - '\x0462'# -> unI64 1123 - -- CYRILLIC CAPITAL LETTER IOTIFIED E - '\x0464'# -> unI64 1125 - -- CYRILLIC CAPITAL LETTER LITTLE YUS - '\x0466'# -> unI64 1127 - -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS - '\x0468'# -> unI64 1129 - -- CYRILLIC CAPITAL LETTER BIG YUS - '\x046a'# -> unI64 1131 - -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS - '\x046c'# -> unI64 1133 - -- CYRILLIC CAPITAL LETTER KSI - '\x046e'# -> unI64 1135 - -- CYRILLIC CAPITAL LETTER PSI - '\x0470'# -> unI64 1137 - -- CYRILLIC CAPITAL LETTER FITA - '\x0472'# -> unI64 1139 - -- CYRILLIC CAPITAL LETTER IZHITSA - '\x0474'# -> unI64 1141 - -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT - '\x0476'# -> unI64 1143 - -- CYRILLIC CAPITAL LETTER UK - '\x0478'# -> unI64 1145 - -- CYRILLIC CAPITAL LETTER ROUND OMEGA - '\x047a'# -> unI64 1147 - -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO - '\x047c'# -> unI64 1149 - -- CYRILLIC CAPITAL LETTER OT - '\x047e'# -> unI64 1151 - -- CYRILLIC CAPITAL LETTER KOPPA - '\x0480'# -> unI64 1153 - -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL - '\x048a'# -> unI64 1163 - -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN - '\x048c'# -> unI64 1165 - -- CYRILLIC CAPITAL LETTER ER WITH TICK - '\x048e'# -> unI64 1167 - -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN - '\x0490'# -> unI64 1169 - -- CYRILLIC CAPITAL LETTER GHE WITH STROKE - '\x0492'# -> unI64 1171 - -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK - '\x0494'# -> unI64 1173 - -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER - '\x0496'# -> unI64 1175 - -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER - '\x0498'# -> unI64 1177 - -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER - '\x049a'# -> unI64 1179 - -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE - '\x049c'# -> unI64 1181 - -- CYRILLIC CAPITAL LETTER KA WITH STROKE - '\x049e'# -> unI64 1183 - -- CYRILLIC CAPITAL LETTER BASHKIR KA - '\x04a0'# -> unI64 1185 - -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER - '\x04a2'# -> unI64 1187 - -- CYRILLIC CAPITAL LIGATURE EN GHE - '\x04a4'# -> unI64 1189 - -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK - '\x04a6'# -> unI64 1191 - -- CYRILLIC CAPITAL LETTER ABKHASIAN HA - '\x04a8'# -> unI64 1193 - -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER - '\x04aa'# -> unI64 1195 - -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER - '\x04ac'# -> unI64 1197 - -- CYRILLIC CAPITAL LETTER STRAIGHT U - '\x04ae'# -> unI64 1199 - -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE - '\x04b0'# -> unI64 1201 - -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER - '\x04b2'# -> unI64 1203 - -- CYRILLIC CAPITAL LIGATURE TE TSE - '\x04b4'# -> unI64 1205 - -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER - '\x04b6'# -> unI64 1207 - -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE - '\x04b8'# -> unI64 1209 - -- CYRILLIC CAPITAL LETTER SHHA - '\x04ba'# -> unI64 1211 - -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE - '\x04bc'# -> unI64 1213 - -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER - '\x04be'# -> unI64 1215 - -- CYRILLIC LETTER PALOCHKA - '\x04c0'# -> unI64 1231 - -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE - '\x04c1'# -> unI64 1218 - -- CYRILLIC CAPITAL LETTER KA WITH HOOK - '\x04c3'# -> unI64 1220 - -- CYRILLIC CAPITAL LETTER EL WITH TAIL - '\x04c5'# -> unI64 1222 - -- CYRILLIC CAPITAL LETTER EN WITH HOOK - '\x04c7'# -> unI64 1224 - -- CYRILLIC CAPITAL LETTER EN WITH TAIL - '\x04c9'# -> unI64 1226 - -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE - '\x04cb'# -> unI64 1228 - -- CYRILLIC CAPITAL LETTER EM WITH TAIL - '\x04cd'# -> unI64 1230 - -- CYRILLIC CAPITAL LETTER A WITH BREVE - '\x04d0'# -> unI64 1233 - -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS - '\x04d2'# -> unI64 1235 - -- CYRILLIC CAPITAL LIGATURE A IE - '\x04d4'# -> unI64 1237 - -- CYRILLIC CAPITAL LETTER IE WITH BREVE - '\x04d6'# -> unI64 1239 - -- CYRILLIC CAPITAL LETTER SCHWA - '\x04d8'# -> unI64 1241 - -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS - '\x04da'# -> unI64 1243 - -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS - '\x04dc'# -> unI64 1245 - -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS - '\x04de'# -> unI64 1247 - -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE - '\x04e0'# -> unI64 1249 - -- CYRILLIC CAPITAL LETTER I WITH MACRON - '\x04e2'# -> unI64 1251 - -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS - '\x04e4'# -> unI64 1253 - -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS - '\x04e6'# -> unI64 1255 - -- CYRILLIC CAPITAL LETTER BARRED O - '\x04e8'# -> unI64 1257 - -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS - '\x04ea'# -> unI64 1259 - -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS - '\x04ec'# -> unI64 1261 - -- CYRILLIC CAPITAL LETTER U WITH MACRON - '\x04ee'# -> unI64 1263 - -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS - '\x04f0'# -> unI64 1265 - -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE - '\x04f2'# -> unI64 1267 - -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS - '\x04f4'# -> unI64 1269 - -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER - '\x04f6'# -> unI64 1271 - -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS - '\x04f8'# -> unI64 1273 - -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK - '\x04fa'# -> unI64 1275 - -- CYRILLIC CAPITAL LETTER HA WITH HOOK - '\x04fc'# -> unI64 1277 - -- CYRILLIC CAPITAL LETTER HA WITH STROKE - '\x04fe'# -> unI64 1279 - -- CYRILLIC CAPITAL LETTER KOMI DE - '\x0500'# -> unI64 1281 - -- CYRILLIC CAPITAL LETTER KOMI DJE - '\x0502'# -> unI64 1283 - -- CYRILLIC CAPITAL LETTER KOMI ZJE - '\x0504'# -> unI64 1285 - -- CYRILLIC CAPITAL LETTER KOMI DZJE - '\x0506'# -> unI64 1287 - -- CYRILLIC CAPITAL LETTER KOMI LJE - '\x0508'# -> unI64 1289 - -- CYRILLIC CAPITAL LETTER KOMI NJE - '\x050a'# -> unI64 1291 - -- CYRILLIC CAPITAL LETTER KOMI SJE - '\x050c'# -> unI64 1293 - -- CYRILLIC CAPITAL LETTER KOMI TJE - '\x050e'# -> unI64 1295 - -- CYRILLIC CAPITAL LETTER REVERSED ZE - '\x0510'# -> unI64 1297 - -- CYRILLIC CAPITAL LETTER EL WITH HOOK - '\x0512'# -> unI64 1299 - -- CYRILLIC CAPITAL LETTER LHA - '\x0514'# -> unI64 1301 - -- CYRILLIC CAPITAL LETTER RHA - '\x0516'# -> unI64 1303 - -- CYRILLIC CAPITAL LETTER YAE - '\x0518'# -> unI64 1305 - -- CYRILLIC CAPITAL LETTER QA - '\x051a'# -> unI64 1307 - -- CYRILLIC CAPITAL LETTER WE - '\x051c'# -> unI64 1309 - -- CYRILLIC CAPITAL LETTER ALEUT KA - '\x051e'# -> unI64 1311 - -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK - '\x0520'# -> unI64 1313 - -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK - '\x0522'# -> unI64 1315 - -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER - '\x0524'# -> unI64 1317 - -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER - '\x0526'# -> unI64 1319 - -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK - '\x0528'# -> unI64 1321 - -- CYRILLIC CAPITAL LETTER DZZHE - '\x052a'# -> unI64 1323 - -- CYRILLIC CAPITAL LETTER DCHE - '\x052c'# -> unI64 1325 - -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER - '\x052e'# -> unI64 1327 - -- ARMENIAN CAPITAL LETTER AYB - '\x0531'# -> unI64 1377 - -- ARMENIAN CAPITAL LETTER BEN - '\x0532'# -> unI64 1378 - -- ARMENIAN CAPITAL LETTER GIM - '\x0533'# -> unI64 1379 - -- ARMENIAN CAPITAL LETTER DA - '\x0534'# -> unI64 1380 - -- ARMENIAN CAPITAL LETTER ECH - '\x0535'# -> unI64 1381 - -- ARMENIAN CAPITAL LETTER ZA - '\x0536'# -> unI64 1382 - -- ARMENIAN CAPITAL LETTER EH - '\x0537'# -> unI64 1383 - -- ARMENIAN CAPITAL LETTER ET - '\x0538'# -> unI64 1384 - -- ARMENIAN CAPITAL LETTER TO - '\x0539'# -> unI64 1385 - -- ARMENIAN CAPITAL LETTER ZHE - '\x053a'# -> unI64 1386 - -- ARMENIAN CAPITAL LETTER INI - '\x053b'# -> unI64 1387 - -- ARMENIAN CAPITAL LETTER LIWN - '\x053c'# -> unI64 1388 - -- ARMENIAN CAPITAL LETTER XEH - '\x053d'# -> unI64 1389 - -- ARMENIAN CAPITAL LETTER CA - '\x053e'# -> unI64 1390 - -- ARMENIAN CAPITAL LETTER KEN - '\x053f'# -> unI64 1391 - -- ARMENIAN CAPITAL LETTER HO - '\x0540'# -> unI64 1392 - -- ARMENIAN CAPITAL LETTER JA - '\x0541'# -> unI64 1393 - -- ARMENIAN CAPITAL LETTER GHAD - '\x0542'# -> unI64 1394 - -- ARMENIAN CAPITAL LETTER CHEH - '\x0543'# -> unI64 1395 - -- ARMENIAN CAPITAL LETTER MEN - '\x0544'# -> unI64 1396 - -- ARMENIAN CAPITAL LETTER YI - '\x0545'# -> unI64 1397 - -- ARMENIAN CAPITAL LETTER NOW - '\x0546'# -> unI64 1398 - -- ARMENIAN CAPITAL LETTER SHA - '\x0547'# -> unI64 1399 - -- ARMENIAN CAPITAL LETTER VO - '\x0548'# -> unI64 1400 - -- ARMENIAN CAPITAL LETTER CHA - '\x0549'# -> unI64 1401 - -- ARMENIAN CAPITAL LETTER PEH - '\x054a'# -> unI64 1402 - -- ARMENIAN CAPITAL LETTER JHEH - '\x054b'# -> unI64 1403 - -- ARMENIAN CAPITAL LETTER RA - '\x054c'# -> unI64 1404 - -- ARMENIAN CAPITAL LETTER SEH - '\x054d'# -> unI64 1405 - -- ARMENIAN CAPITAL LETTER VEW - '\x054e'# -> unI64 1406 - -- ARMENIAN CAPITAL LETTER TIWN - '\x054f'# -> unI64 1407 - -- ARMENIAN CAPITAL LETTER REH - '\x0550'# -> unI64 1408 - -- ARMENIAN CAPITAL LETTER CO - '\x0551'# -> unI64 1409 - -- ARMENIAN CAPITAL LETTER YIWN - '\x0552'# -> unI64 1410 - -- ARMENIAN CAPITAL LETTER PIWR - '\x0553'# -> unI64 1411 - -- ARMENIAN CAPITAL LETTER KEH - '\x0554'# -> unI64 1412 - -- ARMENIAN CAPITAL LETTER OH - '\x0555'# -> unI64 1413 - -- ARMENIAN CAPITAL LETTER FEH - '\x0556'# -> unI64 1414 - -- ARMENIAN SMALL LIGATURE ECH YIWN - '\x0587'# -> unI64 2956985701 - -- GEORGIAN CAPITAL LETTER AN - '\x10a0'# -> unI64 11520 - -- GEORGIAN CAPITAL LETTER BAN - '\x10a1'# -> unI64 11521 - -- GEORGIAN CAPITAL LETTER GAN - '\x10a2'# -> unI64 11522 - -- GEORGIAN CAPITAL LETTER DON - '\x10a3'# -> unI64 11523 - -- GEORGIAN CAPITAL LETTER EN - '\x10a4'# -> unI64 11524 - -- GEORGIAN CAPITAL LETTER VIN - '\x10a5'# -> unI64 11525 - -- GEORGIAN CAPITAL LETTER ZEN - '\x10a6'# -> unI64 11526 - -- GEORGIAN CAPITAL LETTER TAN - '\x10a7'# -> unI64 11527 - -- GEORGIAN CAPITAL LETTER IN - '\x10a8'# -> unI64 11528 - -- GEORGIAN CAPITAL LETTER KAN - '\x10a9'# -> unI64 11529 - -- GEORGIAN CAPITAL LETTER LAS - '\x10aa'# -> unI64 11530 - -- GEORGIAN CAPITAL LETTER MAN - '\x10ab'# -> unI64 11531 - -- GEORGIAN CAPITAL LETTER NAR - '\x10ac'# -> unI64 11532 - -- GEORGIAN CAPITAL LETTER ON - '\x10ad'# -> unI64 11533 - -- GEORGIAN CAPITAL LETTER PAR - '\x10ae'# -> unI64 11534 - -- GEORGIAN CAPITAL LETTER ZHAR - '\x10af'# -> unI64 11535 - -- GEORGIAN CAPITAL LETTER RAE - '\x10b0'# -> unI64 11536 - -- GEORGIAN CAPITAL LETTER SAN - '\x10b1'# -> unI64 11537 - -- GEORGIAN CAPITAL LETTER TAR - '\x10b2'# -> unI64 11538 - -- GEORGIAN CAPITAL LETTER UN - '\x10b3'# -> unI64 11539 - -- GEORGIAN CAPITAL LETTER PHAR - '\x10b4'# -> unI64 11540 - -- GEORGIAN CAPITAL LETTER KHAR - '\x10b5'# -> unI64 11541 - -- GEORGIAN CAPITAL LETTER GHAN - '\x10b6'# -> unI64 11542 - -- GEORGIAN CAPITAL LETTER QAR - '\x10b7'# -> unI64 11543 - -- GEORGIAN CAPITAL LETTER SHIN - '\x10b8'# -> unI64 11544 - -- GEORGIAN CAPITAL LETTER CHIN - '\x10b9'# -> unI64 11545 - -- GEORGIAN CAPITAL LETTER CAN - '\x10ba'# -> unI64 11546 - -- GEORGIAN CAPITAL LETTER JIL - '\x10bb'# -> unI64 11547 - -- GEORGIAN CAPITAL LETTER CIL - '\x10bc'# -> unI64 11548 - -- GEORGIAN CAPITAL LETTER CHAR - '\x10bd'# -> unI64 11549 - -- GEORGIAN CAPITAL LETTER XAN - '\x10be'# -> unI64 11550 - -- GEORGIAN CAPITAL LETTER JHAN - '\x10bf'# -> unI64 11551 - -- GEORGIAN CAPITAL LETTER HAE - '\x10c0'# -> unI64 11552 - -- GEORGIAN CAPITAL LETTER HE - '\x10c1'# -> unI64 11553 - -- GEORGIAN CAPITAL LETTER HIE - '\x10c2'# -> unI64 11554 - -- GEORGIAN CAPITAL LETTER WE - '\x10c3'# -> unI64 11555 - -- GEORGIAN CAPITAL LETTER HAR - '\x10c4'# -> unI64 11556 - -- GEORGIAN CAPITAL LETTER HOE - '\x10c5'# -> unI64 11557 - -- GEORGIAN CAPITAL LETTER YN - '\x10c7'# -> unI64 11559 - -- GEORGIAN CAPITAL LETTER AEN - '\x10cd'# -> unI64 11565 - -- CHEROKEE SMALL LETTER YE - '\x13f8'# -> unI64 5104 - -- CHEROKEE SMALL LETTER YI - '\x13f9'# -> unI64 5105 - -- CHEROKEE SMALL LETTER YO - '\x13fa'# -> unI64 5106 - -- CHEROKEE SMALL LETTER YU - '\x13fb'# -> unI64 5107 - -- CHEROKEE SMALL LETTER YV - '\x13fc'# -> unI64 5108 - -- CHEROKEE SMALL LETTER MV - '\x13fd'# -> unI64 5109 - -- CYRILLIC SMALL LETTER ROUNDED VE - '\x1c80'# -> unI64 1074 - -- CYRILLIC SMALL LETTER LONG-LEGGED DE - '\x1c81'# -> unI64 1076 - -- CYRILLIC SMALL LETTER NARROW O - '\x1c82'# -> unI64 1086 - -- CYRILLIC SMALL LETTER WIDE ES - '\x1c83'# -> unI64 1089 - -- CYRILLIC SMALL LETTER TALL TE - '\x1c84'# -> unI64 1090 - -- CYRILLIC SMALL LETTER THREE-LEGGED TE - '\x1c85'# -> unI64 1090 - -- CYRILLIC SMALL LETTER TALL HARD SIGN - '\x1c86'# -> unI64 1098 - -- CYRILLIC SMALL LETTER TALL YAT - '\x1c87'# -> unI64 1123 - -- CYRILLIC SMALL LETTER UNBLENDED UK - '\x1c88'# -> unI64 42571 - -- GEORGIAN MTAVRULI CAPITAL LETTER AN - '\x1c90'# -> unI64 4304 - -- GEORGIAN MTAVRULI CAPITAL LETTER BAN - '\x1c91'# -> unI64 4305 - -- GEORGIAN MTAVRULI CAPITAL LETTER GAN - '\x1c92'# -> unI64 4306 - -- GEORGIAN MTAVRULI CAPITAL LETTER DON - '\x1c93'# -> unI64 4307 - -- GEORGIAN MTAVRULI CAPITAL LETTER EN - '\x1c94'# -> unI64 4308 - -- GEORGIAN MTAVRULI CAPITAL LETTER VIN - '\x1c95'# -> unI64 4309 - -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN - '\x1c96'# -> unI64 4310 - -- GEORGIAN MTAVRULI CAPITAL LETTER TAN - '\x1c97'# -> unI64 4311 - -- GEORGIAN MTAVRULI CAPITAL LETTER IN - '\x1c98'# -> unI64 4312 - -- GEORGIAN MTAVRULI CAPITAL LETTER KAN - '\x1c99'# -> unI64 4313 - -- GEORGIAN MTAVRULI CAPITAL LETTER LAS - '\x1c9a'# -> unI64 4314 - -- GEORGIAN MTAVRULI CAPITAL LETTER MAN - '\x1c9b'# -> unI64 4315 - -- GEORGIAN MTAVRULI CAPITAL LETTER NAR - '\x1c9c'# -> unI64 4316 - -- GEORGIAN MTAVRULI CAPITAL LETTER ON - '\x1c9d'# -> unI64 4317 - -- GEORGIAN MTAVRULI CAPITAL LETTER PAR - '\x1c9e'# -> unI64 4318 - -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR - '\x1c9f'# -> unI64 4319 - -- GEORGIAN MTAVRULI CAPITAL LETTER RAE - '\x1ca0'# -> unI64 4320 - -- GEORGIAN MTAVRULI CAPITAL LETTER SAN - '\x1ca1'# -> unI64 4321 - -- GEORGIAN MTAVRULI CAPITAL LETTER TAR - '\x1ca2'# -> unI64 4322 - -- GEORGIAN MTAVRULI CAPITAL LETTER UN - '\x1ca3'# -> unI64 4323 - -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR - '\x1ca4'# -> unI64 4324 - -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR - '\x1ca5'# -> unI64 4325 - -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN - '\x1ca6'# -> unI64 4326 - -- GEORGIAN MTAVRULI CAPITAL LETTER QAR - '\x1ca7'# -> unI64 4327 - -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN - '\x1ca8'# -> unI64 4328 - -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN - '\x1ca9'# -> unI64 4329 - -- GEORGIAN MTAVRULI CAPITAL LETTER CAN - '\x1caa'# -> unI64 4330 - -- GEORGIAN MTAVRULI CAPITAL LETTER JIL - '\x1cab'# -> unI64 4331 - -- GEORGIAN MTAVRULI CAPITAL LETTER CIL - '\x1cac'# -> unI64 4332 - -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR - '\x1cad'# -> unI64 4333 - -- GEORGIAN MTAVRULI CAPITAL LETTER XAN - '\x1cae'# -> unI64 4334 - -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN - '\x1caf'# -> unI64 4335 - -- GEORGIAN MTAVRULI CAPITAL LETTER HAE - '\x1cb0'# -> unI64 4336 - -- GEORGIAN MTAVRULI CAPITAL LETTER HE - '\x1cb1'# -> unI64 4337 - -- GEORGIAN MTAVRULI CAPITAL LETTER HIE - '\x1cb2'# -> unI64 4338 - -- GEORGIAN MTAVRULI CAPITAL LETTER WE - '\x1cb3'# -> unI64 4339 - -- GEORGIAN MTAVRULI CAPITAL LETTER HAR - '\x1cb4'# -> unI64 4340 - -- GEORGIAN MTAVRULI CAPITAL LETTER HOE - '\x1cb5'# -> unI64 4341 - -- GEORGIAN MTAVRULI CAPITAL LETTER FI - '\x1cb6'# -> unI64 4342 - -- GEORGIAN MTAVRULI CAPITAL LETTER YN - '\x1cb7'# -> unI64 4343 - -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI - '\x1cb8'# -> unI64 4344 - -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN - '\x1cb9'# -> unI64 4345 - -- GEORGIAN MTAVRULI CAPITAL LETTER AIN - '\x1cba'# -> unI64 4346 - -- GEORGIAN MTAVRULI CAPITAL LETTER AEN - '\x1cbd'# -> unI64 4349 - -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN - '\x1cbe'# -> unI64 4350 - -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN - '\x1cbf'# -> unI64 4351 - -- LATIN CAPITAL LETTER A WITH RING BELOW - '\x1e00'# -> unI64 7681 - -- LATIN CAPITAL LETTER B WITH DOT ABOVE - '\x1e02'# -> unI64 7683 - -- LATIN CAPITAL LETTER B WITH DOT BELOW - '\x1e04'# -> unI64 7685 - -- LATIN CAPITAL LETTER B WITH LINE BELOW - '\x1e06'# -> unI64 7687 - -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE - '\x1e08'# -> unI64 7689 - -- LATIN CAPITAL LETTER D WITH DOT ABOVE - '\x1e0a'# -> unI64 7691 - -- LATIN CAPITAL LETTER D WITH DOT BELOW - '\x1e0c'# -> unI64 7693 - -- LATIN CAPITAL LETTER D WITH LINE BELOW - '\x1e0e'# -> unI64 7695 - -- LATIN CAPITAL LETTER D WITH CEDILLA - '\x1e10'# -> unI64 7697 - -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW - '\x1e12'# -> unI64 7699 - -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE - '\x1e14'# -> unI64 7701 - -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE - '\x1e16'# -> unI64 7703 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW - '\x1e18'# -> unI64 7705 - -- LATIN CAPITAL LETTER E WITH TILDE BELOW - '\x1e1a'# -> unI64 7707 - -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE - '\x1e1c'# -> unI64 7709 - -- LATIN CAPITAL LETTER F WITH DOT ABOVE - '\x1e1e'# -> unI64 7711 - -- LATIN CAPITAL LETTER G WITH MACRON - '\x1e20'# -> unI64 7713 - -- LATIN CAPITAL LETTER H WITH DOT ABOVE - '\x1e22'# -> unI64 7715 - -- LATIN CAPITAL LETTER H WITH DOT BELOW - '\x1e24'# -> unI64 7717 - -- LATIN CAPITAL LETTER H WITH DIAERESIS - '\x1e26'# -> unI64 7719 - -- LATIN CAPITAL LETTER H WITH CEDILLA - '\x1e28'# -> unI64 7721 - -- LATIN CAPITAL LETTER H WITH BREVE BELOW - '\x1e2a'# -> unI64 7723 - -- LATIN CAPITAL LETTER I WITH TILDE BELOW - '\x1e2c'# -> unI64 7725 - -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE - '\x1e2e'# -> unI64 7727 - -- LATIN CAPITAL LETTER K WITH ACUTE - '\x1e30'# -> unI64 7729 - -- LATIN CAPITAL LETTER K WITH DOT BELOW - '\x1e32'# -> unI64 7731 - -- LATIN CAPITAL LETTER K WITH LINE BELOW - '\x1e34'# -> unI64 7733 - -- LATIN CAPITAL LETTER L WITH DOT BELOW - '\x1e36'# -> unI64 7735 - -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON - '\x1e38'# -> unI64 7737 - -- LATIN CAPITAL LETTER L WITH LINE BELOW - '\x1e3a'# -> unI64 7739 - -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW - '\x1e3c'# -> unI64 7741 - -- LATIN CAPITAL LETTER M WITH ACUTE - '\x1e3e'# -> unI64 7743 - -- LATIN CAPITAL LETTER M WITH DOT ABOVE - '\x1e40'# -> unI64 7745 - -- LATIN CAPITAL LETTER M WITH DOT BELOW - '\x1e42'# -> unI64 7747 - -- LATIN CAPITAL LETTER N WITH DOT ABOVE - '\x1e44'# -> unI64 7749 - -- LATIN CAPITAL LETTER N WITH DOT BELOW - '\x1e46'# -> unI64 7751 - -- LATIN CAPITAL LETTER N WITH LINE BELOW - '\x1e48'# -> unI64 7753 - -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW - '\x1e4a'# -> unI64 7755 - -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE - '\x1e4c'# -> unI64 7757 - -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS - '\x1e4e'# -> unI64 7759 - -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE - '\x1e50'# -> unI64 7761 - -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE - '\x1e52'# -> unI64 7763 - -- LATIN CAPITAL LETTER P WITH ACUTE - '\x1e54'# -> unI64 7765 - -- LATIN CAPITAL LETTER P WITH DOT ABOVE - '\x1e56'# -> unI64 7767 - -- LATIN CAPITAL LETTER R WITH DOT ABOVE - '\x1e58'# -> unI64 7769 - -- LATIN CAPITAL LETTER R WITH DOT BELOW - '\x1e5a'# -> unI64 7771 - -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON - '\x1e5c'# -> unI64 7773 - -- LATIN CAPITAL LETTER R WITH LINE BELOW - '\x1e5e'# -> unI64 7775 - -- LATIN CAPITAL LETTER S WITH DOT ABOVE - '\x1e60'# -> unI64 7777 - -- LATIN CAPITAL LETTER S WITH DOT BELOW - '\x1e62'# -> unI64 7779 - -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE - '\x1e64'# -> unI64 7781 - -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE - '\x1e66'# -> unI64 7783 - -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE - '\x1e68'# -> unI64 7785 - -- LATIN CAPITAL LETTER T WITH DOT ABOVE - '\x1e6a'# -> unI64 7787 - -- LATIN CAPITAL LETTER T WITH DOT BELOW - '\x1e6c'# -> unI64 7789 - -- LATIN CAPITAL LETTER T WITH LINE BELOW - '\x1e6e'# -> unI64 7791 - -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW - '\x1e70'# -> unI64 7793 - -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW - '\x1e72'# -> unI64 7795 - -- LATIN CAPITAL LETTER U WITH TILDE BELOW - '\x1e74'# -> unI64 7797 - -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW - '\x1e76'# -> unI64 7799 - -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE - '\x1e78'# -> unI64 7801 - -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS - '\x1e7a'# -> unI64 7803 - -- LATIN CAPITAL LETTER V WITH TILDE - '\x1e7c'# -> unI64 7805 - -- LATIN CAPITAL LETTER V WITH DOT BELOW - '\x1e7e'# -> unI64 7807 - -- LATIN CAPITAL LETTER W WITH GRAVE - '\x1e80'# -> unI64 7809 - -- LATIN CAPITAL LETTER W WITH ACUTE - '\x1e82'# -> unI64 7811 - -- LATIN CAPITAL LETTER W WITH DIAERESIS - '\x1e84'# -> unI64 7813 - -- LATIN CAPITAL LETTER W WITH DOT ABOVE - '\x1e86'# -> unI64 7815 - -- LATIN CAPITAL LETTER W WITH DOT BELOW - '\x1e88'# -> unI64 7817 - -- LATIN CAPITAL LETTER X WITH DOT ABOVE - '\x1e8a'# -> unI64 7819 - -- LATIN CAPITAL LETTER X WITH DIAERESIS - '\x1e8c'# -> unI64 7821 - -- LATIN CAPITAL LETTER Y WITH DOT ABOVE - '\x1e8e'# -> unI64 7823 - -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX - '\x1e90'# -> unI64 7825 - -- LATIN CAPITAL LETTER Z WITH DOT BELOW - '\x1e92'# -> unI64 7827 - -- LATIN CAPITAL LETTER Z WITH LINE BELOW - '\x1e94'# -> unI64 7829 - -- LATIN SMALL LETTER H WITH LINE BELOW - '\x1e96'# -> unI64 1713373288 - -- LATIN SMALL LETTER T WITH DIAERESIS - '\x1e97'# -> unI64 1627390068 - -- LATIN SMALL LETTER W WITH RING ABOVE - '\x1e98'# -> unI64 1631584375 - -- LATIN SMALL LETTER Y WITH RING ABOVE - '\x1e99'# -> unI64 1631584377 - -- LATIN SMALL LETTER A WITH RIGHT HALF RING - '\x1e9a'# -> unI64 1472200801 - -- LATIN SMALL LETTER LONG S WITH DOT ABOVE - '\x1e9b'# -> unI64 7777 - -- LATIN CAPITAL LETTER SHARP S - '\x1e9e'# -> unI64 241172595 - -- LATIN CAPITAL LETTER A WITH DOT BELOW - '\x1ea0'# -> unI64 7841 - -- LATIN CAPITAL LETTER A WITH HOOK ABOVE - '\x1ea2'# -> unI64 7843 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE - '\x1ea4'# -> unI64 7845 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE - '\x1ea6'# -> unI64 7847 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE - '\x1ea8'# -> unI64 7849 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE - '\x1eaa'# -> unI64 7851 - -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW - '\x1eac'# -> unI64 7853 - -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE - '\x1eae'# -> unI64 7855 - -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE - '\x1eb0'# -> unI64 7857 - -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE - '\x1eb2'# -> unI64 7859 - -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE - '\x1eb4'# -> unI64 7861 - -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW - '\x1eb6'# -> unI64 7863 - -- LATIN CAPITAL LETTER E WITH DOT BELOW - '\x1eb8'# -> unI64 7865 - -- LATIN CAPITAL LETTER E WITH HOOK ABOVE - '\x1eba'# -> unI64 7867 - -- LATIN CAPITAL LETTER E WITH TILDE - '\x1ebc'# -> unI64 7869 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE - '\x1ebe'# -> unI64 7871 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE - '\x1ec0'# -> unI64 7873 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE - '\x1ec2'# -> unI64 7875 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE - '\x1ec4'# -> unI64 7877 - -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW - '\x1ec6'# -> unI64 7879 - -- LATIN CAPITAL LETTER I WITH HOOK ABOVE - '\x1ec8'# -> unI64 7881 - -- LATIN CAPITAL LETTER I WITH DOT BELOW - '\x1eca'# -> unI64 7883 - -- LATIN CAPITAL LETTER O WITH DOT BELOW - '\x1ecc'# -> unI64 7885 - -- LATIN CAPITAL LETTER O WITH HOOK ABOVE - '\x1ece'# -> unI64 7887 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE - '\x1ed0'# -> unI64 7889 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE - '\x1ed2'# -> unI64 7891 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE - '\x1ed4'# -> unI64 7893 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE - '\x1ed6'# -> unI64 7895 - -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW - '\x1ed8'# -> unI64 7897 - -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE - '\x1eda'# -> unI64 7899 - -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE - '\x1edc'# -> unI64 7901 - -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE - '\x1ede'# -> unI64 7903 - -- LATIN CAPITAL LETTER O WITH HORN AND TILDE - '\x1ee0'# -> unI64 7905 - -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW - '\x1ee2'# -> unI64 7907 - -- LATIN CAPITAL LETTER U WITH DOT BELOW - '\x1ee4'# -> unI64 7909 - -- LATIN CAPITAL LETTER U WITH HOOK ABOVE - '\x1ee6'# -> unI64 7911 - -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE - '\x1ee8'# -> unI64 7913 - -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE - '\x1eea'# -> unI64 7915 - -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE - '\x1eec'# -> unI64 7917 - -- LATIN CAPITAL LETTER U WITH HORN AND TILDE - '\x1eee'# -> unI64 7919 - -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW - '\x1ef0'# -> unI64 7921 - -- LATIN CAPITAL LETTER Y WITH GRAVE - '\x1ef2'# -> unI64 7923 - -- LATIN CAPITAL LETTER Y WITH DOT BELOW - '\x1ef4'# -> unI64 7925 - -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE - '\x1ef6'# -> unI64 7927 - -- LATIN CAPITAL LETTER Y WITH TILDE - '\x1ef8'# -> unI64 7929 - -- LATIN CAPITAL LETTER MIDDLE-WELSH LL - '\x1efa'# -> unI64 7931 - -- LATIN CAPITAL LETTER MIDDLE-WELSH V - '\x1efc'# -> unI64 7933 - -- LATIN CAPITAL LETTER Y WITH LOOP - '\x1efe'# -> unI64 7935 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI - '\x1f08'# -> unI64 7936 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA - '\x1f09'# -> unI64 7937 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA - '\x1f0a'# -> unI64 7938 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA - '\x1f0b'# -> unI64 7939 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA - '\x1f0c'# -> unI64 7940 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA - '\x1f0d'# -> unI64 7941 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI - '\x1f0e'# -> unI64 7942 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI - '\x1f0f'# -> unI64 7943 - -- GREEK CAPITAL LETTER EPSILON WITH PSILI - '\x1f18'# -> unI64 7952 - -- GREEK CAPITAL LETTER EPSILON WITH DASIA - '\x1f19'# -> unI64 7953 - -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA - '\x1f1a'# -> unI64 7954 - -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA - '\x1f1b'# -> unI64 7955 - -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA - '\x1f1c'# -> unI64 7956 - -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA - '\x1f1d'# -> unI64 7957 - -- GREEK CAPITAL LETTER ETA WITH PSILI - '\x1f28'# -> unI64 7968 - -- GREEK CAPITAL LETTER ETA WITH DASIA - '\x1f29'# -> unI64 7969 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA - '\x1f2a'# -> unI64 7970 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA - '\x1f2b'# -> unI64 7971 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA - '\x1f2c'# -> unI64 7972 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA - '\x1f2d'# -> unI64 7973 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI - '\x1f2e'# -> unI64 7974 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI - '\x1f2f'# -> unI64 7975 - -- GREEK CAPITAL LETTER IOTA WITH PSILI - '\x1f38'# -> unI64 7984 - -- GREEK CAPITAL LETTER IOTA WITH DASIA - '\x1f39'# -> unI64 7985 - -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA - '\x1f3a'# -> unI64 7986 - -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA - '\x1f3b'# -> unI64 7987 - -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA - '\x1f3c'# -> unI64 7988 - -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA - '\x1f3d'# -> unI64 7989 - -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI - '\x1f3e'# -> unI64 7990 - -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI - '\x1f3f'# -> unI64 7991 - -- GREEK CAPITAL LETTER OMICRON WITH PSILI - '\x1f48'# -> unI64 8000 - -- GREEK CAPITAL LETTER OMICRON WITH DASIA - '\x1f49'# -> unI64 8001 - -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA - '\x1f4a'# -> unI64 8002 - -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA - '\x1f4b'# -> unI64 8003 - -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA - '\x1f4c'# -> unI64 8004 - -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA - '\x1f4d'# -> unI64 8005 - -- GREEK SMALL LETTER UPSILON WITH PSILI - '\x1f50'# -> unI64 1650459589 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA - '\x1f52'# -> unI64 3377701370987461 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA - '\x1f54'# -> unI64 3382099417498565 - -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI - '\x1f56'# -> unI64 3667972440720325 - -- GREEK CAPITAL LETTER UPSILON WITH DASIA - '\x1f59'# -> unI64 8017 - -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA - '\x1f5b'# -> unI64 8019 - -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA - '\x1f5d'# -> unI64 8021 - -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI - '\x1f5f'# -> unI64 8023 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI - '\x1f68'# -> unI64 8032 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA - '\x1f69'# -> unI64 8033 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA - '\x1f6a'# -> unI64 8034 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA - '\x1f6b'# -> unI64 8035 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA - '\x1f6c'# -> unI64 8036 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA - '\x1f6d'# -> unI64 8037 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI - '\x1f6e'# -> unI64 8038 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI - '\x1f6f'# -> unI64 8039 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI - '\x1f80'# -> unI64 1998593792 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI - '\x1f81'# -> unI64 1998593793 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1f82'# -> unI64 1998593794 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1f83'# -> unI64 1998593795 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1f84'# -> unI64 1998593796 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1f85'# -> unI64 1998593797 - -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f86'# -> unI64 1998593798 - -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f87'# -> unI64 1998593799 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI - '\x1f88'# -> unI64 1998593792 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI - '\x1f89'# -> unI64 1998593793 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1f8a'# -> unI64 1998593794 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1f8b'# -> unI64 1998593795 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1f8c'# -> unI64 1998593796 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1f8d'# -> unI64 1998593797 - -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f8e'# -> unI64 1998593798 - -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f8f'# -> unI64 1998593799 - -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI - '\x1f90'# -> unI64 1998593824 - -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI - '\x1f91'# -> unI64 1998593825 - -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1f92'# -> unI64 1998593826 - -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1f93'# -> unI64 1998593827 - -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1f94'# -> unI64 1998593828 - -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1f95'# -> unI64 1998593829 - -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f96'# -> unI64 1998593830 - -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1f97'# -> unI64 1998593831 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI - '\x1f98'# -> unI64 1998593824 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI - '\x1f99'# -> unI64 1998593825 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1f9a'# -> unI64 1998593826 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1f9b'# -> unI64 1998593827 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1f9c'# -> unI64 1998593828 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1f9d'# -> unI64 1998593829 - -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f9e'# -> unI64 1998593830 - -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1f9f'# -> unI64 1998593831 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI - '\x1fa0'# -> unI64 1998593888 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI - '\x1fa1'# -> unI64 1998593889 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI - '\x1fa2'# -> unI64 1998593890 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI - '\x1fa3'# -> unI64 1998593891 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI - '\x1fa4'# -> unI64 1998593892 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI - '\x1fa5'# -> unI64 1998593893 - -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI - '\x1fa6'# -> unI64 1998593894 - -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI - '\x1fa7'# -> unI64 1998593895 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI - '\x1fa8'# -> unI64 1998593888 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI - '\x1fa9'# -> unI64 1998593889 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI - '\x1faa'# -> unI64 1998593890 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI - '\x1fab'# -> unI64 1998593891 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI - '\x1fac'# -> unI64 1998593892 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI - '\x1fad'# -> unI64 1998593893 - -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI - '\x1fae'# -> unI64 1998593894 - -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI - '\x1faf'# -> unI64 1998593895 - -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI - '\x1fb2'# -> unI64 1998593904 - -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI - '\x1fb3'# -> unI64 1998586801 - -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI - '\x1fb4'# -> unI64 1998586796 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI - '\x1fb6'# -> unI64 1749025713 - -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fb7'# -> unI64 4191340074107825 - -- GREEK CAPITAL LETTER ALPHA WITH VRACHY - '\x1fb8'# -> unI64 8112 - -- GREEK CAPITAL LETTER ALPHA WITH MACRON - '\x1fb9'# -> unI64 8113 - -- GREEK CAPITAL LETTER ALPHA WITH VARIA - '\x1fba'# -> unI64 8048 - -- GREEK CAPITAL LETTER ALPHA WITH OXIA - '\x1fbb'# -> unI64 8049 - -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI - '\x1fbc'# -> unI64 1998586801 - -- GREEK PROSGEGRAMMENI - '\x1fbe'# -> unI64 953 - -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI - '\x1fc2'# -> unI64 1998593908 - -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI - '\x1fc3'# -> unI64 1998586807 - -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI - '\x1fc4'# -> unI64 1998586798 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI - '\x1fc6'# -> unI64 1749025719 - -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1fc7'# -> unI64 4191340074107831 - -- GREEK CAPITAL LETTER EPSILON WITH VARIA - '\x1fc8'# -> unI64 8050 - -- GREEK CAPITAL LETTER EPSILON WITH OXIA - '\x1fc9'# -> unI64 8051 - -- GREEK CAPITAL LETTER ETA WITH VARIA - '\x1fca'# -> unI64 8052 - -- GREEK CAPITAL LETTER ETA WITH OXIA - '\x1fcb'# -> unI64 8053 - -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI - '\x1fcc'# -> unI64 1998586807 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA - '\x1fd2'# -> unI64 3377701347918777 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA - '\x1fd3'# -> unI64 3382099394429881 - -- GREEK SMALL LETTER IOTA WITH PERISPOMENI - '\x1fd6'# -> unI64 1749025721 - -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI - '\x1fd7'# -> unI64 3667972417651641 - -- GREEK CAPITAL LETTER IOTA WITH VRACHY - '\x1fd8'# -> unI64 8144 - -- GREEK CAPITAL LETTER IOTA WITH MACRON - '\x1fd9'# -> unI64 8145 - -- GREEK CAPITAL LETTER IOTA WITH VARIA - '\x1fda'# -> unI64 8054 - -- GREEK CAPITAL LETTER IOTA WITH OXIA - '\x1fdb'# -> unI64 8055 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA - '\x1fe2'# -> unI64 3377701347918789 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA - '\x1fe3'# -> unI64 3382099394429893 - -- GREEK SMALL LETTER RHO WITH PSILI - '\x1fe4'# -> unI64 1650459585 - -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI - '\x1fe6'# -> unI64 1749025733 - -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI - '\x1fe7'# -> unI64 3667972417651653 - -- GREEK CAPITAL LETTER UPSILON WITH VRACHY - '\x1fe8'# -> unI64 8160 - -- GREEK CAPITAL LETTER UPSILON WITH MACRON - '\x1fe9'# -> unI64 8161 - -- GREEK CAPITAL LETTER UPSILON WITH VARIA - '\x1fea'# -> unI64 8058 - -- GREEK CAPITAL LETTER UPSILON WITH OXIA - '\x1feb'# -> unI64 8059 - -- GREEK CAPITAL LETTER RHO WITH DASIA - '\x1fec'# -> unI64 8165 - -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI - '\x1ff2'# -> unI64 1998593916 - -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI - '\x1ff3'# -> unI64 1998586825 - -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI - '\x1ff4'# -> unI64 1998586830 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI - '\x1ff6'# -> unI64 1749025737 - -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI - '\x1ff7'# -> unI64 4191340074107849 - -- GREEK CAPITAL LETTER OMICRON WITH VARIA - '\x1ff8'# -> unI64 8056 - -- GREEK CAPITAL LETTER OMICRON WITH OXIA - '\x1ff9'# -> unI64 8057 - -- GREEK CAPITAL LETTER OMEGA WITH VARIA - '\x1ffa'# -> unI64 8060 - -- GREEK CAPITAL LETTER OMEGA WITH OXIA - '\x1ffb'# -> unI64 8061 - -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI - '\x1ffc'# -> unI64 1998586825 - -- OHM SIGN - '\x2126'# -> unI64 969 - -- KELVIN SIGN - '\x212a'# -> unI64 107 - -- ANGSTROM SIGN - '\x212b'# -> unI64 229 - -- TURNED CAPITAL F - '\x2132'# -> unI64 8526 - -- ROMAN NUMERAL ONE - '\x2160'# -> unI64 8560 - -- ROMAN NUMERAL TWO - '\x2161'# -> unI64 8561 - -- ROMAN NUMERAL THREE - '\x2162'# -> unI64 8562 - -- ROMAN NUMERAL FOUR - '\x2163'# -> unI64 8563 - -- ROMAN NUMERAL FIVE - '\x2164'# -> unI64 8564 - -- ROMAN NUMERAL SIX - '\x2165'# -> unI64 8565 - -- ROMAN NUMERAL SEVEN - '\x2166'# -> unI64 8566 - -- ROMAN NUMERAL EIGHT - '\x2167'# -> unI64 8567 - -- ROMAN NUMERAL NINE - '\x2168'# -> unI64 8568 - -- ROMAN NUMERAL TEN - '\x2169'# -> unI64 8569 - -- ROMAN NUMERAL ELEVEN - '\x216a'# -> unI64 8570 - -- ROMAN NUMERAL TWELVE - '\x216b'# -> unI64 8571 - -- ROMAN NUMERAL FIFTY - '\x216c'# -> unI64 8572 - -- ROMAN NUMERAL ONE HUNDRED - '\x216d'# -> unI64 8573 - -- ROMAN NUMERAL FIVE HUNDRED - '\x216e'# -> unI64 8574 - -- ROMAN NUMERAL ONE THOUSAND - '\x216f'# -> unI64 8575 - -- ROMAN NUMERAL REVERSED ONE HUNDRED - '\x2183'# -> unI64 8580 - -- CIRCLED LATIN CAPITAL LETTER A - '\x24b6'# -> unI64 9424 - -- CIRCLED LATIN CAPITAL LETTER B - '\x24b7'# -> unI64 9425 - -- CIRCLED LATIN CAPITAL LETTER C - '\x24b8'# -> unI64 9426 - -- CIRCLED LATIN CAPITAL LETTER D - '\x24b9'# -> unI64 9427 - -- CIRCLED LATIN CAPITAL LETTER E - '\x24ba'# -> unI64 9428 - -- CIRCLED LATIN CAPITAL LETTER F - '\x24bb'# -> unI64 9429 - -- CIRCLED LATIN CAPITAL LETTER G - '\x24bc'# -> unI64 9430 - -- CIRCLED LATIN CAPITAL LETTER H - '\x24bd'# -> unI64 9431 - -- CIRCLED LATIN CAPITAL LETTER I - '\x24be'# -> unI64 9432 - -- CIRCLED LATIN CAPITAL LETTER J - '\x24bf'# -> unI64 9433 - -- CIRCLED LATIN CAPITAL LETTER K - '\x24c0'# -> unI64 9434 - -- CIRCLED LATIN CAPITAL LETTER L - '\x24c1'# -> unI64 9435 - -- CIRCLED LATIN CAPITAL LETTER M - '\x24c2'# -> unI64 9436 - -- CIRCLED LATIN CAPITAL LETTER N - '\x24c3'# -> unI64 9437 - -- CIRCLED LATIN CAPITAL LETTER O - '\x24c4'# -> unI64 9438 - -- CIRCLED LATIN CAPITAL LETTER P - '\x24c5'# -> unI64 9439 - -- CIRCLED LATIN CAPITAL LETTER Q - '\x24c6'# -> unI64 9440 - -- CIRCLED LATIN CAPITAL LETTER R - '\x24c7'# -> unI64 9441 - -- CIRCLED LATIN CAPITAL LETTER S - '\x24c8'# -> unI64 9442 - -- CIRCLED LATIN CAPITAL LETTER T - '\x24c9'# -> unI64 9443 - -- CIRCLED LATIN CAPITAL LETTER U - '\x24ca'# -> unI64 9444 - -- CIRCLED LATIN CAPITAL LETTER V - '\x24cb'# -> unI64 9445 - -- CIRCLED LATIN CAPITAL LETTER W - '\x24cc'# -> unI64 9446 - -- CIRCLED LATIN CAPITAL LETTER X - '\x24cd'# -> unI64 9447 - -- CIRCLED LATIN CAPITAL LETTER Y - '\x24ce'# -> unI64 9448 - -- CIRCLED LATIN CAPITAL LETTER Z - '\x24cf'# -> unI64 9449 - -- GLAGOLITIC CAPITAL LETTER AZU - '\x2c00'# -> unI64 11312 - -- GLAGOLITIC CAPITAL LETTER BUKY - '\x2c01'# -> unI64 11313 - -- GLAGOLITIC CAPITAL LETTER VEDE - '\x2c02'# -> unI64 11314 - -- GLAGOLITIC CAPITAL LETTER GLAGOLI - '\x2c03'# -> unI64 11315 - -- GLAGOLITIC CAPITAL LETTER DOBRO - '\x2c04'# -> unI64 11316 - -- GLAGOLITIC CAPITAL LETTER YESTU - '\x2c05'# -> unI64 11317 - -- GLAGOLITIC CAPITAL LETTER ZHIVETE - '\x2c06'# -> unI64 11318 - -- GLAGOLITIC CAPITAL LETTER DZELO - '\x2c07'# -> unI64 11319 - -- GLAGOLITIC CAPITAL LETTER ZEMLJA - '\x2c08'# -> unI64 11320 - -- GLAGOLITIC CAPITAL LETTER IZHE - '\x2c09'# -> unI64 11321 - -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE - '\x2c0a'# -> unI64 11322 - -- GLAGOLITIC CAPITAL LETTER I - '\x2c0b'# -> unI64 11323 - -- GLAGOLITIC CAPITAL LETTER DJERVI - '\x2c0c'# -> unI64 11324 - -- GLAGOLITIC CAPITAL LETTER KAKO - '\x2c0d'# -> unI64 11325 - -- GLAGOLITIC CAPITAL LETTER LJUDIJE - '\x2c0e'# -> unI64 11326 - -- GLAGOLITIC CAPITAL LETTER MYSLITE - '\x2c0f'# -> unI64 11327 - -- GLAGOLITIC CAPITAL LETTER NASHI - '\x2c10'# -> unI64 11328 - -- GLAGOLITIC CAPITAL LETTER ONU - '\x2c11'# -> unI64 11329 - -- GLAGOLITIC CAPITAL LETTER POKOJI - '\x2c12'# -> unI64 11330 - -- GLAGOLITIC CAPITAL LETTER RITSI - '\x2c13'# -> unI64 11331 - -- GLAGOLITIC CAPITAL LETTER SLOVO - '\x2c14'# -> unI64 11332 - -- GLAGOLITIC CAPITAL LETTER TVRIDO - '\x2c15'# -> unI64 11333 - -- GLAGOLITIC CAPITAL LETTER UKU - '\x2c16'# -> unI64 11334 - -- GLAGOLITIC CAPITAL LETTER FRITU - '\x2c17'# -> unI64 11335 - -- GLAGOLITIC CAPITAL LETTER HERU - '\x2c18'# -> unI64 11336 - -- GLAGOLITIC CAPITAL LETTER OTU - '\x2c19'# -> unI64 11337 - -- GLAGOLITIC CAPITAL LETTER PE - '\x2c1a'# -> unI64 11338 - -- GLAGOLITIC CAPITAL LETTER SHTA - '\x2c1b'# -> unI64 11339 - -- GLAGOLITIC CAPITAL LETTER TSI - '\x2c1c'# -> unI64 11340 - -- GLAGOLITIC CAPITAL LETTER CHRIVI - '\x2c1d'# -> unI64 11341 - -- GLAGOLITIC CAPITAL LETTER SHA - '\x2c1e'# -> unI64 11342 - -- GLAGOLITIC CAPITAL LETTER YERU - '\x2c1f'# -> unI64 11343 - -- GLAGOLITIC CAPITAL LETTER YERI - '\x2c20'# -> unI64 11344 - -- GLAGOLITIC CAPITAL LETTER YATI - '\x2c21'# -> unI64 11345 - -- GLAGOLITIC CAPITAL LETTER SPIDERY HA - '\x2c22'# -> unI64 11346 - -- GLAGOLITIC CAPITAL LETTER YU - '\x2c23'# -> unI64 11347 - -- GLAGOLITIC CAPITAL LETTER SMALL YUS - '\x2c24'# -> unI64 11348 - -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL - '\x2c25'# -> unI64 11349 - -- GLAGOLITIC CAPITAL LETTER YO - '\x2c26'# -> unI64 11350 - -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS - '\x2c27'# -> unI64 11351 - -- GLAGOLITIC CAPITAL LETTER BIG YUS - '\x2c28'# -> unI64 11352 - -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS - '\x2c29'# -> unI64 11353 - -- GLAGOLITIC CAPITAL LETTER FITA - '\x2c2a'# -> unI64 11354 - -- GLAGOLITIC CAPITAL LETTER IZHITSA - '\x2c2b'# -> unI64 11355 - -- GLAGOLITIC CAPITAL LETTER SHTAPIC - '\x2c2c'# -> unI64 11356 - -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A - '\x2c2d'# -> unI64 11357 - -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE - '\x2c2e'# -> unI64 11358 - -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI - '\x2c2f'# -> unI64 11359 - -- LATIN CAPITAL LETTER L WITH DOUBLE BAR - '\x2c60'# -> unI64 11361 - -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE - '\x2c62'# -> unI64 619 - -- LATIN CAPITAL LETTER P WITH STROKE - '\x2c63'# -> unI64 7549 - -- LATIN CAPITAL LETTER R WITH TAIL - '\x2c64'# -> unI64 637 - -- LATIN CAPITAL LETTER H WITH DESCENDER - '\x2c67'# -> unI64 11368 - -- LATIN CAPITAL LETTER K WITH DESCENDER - '\x2c69'# -> unI64 11370 - -- LATIN CAPITAL LETTER Z WITH DESCENDER - '\x2c6b'# -> unI64 11372 - -- LATIN CAPITAL LETTER ALPHA - '\x2c6d'# -> unI64 593 - -- LATIN CAPITAL LETTER M WITH HOOK - '\x2c6e'# -> unI64 625 - -- LATIN CAPITAL LETTER TURNED A - '\x2c6f'# -> unI64 592 - -- LATIN CAPITAL LETTER TURNED ALPHA - '\x2c70'# -> unI64 594 - -- LATIN CAPITAL LETTER W WITH HOOK - '\x2c72'# -> unI64 11379 - -- LATIN CAPITAL LETTER HALF H - '\x2c75'# -> unI64 11382 - -- LATIN CAPITAL LETTER S WITH SWASH TAIL - '\x2c7e'# -> unI64 575 - -- LATIN CAPITAL LETTER Z WITH SWASH TAIL - '\x2c7f'# -> unI64 576 - -- COPTIC CAPITAL LETTER ALFA - '\x2c80'# -> unI64 11393 - -- COPTIC CAPITAL LETTER VIDA - '\x2c82'# -> unI64 11395 - -- COPTIC CAPITAL LETTER GAMMA - '\x2c84'# -> unI64 11397 - -- COPTIC CAPITAL LETTER DALDA - '\x2c86'# -> unI64 11399 - -- COPTIC CAPITAL LETTER EIE - '\x2c88'# -> unI64 11401 - -- COPTIC CAPITAL LETTER SOU - '\x2c8a'# -> unI64 11403 - -- COPTIC CAPITAL LETTER ZATA - '\x2c8c'# -> unI64 11405 - -- COPTIC CAPITAL LETTER HATE - '\x2c8e'# -> unI64 11407 - -- COPTIC CAPITAL LETTER THETHE - '\x2c90'# -> unI64 11409 - -- COPTIC CAPITAL LETTER IAUDA - '\x2c92'# -> unI64 11411 - -- COPTIC CAPITAL LETTER KAPA - '\x2c94'# -> unI64 11413 - -- COPTIC CAPITAL LETTER LAULA - '\x2c96'# -> unI64 11415 - -- COPTIC CAPITAL LETTER MI - '\x2c98'# -> unI64 11417 - -- COPTIC CAPITAL LETTER NI - '\x2c9a'# -> unI64 11419 - -- COPTIC CAPITAL LETTER KSI - '\x2c9c'# -> unI64 11421 - -- COPTIC CAPITAL LETTER O - '\x2c9e'# -> unI64 11423 - -- COPTIC CAPITAL LETTER PI - '\x2ca0'# -> unI64 11425 - -- COPTIC CAPITAL LETTER RO - '\x2ca2'# -> unI64 11427 - -- COPTIC CAPITAL LETTER SIMA - '\x2ca4'# -> unI64 11429 - -- COPTIC CAPITAL LETTER TAU - '\x2ca6'# -> unI64 11431 - -- COPTIC CAPITAL LETTER UA - '\x2ca8'# -> unI64 11433 - -- COPTIC CAPITAL LETTER FI - '\x2caa'# -> unI64 11435 - -- COPTIC CAPITAL LETTER KHI - '\x2cac'# -> unI64 11437 - -- COPTIC CAPITAL LETTER PSI - '\x2cae'# -> unI64 11439 - -- COPTIC CAPITAL LETTER OOU - '\x2cb0'# -> unI64 11441 - -- COPTIC CAPITAL LETTER DIALECT-P ALEF - '\x2cb2'# -> unI64 11443 - -- COPTIC CAPITAL LETTER OLD COPTIC AIN - '\x2cb4'# -> unI64 11445 - -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE - '\x2cb6'# -> unI64 11447 - -- COPTIC CAPITAL LETTER DIALECT-P KAPA - '\x2cb8'# -> unI64 11449 - -- COPTIC CAPITAL LETTER DIALECT-P NI - '\x2cba'# -> unI64 11451 - -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI - '\x2cbc'# -> unI64 11453 - -- COPTIC CAPITAL LETTER OLD COPTIC OOU - '\x2cbe'# -> unI64 11455 - -- COPTIC CAPITAL LETTER SAMPI - '\x2cc0'# -> unI64 11457 - -- COPTIC CAPITAL LETTER CROSSED SHEI - '\x2cc2'# -> unI64 11459 - -- COPTIC CAPITAL LETTER OLD COPTIC SHEI - '\x2cc4'# -> unI64 11461 - -- COPTIC CAPITAL LETTER OLD COPTIC ESH - '\x2cc6'# -> unI64 11463 - -- COPTIC CAPITAL LETTER AKHMIMIC KHEI - '\x2cc8'# -> unI64 11465 - -- COPTIC CAPITAL LETTER DIALECT-P HORI - '\x2cca'# -> unI64 11467 - -- COPTIC CAPITAL LETTER OLD COPTIC HORI - '\x2ccc'# -> unI64 11469 - -- COPTIC CAPITAL LETTER OLD COPTIC HA - '\x2cce'# -> unI64 11471 - -- COPTIC CAPITAL LETTER L-SHAPED HA - '\x2cd0'# -> unI64 11473 - -- COPTIC CAPITAL LETTER OLD COPTIC HEI - '\x2cd2'# -> unI64 11475 - -- COPTIC CAPITAL LETTER OLD COPTIC HAT - '\x2cd4'# -> unI64 11477 - -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA - '\x2cd6'# -> unI64 11479 - -- COPTIC CAPITAL LETTER OLD COPTIC DJA - '\x2cd8'# -> unI64 11481 - -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA - '\x2cda'# -> unI64 11483 - -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA - '\x2cdc'# -> unI64 11485 - -- COPTIC CAPITAL LETTER OLD NUBIAN NGI - '\x2cde'# -> unI64 11487 - -- COPTIC CAPITAL LETTER OLD NUBIAN NYI - '\x2ce0'# -> unI64 11489 - -- COPTIC CAPITAL LETTER OLD NUBIAN WAU - '\x2ce2'# -> unI64 11491 - -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI - '\x2ceb'# -> unI64 11500 - -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA - '\x2ced'# -> unI64 11502 - -- COPTIC CAPITAL LETTER BOHAIRIC KHEI - '\x2cf2'# -> unI64 11507 - -- CYRILLIC CAPITAL LETTER ZEMLYA - '\xa640'# -> unI64 42561 - -- CYRILLIC CAPITAL LETTER DZELO - '\xa642'# -> unI64 42563 - -- CYRILLIC CAPITAL LETTER REVERSED DZE - '\xa644'# -> unI64 42565 - -- CYRILLIC CAPITAL LETTER IOTA - '\xa646'# -> unI64 42567 - -- CYRILLIC CAPITAL LETTER DJERV - '\xa648'# -> unI64 42569 - -- CYRILLIC CAPITAL LETTER MONOGRAPH UK - '\xa64a'# -> unI64 42571 - -- CYRILLIC CAPITAL LETTER BROAD OMEGA - '\xa64c'# -> unI64 42573 - -- CYRILLIC CAPITAL LETTER NEUTRAL YER - '\xa64e'# -> unI64 42575 - -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER - '\xa650'# -> unI64 42577 - -- CYRILLIC CAPITAL LETTER IOTIFIED YAT - '\xa652'# -> unI64 42579 - -- CYRILLIC CAPITAL LETTER REVERSED YU - '\xa654'# -> unI64 42581 - -- CYRILLIC CAPITAL LETTER IOTIFIED A - '\xa656'# -> unI64 42583 - -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS - '\xa658'# -> unI64 42585 - -- CYRILLIC CAPITAL LETTER BLENDED YUS - '\xa65a'# -> unI64 42587 - -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS - '\xa65c'# -> unI64 42589 - -- CYRILLIC CAPITAL LETTER YN - '\xa65e'# -> unI64 42591 - -- CYRILLIC CAPITAL LETTER REVERSED TSE - '\xa660'# -> unI64 42593 - -- CYRILLIC CAPITAL LETTER SOFT DE - '\xa662'# -> unI64 42595 - -- CYRILLIC CAPITAL LETTER SOFT EL - '\xa664'# -> unI64 42597 - -- CYRILLIC CAPITAL LETTER SOFT EM - '\xa666'# -> unI64 42599 - -- CYRILLIC CAPITAL LETTER MONOCULAR O - '\xa668'# -> unI64 42601 - -- CYRILLIC CAPITAL LETTER BINOCULAR O - '\xa66a'# -> unI64 42603 - -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O - '\xa66c'# -> unI64 42605 - -- CYRILLIC CAPITAL LETTER DWE - '\xa680'# -> unI64 42625 - -- CYRILLIC CAPITAL LETTER DZWE - '\xa682'# -> unI64 42627 - -- CYRILLIC CAPITAL LETTER ZHWE - '\xa684'# -> unI64 42629 - -- CYRILLIC CAPITAL LETTER CCHE - '\xa686'# -> unI64 42631 - -- CYRILLIC CAPITAL LETTER DZZE - '\xa688'# -> unI64 42633 - -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK - '\xa68a'# -> unI64 42635 - -- CYRILLIC CAPITAL LETTER TWE - '\xa68c'# -> unI64 42637 - -- CYRILLIC CAPITAL LETTER TSWE - '\xa68e'# -> unI64 42639 - -- CYRILLIC CAPITAL LETTER TSSE - '\xa690'# -> unI64 42641 - -- CYRILLIC CAPITAL LETTER TCHE - '\xa692'# -> unI64 42643 - -- CYRILLIC CAPITAL LETTER HWE - '\xa694'# -> unI64 42645 - -- CYRILLIC CAPITAL LETTER SHWE - '\xa696'# -> unI64 42647 - -- CYRILLIC CAPITAL LETTER DOUBLE O - '\xa698'# -> unI64 42649 - -- CYRILLIC CAPITAL LETTER CROSSED O - '\xa69a'# -> unI64 42651 - -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF - '\xa722'# -> unI64 42787 - -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN - '\xa724'# -> unI64 42789 - -- LATIN CAPITAL LETTER HENG - '\xa726'# -> unI64 42791 - -- LATIN CAPITAL LETTER TZ - '\xa728'# -> unI64 42793 - -- LATIN CAPITAL LETTER TRESILLO - '\xa72a'# -> unI64 42795 - -- LATIN CAPITAL LETTER CUATRILLO - '\xa72c'# -> unI64 42797 - -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA - '\xa72e'# -> unI64 42799 - -- LATIN CAPITAL LETTER AA - '\xa732'# -> unI64 42803 - -- LATIN CAPITAL LETTER AO - '\xa734'# -> unI64 42805 - -- LATIN CAPITAL LETTER AU - '\xa736'# -> unI64 42807 - -- LATIN CAPITAL LETTER AV - '\xa738'# -> unI64 42809 - -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR - '\xa73a'# -> unI64 42811 - -- LATIN CAPITAL LETTER AY - '\xa73c'# -> unI64 42813 - -- LATIN CAPITAL LETTER REVERSED C WITH DOT - '\xa73e'# -> unI64 42815 - -- LATIN CAPITAL LETTER K WITH STROKE - '\xa740'# -> unI64 42817 - -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE - '\xa742'# -> unI64 42819 - -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE - '\xa744'# -> unI64 42821 - -- LATIN CAPITAL LETTER BROKEN L - '\xa746'# -> unI64 42823 - -- LATIN CAPITAL LETTER L WITH HIGH STROKE - '\xa748'# -> unI64 42825 - -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY - '\xa74a'# -> unI64 42827 - -- LATIN CAPITAL LETTER O WITH LOOP - '\xa74c'# -> unI64 42829 - -- LATIN CAPITAL LETTER OO - '\xa74e'# -> unI64 42831 - -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER - '\xa750'# -> unI64 42833 - -- LATIN CAPITAL LETTER P WITH FLOURISH - '\xa752'# -> unI64 42835 - -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL - '\xa754'# -> unI64 42837 - -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER - '\xa756'# -> unI64 42839 - -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE - '\xa758'# -> unI64 42841 - -- LATIN CAPITAL LETTER R ROTUNDA - '\xa75a'# -> unI64 42843 - -- LATIN CAPITAL LETTER RUM ROTUNDA - '\xa75c'# -> unI64 42845 - -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE - '\xa75e'# -> unI64 42847 - -- LATIN CAPITAL LETTER VY - '\xa760'# -> unI64 42849 - -- LATIN CAPITAL LETTER VISIGOTHIC Z - '\xa762'# -> unI64 42851 - -- LATIN CAPITAL LETTER THORN WITH STROKE - '\xa764'# -> unI64 42853 - -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER - '\xa766'# -> unI64 42855 - -- LATIN CAPITAL LETTER VEND - '\xa768'# -> unI64 42857 - -- LATIN CAPITAL LETTER ET - '\xa76a'# -> unI64 42859 - -- LATIN CAPITAL LETTER IS - '\xa76c'# -> unI64 42861 - -- LATIN CAPITAL LETTER CON - '\xa76e'# -> unI64 42863 - -- LATIN CAPITAL LETTER INSULAR D - '\xa779'# -> unI64 42874 - -- LATIN CAPITAL LETTER INSULAR F - '\xa77b'# -> unI64 42876 - -- LATIN CAPITAL LETTER INSULAR G - '\xa77d'# -> unI64 7545 - -- LATIN CAPITAL LETTER TURNED INSULAR G - '\xa77e'# -> unI64 42879 - -- LATIN CAPITAL LETTER TURNED L - '\xa780'# -> unI64 42881 - -- LATIN CAPITAL LETTER INSULAR R - '\xa782'# -> unI64 42883 - -- LATIN CAPITAL LETTER INSULAR S - '\xa784'# -> unI64 42885 - -- LATIN CAPITAL LETTER INSULAR T - '\xa786'# -> unI64 42887 - -- LATIN CAPITAL LETTER SALTILLO - '\xa78b'# -> unI64 42892 - -- LATIN CAPITAL LETTER TURNED H - '\xa78d'# -> unI64 613 - -- LATIN CAPITAL LETTER N WITH DESCENDER - '\xa790'# -> unI64 42897 - -- LATIN CAPITAL LETTER C WITH BAR - '\xa792'# -> unI64 42899 - -- LATIN CAPITAL LETTER B WITH FLOURISH - '\xa796'# -> unI64 42903 - -- LATIN CAPITAL LETTER F WITH STROKE - '\xa798'# -> unI64 42905 - -- LATIN CAPITAL LETTER VOLAPUK AE - '\xa79a'# -> unI64 42907 - -- LATIN CAPITAL LETTER VOLAPUK OE - '\xa79c'# -> unI64 42909 - -- LATIN CAPITAL LETTER VOLAPUK UE - '\xa79e'# -> unI64 42911 - -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE - '\xa7a0'# -> unI64 42913 - -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE - '\xa7a2'# -> unI64 42915 - -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE - '\xa7a4'# -> unI64 42917 - -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE - '\xa7a6'# -> unI64 42919 - -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE - '\xa7a8'# -> unI64 42921 - -- LATIN CAPITAL LETTER H WITH HOOK - '\xa7aa'# -> unI64 614 - -- LATIN CAPITAL LETTER REVERSED OPEN E - '\xa7ab'# -> unI64 604 - -- LATIN CAPITAL LETTER SCRIPT G - '\xa7ac'# -> unI64 609 - -- LATIN CAPITAL LETTER L WITH BELT - '\xa7ad'# -> unI64 620 - -- LATIN CAPITAL LETTER SMALL CAPITAL I - '\xa7ae'# -> unI64 618 - -- LATIN CAPITAL LETTER TURNED K - '\xa7b0'# -> unI64 670 - -- LATIN CAPITAL LETTER TURNED T - '\xa7b1'# -> unI64 647 - -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL - '\xa7b2'# -> unI64 669 - -- LATIN CAPITAL LETTER CHI - '\xa7b3'# -> unI64 43859 - -- LATIN CAPITAL LETTER BETA - '\xa7b4'# -> unI64 42933 - -- LATIN CAPITAL LETTER OMEGA - '\xa7b6'# -> unI64 42935 - -- LATIN CAPITAL LETTER U WITH STROKE - '\xa7b8'# -> unI64 42937 - -- LATIN CAPITAL LETTER GLOTTAL A - '\xa7ba'# -> unI64 42939 - -- LATIN CAPITAL LETTER GLOTTAL I - '\xa7bc'# -> unI64 42941 - -- LATIN CAPITAL LETTER GLOTTAL U - '\xa7be'# -> unI64 42943 - -- LATIN CAPITAL LETTER OLD POLISH O - '\xa7c0'# -> unI64 42945 - -- LATIN CAPITAL LETTER ANGLICANA W - '\xa7c2'# -> unI64 42947 - -- LATIN CAPITAL LETTER C WITH PALATAL HOOK - '\xa7c4'# -> unI64 42900 - -- LATIN CAPITAL LETTER S WITH HOOK - '\xa7c5'# -> unI64 642 - -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK - '\xa7c6'# -> unI64 7566 - -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY - '\xa7c7'# -> unI64 42952 - -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY - '\xa7c9'# -> unI64 42954 - -- LATIN CAPITAL LETTER CLOSED INSULAR G - '\xa7d0'# -> unI64 42961 - -- LATIN CAPITAL LETTER MIDDLE SCOTS S - '\xa7d6'# -> unI64 42967 - -- LATIN CAPITAL LETTER SIGMOID S - '\xa7d8'# -> unI64 42969 - -- LATIN CAPITAL LETTER REVERSED HALF H - '\xa7f5'# -> unI64 42998 - -- CHEROKEE SMALL LETTER A - '\xab70'# -> unI64 5024 - -- CHEROKEE SMALL LETTER E - '\xab71'# -> unI64 5025 - -- CHEROKEE SMALL LETTER I - '\xab72'# -> unI64 5026 - -- CHEROKEE SMALL LETTER O - '\xab73'# -> unI64 5027 - -- CHEROKEE SMALL LETTER U - '\xab74'# -> unI64 5028 - -- CHEROKEE SMALL LETTER V - '\xab75'# -> unI64 5029 - -- CHEROKEE SMALL LETTER GA - '\xab76'# -> unI64 5030 - -- CHEROKEE SMALL LETTER KA - '\xab77'# -> unI64 5031 - -- CHEROKEE SMALL LETTER GE - '\xab78'# -> unI64 5032 - -- CHEROKEE SMALL LETTER GI - '\xab79'# -> unI64 5033 - -- CHEROKEE SMALL LETTER GO - '\xab7a'# -> unI64 5034 - -- CHEROKEE SMALL LETTER GU - '\xab7b'# -> unI64 5035 - -- CHEROKEE SMALL LETTER GV - '\xab7c'# -> unI64 5036 - -- CHEROKEE SMALL LETTER HA - '\xab7d'# -> unI64 5037 - -- CHEROKEE SMALL LETTER HE - '\xab7e'# -> unI64 5038 - -- CHEROKEE SMALL LETTER HI - '\xab7f'# -> unI64 5039 - -- CHEROKEE SMALL LETTER HO - '\xab80'# -> unI64 5040 - -- CHEROKEE SMALL LETTER HU - '\xab81'# -> unI64 5041 - -- CHEROKEE SMALL LETTER HV - '\xab82'# -> unI64 5042 - -- CHEROKEE SMALL LETTER LA - '\xab83'# -> unI64 5043 - -- CHEROKEE SMALL LETTER LE - '\xab84'# -> unI64 5044 - -- CHEROKEE SMALL LETTER LI - '\xab85'# -> unI64 5045 - -- CHEROKEE SMALL LETTER LO - '\xab86'# -> unI64 5046 - -- CHEROKEE SMALL LETTER LU - '\xab87'# -> unI64 5047 - -- CHEROKEE SMALL LETTER LV - '\xab88'# -> unI64 5048 - -- CHEROKEE SMALL LETTER MA - '\xab89'# -> unI64 5049 - -- CHEROKEE SMALL LETTER ME - '\xab8a'# -> unI64 5050 - -- CHEROKEE SMALL LETTER MI - '\xab8b'# -> unI64 5051 - -- CHEROKEE SMALL LETTER MO - '\xab8c'# -> unI64 5052 - -- CHEROKEE SMALL LETTER MU - '\xab8d'# -> unI64 5053 - -- CHEROKEE SMALL LETTER NA - '\xab8e'# -> unI64 5054 - -- CHEROKEE SMALL LETTER HNA - '\xab8f'# -> unI64 5055 - -- CHEROKEE SMALL LETTER NAH - '\xab90'# -> unI64 5056 - -- CHEROKEE SMALL LETTER NE - '\xab91'# -> unI64 5057 - -- CHEROKEE SMALL LETTER NI - '\xab92'# -> unI64 5058 - -- CHEROKEE SMALL LETTER NO - '\xab93'# -> unI64 5059 - -- CHEROKEE SMALL LETTER NU - '\xab94'# -> unI64 5060 - -- CHEROKEE SMALL LETTER NV - '\xab95'# -> unI64 5061 - -- CHEROKEE SMALL LETTER QUA - '\xab96'# -> unI64 5062 - -- CHEROKEE SMALL LETTER QUE - '\xab97'# -> unI64 5063 - -- CHEROKEE SMALL LETTER QUI - '\xab98'# -> unI64 5064 - -- CHEROKEE SMALL LETTER QUO - '\xab99'# -> unI64 5065 - -- CHEROKEE SMALL LETTER QUU - '\xab9a'# -> unI64 5066 - -- CHEROKEE SMALL LETTER QUV - '\xab9b'# -> unI64 5067 - -- CHEROKEE SMALL LETTER SA - '\xab9c'# -> unI64 5068 - -- CHEROKEE SMALL LETTER S - '\xab9d'# -> unI64 5069 - -- CHEROKEE SMALL LETTER SE - '\xab9e'# -> unI64 5070 - -- CHEROKEE SMALL LETTER SI - '\xab9f'# -> unI64 5071 - -- CHEROKEE SMALL LETTER SO - '\xaba0'# -> unI64 5072 - -- CHEROKEE SMALL LETTER SU - '\xaba1'# -> unI64 5073 - -- CHEROKEE SMALL LETTER SV - '\xaba2'# -> unI64 5074 - -- CHEROKEE SMALL LETTER DA - '\xaba3'# -> unI64 5075 - -- CHEROKEE SMALL LETTER TA - '\xaba4'# -> unI64 5076 - -- CHEROKEE SMALL LETTER DE - '\xaba5'# -> unI64 5077 - -- CHEROKEE SMALL LETTER TE - '\xaba6'# -> unI64 5078 - -- CHEROKEE SMALL LETTER DI - '\xaba7'# -> unI64 5079 - -- CHEROKEE SMALL LETTER TI - '\xaba8'# -> unI64 5080 - -- CHEROKEE SMALL LETTER DO - '\xaba9'# -> unI64 5081 - -- CHEROKEE SMALL LETTER DU - '\xabaa'# -> unI64 5082 - -- CHEROKEE SMALL LETTER DV - '\xabab'# -> unI64 5083 - -- CHEROKEE SMALL LETTER DLA - '\xabac'# -> unI64 5084 - -- CHEROKEE SMALL LETTER TLA - '\xabad'# -> unI64 5085 - -- CHEROKEE SMALL LETTER TLE - '\xabae'# -> unI64 5086 - -- CHEROKEE SMALL LETTER TLI - '\xabaf'# -> unI64 5087 - -- CHEROKEE SMALL LETTER TLO - '\xabb0'# -> unI64 5088 - -- CHEROKEE SMALL LETTER TLU - '\xabb1'# -> unI64 5089 - -- CHEROKEE SMALL LETTER TLV - '\xabb2'# -> unI64 5090 - -- CHEROKEE SMALL LETTER TSA - '\xabb3'# -> unI64 5091 - -- CHEROKEE SMALL LETTER TSE - '\xabb4'# -> unI64 5092 - -- CHEROKEE SMALL LETTER TSI - '\xabb5'# -> unI64 5093 - -- CHEROKEE SMALL LETTER TSO - '\xabb6'# -> unI64 5094 - -- CHEROKEE SMALL LETTER TSU - '\xabb7'# -> unI64 5095 - -- CHEROKEE SMALL LETTER TSV - '\xabb8'# -> unI64 5096 - -- CHEROKEE SMALL LETTER WA - '\xabb9'# -> unI64 5097 - -- CHEROKEE SMALL LETTER WE - '\xabba'# -> unI64 5098 - -- CHEROKEE SMALL LETTER WI - '\xabbb'# -> unI64 5099 - -- CHEROKEE SMALL LETTER WO - '\xabbc'# -> unI64 5100 - -- CHEROKEE SMALL LETTER WU - '\xabbd'# -> unI64 5101 - -- CHEROKEE SMALL LETTER WV - '\xabbe'# -> unI64 5102 - -- CHEROKEE SMALL LETTER YA - '\xabbf'# -> unI64 5103 - -- LATIN SMALL LIGATURE FF - '\xfb00'# -> unI64 213909606 - -- LATIN SMALL LIGATURE FI - '\xfb01'# -> unI64 220201062 - -- LATIN SMALL LIGATURE FL - '\xfb02'# -> unI64 226492518 - -- LATIN SMALL LIGATURE FFI - '\xfb03'# -> unI64 461795097575526 - -- LATIN SMALL LIGATURE FFL - '\xfb04'# -> unI64 474989237108838 - -- LATIN SMALL LIGATURE LONG S T - '\xfb05'# -> unI64 243269747 - -- LATIN SMALL LIGATURE ST - '\xfb06'# -> unI64 243269747 - -- ARMENIAN SMALL LIGATURE MEN NOW - '\xfb13'# -> unI64 2931819892 - -- ARMENIAN SMALL LIGATURE MEN ECH - '\xfb14'# -> unI64 2896168308 - -- ARMENIAN SMALL LIGATURE MEN INI - '\xfb15'# -> unI64 2908751220 - -- ARMENIAN SMALL LIGATURE VEW NOW - '\xfb16'# -> unI64 2931819902 - -- ARMENIAN SMALL LIGATURE MEN XEH - '\xfb17'# -> unI64 2912945524 - -- FULLWIDTH LATIN CAPITAL LETTER A - '\xff21'# -> unI64 65345 - -- FULLWIDTH LATIN CAPITAL LETTER B - '\xff22'# -> unI64 65346 - -- FULLWIDTH LATIN CAPITAL LETTER C - '\xff23'# -> unI64 65347 - -- FULLWIDTH LATIN CAPITAL LETTER D - '\xff24'# -> unI64 65348 - -- FULLWIDTH LATIN CAPITAL LETTER E - '\xff25'# -> unI64 65349 - -- FULLWIDTH LATIN CAPITAL LETTER F - '\xff26'# -> unI64 65350 - -- FULLWIDTH LATIN CAPITAL LETTER G - '\xff27'# -> unI64 65351 - -- FULLWIDTH LATIN CAPITAL LETTER H - '\xff28'# -> unI64 65352 - -- FULLWIDTH LATIN CAPITAL LETTER I - '\xff29'# -> unI64 65353 - -- FULLWIDTH LATIN CAPITAL LETTER J - '\xff2a'# -> unI64 65354 - -- FULLWIDTH LATIN CAPITAL LETTER K - '\xff2b'# -> unI64 65355 - -- FULLWIDTH LATIN CAPITAL LETTER L - '\xff2c'# -> unI64 65356 - -- FULLWIDTH LATIN CAPITAL LETTER M - '\xff2d'# -> unI64 65357 - -- FULLWIDTH LATIN CAPITAL LETTER N - '\xff2e'# -> unI64 65358 - -- FULLWIDTH LATIN CAPITAL LETTER O - '\xff2f'# -> unI64 65359 - -- FULLWIDTH LATIN CAPITAL LETTER P - '\xff30'# -> unI64 65360 - -- FULLWIDTH LATIN CAPITAL LETTER Q - '\xff31'# -> unI64 65361 - -- FULLWIDTH LATIN CAPITAL LETTER R - '\xff32'# -> unI64 65362 - -- FULLWIDTH LATIN CAPITAL LETTER S - '\xff33'# -> unI64 65363 - -- FULLWIDTH LATIN CAPITAL LETTER T - '\xff34'# -> unI64 65364 - -- FULLWIDTH LATIN CAPITAL LETTER U - '\xff35'# -> unI64 65365 - -- FULLWIDTH LATIN CAPITAL LETTER V - '\xff36'# -> unI64 65366 - -- FULLWIDTH LATIN CAPITAL LETTER W - '\xff37'# -> unI64 65367 - -- FULLWIDTH LATIN CAPITAL LETTER X - '\xff38'# -> unI64 65368 - -- FULLWIDTH LATIN CAPITAL LETTER Y - '\xff39'# -> unI64 65369 - -- FULLWIDTH LATIN CAPITAL LETTER Z - '\xff3a'# -> unI64 65370 - -- DESERET CAPITAL LETTER LONG I - '\x10400'# -> unI64 66600 - -- DESERET CAPITAL LETTER LONG E - '\x10401'# -> unI64 66601 - -- DESERET CAPITAL LETTER LONG A - '\x10402'# -> unI64 66602 - -- DESERET CAPITAL LETTER LONG AH - '\x10403'# -> unI64 66603 - -- DESERET CAPITAL LETTER LONG O - '\x10404'# -> unI64 66604 - -- DESERET CAPITAL LETTER LONG OO - '\x10405'# -> unI64 66605 - -- DESERET CAPITAL LETTER SHORT I - '\x10406'# -> unI64 66606 - -- DESERET CAPITAL LETTER SHORT E - '\x10407'# -> unI64 66607 - -- DESERET CAPITAL LETTER SHORT A - '\x10408'# -> unI64 66608 - -- DESERET CAPITAL LETTER SHORT AH - '\x10409'# -> unI64 66609 - -- DESERET CAPITAL LETTER SHORT O - '\x1040a'# -> unI64 66610 - -- DESERET CAPITAL LETTER SHORT OO - '\x1040b'# -> unI64 66611 - -- DESERET CAPITAL LETTER AY - '\x1040c'# -> unI64 66612 - -- DESERET CAPITAL LETTER OW - '\x1040d'# -> unI64 66613 - -- DESERET CAPITAL LETTER WU - '\x1040e'# -> unI64 66614 - -- DESERET CAPITAL LETTER YEE - '\x1040f'# -> unI64 66615 - -- DESERET CAPITAL LETTER H - '\x10410'# -> unI64 66616 - -- DESERET CAPITAL LETTER PEE - '\x10411'# -> unI64 66617 - -- DESERET CAPITAL LETTER BEE - '\x10412'# -> unI64 66618 - -- DESERET CAPITAL LETTER TEE - '\x10413'# -> unI64 66619 - -- DESERET CAPITAL LETTER DEE - '\x10414'# -> unI64 66620 - -- DESERET CAPITAL LETTER CHEE - '\x10415'# -> unI64 66621 - -- DESERET CAPITAL LETTER JEE - '\x10416'# -> unI64 66622 - -- DESERET CAPITAL LETTER KAY - '\x10417'# -> unI64 66623 - -- DESERET CAPITAL LETTER GAY - '\x10418'# -> unI64 66624 - -- DESERET CAPITAL LETTER EF - '\x10419'# -> unI64 66625 - -- DESERET CAPITAL LETTER VEE - '\x1041a'# -> unI64 66626 - -- DESERET CAPITAL LETTER ETH - '\x1041b'# -> unI64 66627 - -- DESERET CAPITAL LETTER THEE - '\x1041c'# -> unI64 66628 - -- DESERET CAPITAL LETTER ES - '\x1041d'# -> unI64 66629 - -- DESERET CAPITAL LETTER ZEE - '\x1041e'# -> unI64 66630 - -- DESERET CAPITAL LETTER ESH - '\x1041f'# -> unI64 66631 - -- DESERET CAPITAL LETTER ZHEE - '\x10420'# -> unI64 66632 - -- DESERET CAPITAL LETTER ER - '\x10421'# -> unI64 66633 - -- DESERET CAPITAL LETTER EL - '\x10422'# -> unI64 66634 - -- DESERET CAPITAL LETTER EM - '\x10423'# -> unI64 66635 - -- DESERET CAPITAL LETTER EN - '\x10424'# -> unI64 66636 - -- DESERET CAPITAL LETTER ENG - '\x10425'# -> unI64 66637 - -- DESERET CAPITAL LETTER OI - '\x10426'# -> unI64 66638 - -- DESERET CAPITAL LETTER EW - '\x10427'# -> unI64 66639 - -- OSAGE CAPITAL LETTER A - '\x104b0'# -> unI64 66776 - -- OSAGE CAPITAL LETTER AI - '\x104b1'# -> unI64 66777 - -- OSAGE CAPITAL LETTER AIN - '\x104b2'# -> unI64 66778 - -- OSAGE CAPITAL LETTER AH - '\x104b3'# -> unI64 66779 - -- OSAGE CAPITAL LETTER BRA - '\x104b4'# -> unI64 66780 - -- OSAGE CAPITAL LETTER CHA - '\x104b5'# -> unI64 66781 - -- OSAGE CAPITAL LETTER EHCHA - '\x104b6'# -> unI64 66782 - -- OSAGE CAPITAL LETTER E - '\x104b7'# -> unI64 66783 - -- OSAGE CAPITAL LETTER EIN - '\x104b8'# -> unI64 66784 - -- OSAGE CAPITAL LETTER HA - '\x104b9'# -> unI64 66785 - -- OSAGE CAPITAL LETTER HYA - '\x104ba'# -> unI64 66786 - -- OSAGE CAPITAL LETTER I - '\x104bb'# -> unI64 66787 - -- OSAGE CAPITAL LETTER KA - '\x104bc'# -> unI64 66788 - -- OSAGE CAPITAL LETTER EHKA - '\x104bd'# -> unI64 66789 - -- OSAGE CAPITAL LETTER KYA - '\x104be'# -> unI64 66790 - -- OSAGE CAPITAL LETTER LA - '\x104bf'# -> unI64 66791 - -- OSAGE CAPITAL LETTER MA - '\x104c0'# -> unI64 66792 - -- OSAGE CAPITAL LETTER NA - '\x104c1'# -> unI64 66793 - -- OSAGE CAPITAL LETTER O - '\x104c2'# -> unI64 66794 - -- OSAGE CAPITAL LETTER OIN - '\x104c3'# -> unI64 66795 - -- OSAGE CAPITAL LETTER PA - '\x104c4'# -> unI64 66796 - -- OSAGE CAPITAL LETTER EHPA - '\x104c5'# -> unI64 66797 - -- OSAGE CAPITAL LETTER SA - '\x104c6'# -> unI64 66798 - -- OSAGE CAPITAL LETTER SHA - '\x104c7'# -> unI64 66799 - -- OSAGE CAPITAL LETTER TA - '\x104c8'# -> unI64 66800 - -- OSAGE CAPITAL LETTER EHTA - '\x104c9'# -> unI64 66801 - -- OSAGE CAPITAL LETTER TSA - '\x104ca'# -> unI64 66802 - -- OSAGE CAPITAL LETTER EHTSA - '\x104cb'# -> unI64 66803 - -- OSAGE CAPITAL LETTER TSHA - '\x104cc'# -> unI64 66804 - -- OSAGE CAPITAL LETTER DHA - '\x104cd'# -> unI64 66805 - -- OSAGE CAPITAL LETTER U - '\x104ce'# -> unI64 66806 - -- OSAGE CAPITAL LETTER WA - '\x104cf'# -> unI64 66807 - -- OSAGE CAPITAL LETTER KHA - '\x104d0'# -> unI64 66808 - -- OSAGE CAPITAL LETTER GHA - '\x104d1'# -> unI64 66809 - -- OSAGE CAPITAL LETTER ZA - '\x104d2'# -> unI64 66810 - -- OSAGE CAPITAL LETTER ZHA - '\x104d3'# -> unI64 66811 - -- VITHKUQI CAPITAL LETTER A - '\x10570'# -> unI64 66967 - -- VITHKUQI CAPITAL LETTER BBE - '\x10571'# -> unI64 66968 - -- VITHKUQI CAPITAL LETTER BE - '\x10572'# -> unI64 66969 - -- VITHKUQI CAPITAL LETTER CE - '\x10573'# -> unI64 66970 - -- VITHKUQI CAPITAL LETTER CHE - '\x10574'# -> unI64 66971 - -- VITHKUQI CAPITAL LETTER DE - '\x10575'# -> unI64 66972 - -- VITHKUQI CAPITAL LETTER DHE - '\x10576'# -> unI64 66973 - -- VITHKUQI CAPITAL LETTER EI - '\x10577'# -> unI64 66974 - -- VITHKUQI CAPITAL LETTER E - '\x10578'# -> unI64 66975 - -- VITHKUQI CAPITAL LETTER FE - '\x10579'# -> unI64 66976 - -- VITHKUQI CAPITAL LETTER GA - '\x1057a'# -> unI64 66977 - -- VITHKUQI CAPITAL LETTER HA - '\x1057c'# -> unI64 66979 - -- VITHKUQI CAPITAL LETTER HHA - '\x1057d'# -> unI64 66980 - -- VITHKUQI CAPITAL LETTER I - '\x1057e'# -> unI64 66981 - -- VITHKUQI CAPITAL LETTER IJE - '\x1057f'# -> unI64 66982 - -- VITHKUQI CAPITAL LETTER JE - '\x10580'# -> unI64 66983 - -- VITHKUQI CAPITAL LETTER KA - '\x10581'# -> unI64 66984 - -- VITHKUQI CAPITAL LETTER LA - '\x10582'# -> unI64 66985 - -- VITHKUQI CAPITAL LETTER LLA - '\x10583'# -> unI64 66986 - -- VITHKUQI CAPITAL LETTER ME - '\x10584'# -> unI64 66987 - -- VITHKUQI CAPITAL LETTER NE - '\x10585'# -> unI64 66988 - -- VITHKUQI CAPITAL LETTER NJE - '\x10586'# -> unI64 66989 - -- VITHKUQI CAPITAL LETTER O - '\x10587'# -> unI64 66990 - -- VITHKUQI CAPITAL LETTER PE - '\x10588'# -> unI64 66991 - -- VITHKUQI CAPITAL LETTER QA - '\x10589'# -> unI64 66992 - -- VITHKUQI CAPITAL LETTER RE - '\x1058a'# -> unI64 66993 - -- VITHKUQI CAPITAL LETTER SE - '\x1058c'# -> unI64 66995 - -- VITHKUQI CAPITAL LETTER SHE - '\x1058d'# -> unI64 66996 - -- VITHKUQI CAPITAL LETTER TE - '\x1058e'# -> unI64 66997 - -- VITHKUQI CAPITAL LETTER THE - '\x1058f'# -> unI64 66998 - -- VITHKUQI CAPITAL LETTER U - '\x10590'# -> unI64 66999 - -- VITHKUQI CAPITAL LETTER VE - '\x10591'# -> unI64 67000 - -- VITHKUQI CAPITAL LETTER XE - '\x10592'# -> unI64 67001 - -- VITHKUQI CAPITAL LETTER Y - '\x10594'# -> unI64 67003 - -- VITHKUQI CAPITAL LETTER ZE - '\x10595'# -> unI64 67004 - -- OLD HUNGARIAN CAPITAL LETTER A - '\x10c80'# -> unI64 68800 - -- OLD HUNGARIAN CAPITAL LETTER AA - '\x10c81'# -> unI64 68801 - -- OLD HUNGARIAN CAPITAL LETTER EB - '\x10c82'# -> unI64 68802 - -- OLD HUNGARIAN CAPITAL LETTER AMB - '\x10c83'# -> unI64 68803 - -- OLD HUNGARIAN CAPITAL LETTER EC - '\x10c84'# -> unI64 68804 - -- OLD HUNGARIAN CAPITAL LETTER ENC - '\x10c85'# -> unI64 68805 - -- OLD HUNGARIAN CAPITAL LETTER ECS - '\x10c86'# -> unI64 68806 - -- OLD HUNGARIAN CAPITAL LETTER ED - '\x10c87'# -> unI64 68807 - -- OLD HUNGARIAN CAPITAL LETTER AND - '\x10c88'# -> unI64 68808 - -- OLD HUNGARIAN CAPITAL LETTER E - '\x10c89'# -> unI64 68809 - -- OLD HUNGARIAN CAPITAL LETTER CLOSE E - '\x10c8a'# -> unI64 68810 - -- OLD HUNGARIAN CAPITAL LETTER EE - '\x10c8b'# -> unI64 68811 - -- OLD HUNGARIAN CAPITAL LETTER EF - '\x10c8c'# -> unI64 68812 - -- OLD HUNGARIAN CAPITAL LETTER EG - '\x10c8d'# -> unI64 68813 - -- OLD HUNGARIAN CAPITAL LETTER EGY - '\x10c8e'# -> unI64 68814 - -- OLD HUNGARIAN CAPITAL LETTER EH - '\x10c8f'# -> unI64 68815 - -- OLD HUNGARIAN CAPITAL LETTER I - '\x10c90'# -> unI64 68816 - -- OLD HUNGARIAN CAPITAL LETTER II - '\x10c91'# -> unI64 68817 - -- OLD HUNGARIAN CAPITAL LETTER EJ - '\x10c92'# -> unI64 68818 - -- OLD HUNGARIAN CAPITAL LETTER EK - '\x10c93'# -> unI64 68819 - -- OLD HUNGARIAN CAPITAL LETTER AK - '\x10c94'# -> unI64 68820 - -- OLD HUNGARIAN CAPITAL LETTER UNK - '\x10c95'# -> unI64 68821 - -- OLD HUNGARIAN CAPITAL LETTER EL - '\x10c96'# -> unI64 68822 - -- OLD HUNGARIAN CAPITAL LETTER ELY - '\x10c97'# -> unI64 68823 - -- OLD HUNGARIAN CAPITAL LETTER EM - '\x10c98'# -> unI64 68824 - -- OLD HUNGARIAN CAPITAL LETTER EN - '\x10c99'# -> unI64 68825 - -- OLD HUNGARIAN CAPITAL LETTER ENY - '\x10c9a'# -> unI64 68826 - -- OLD HUNGARIAN CAPITAL LETTER O - '\x10c9b'# -> unI64 68827 - -- OLD HUNGARIAN CAPITAL LETTER OO - '\x10c9c'# -> unI64 68828 - -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE - '\x10c9d'# -> unI64 68829 - -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE - '\x10c9e'# -> unI64 68830 - -- OLD HUNGARIAN CAPITAL LETTER OEE - '\x10c9f'# -> unI64 68831 - -- OLD HUNGARIAN CAPITAL LETTER EP - '\x10ca0'# -> unI64 68832 - -- OLD HUNGARIAN CAPITAL LETTER EMP - '\x10ca1'# -> unI64 68833 - -- OLD HUNGARIAN CAPITAL LETTER ER - '\x10ca2'# -> unI64 68834 - -- OLD HUNGARIAN CAPITAL LETTER SHORT ER - '\x10ca3'# -> unI64 68835 - -- OLD HUNGARIAN CAPITAL LETTER ES - '\x10ca4'# -> unI64 68836 - -- OLD HUNGARIAN CAPITAL LETTER ESZ - '\x10ca5'# -> unI64 68837 - -- OLD HUNGARIAN CAPITAL LETTER ET - '\x10ca6'# -> unI64 68838 - -- OLD HUNGARIAN CAPITAL LETTER ENT - '\x10ca7'# -> unI64 68839 - -- OLD HUNGARIAN CAPITAL LETTER ETY - '\x10ca8'# -> unI64 68840 - -- OLD HUNGARIAN CAPITAL LETTER ECH - '\x10ca9'# -> unI64 68841 - -- OLD HUNGARIAN CAPITAL LETTER U - '\x10caa'# -> unI64 68842 - -- OLD HUNGARIAN CAPITAL LETTER UU - '\x10cab'# -> unI64 68843 - -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE - '\x10cac'# -> unI64 68844 - -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE - '\x10cad'# -> unI64 68845 - -- OLD HUNGARIAN CAPITAL LETTER EV - '\x10cae'# -> unI64 68846 - -- OLD HUNGARIAN CAPITAL LETTER EZ - '\x10caf'# -> unI64 68847 - -- OLD HUNGARIAN CAPITAL LETTER EZS - '\x10cb0'# -> unI64 68848 - -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN - '\x10cb1'# -> unI64 68849 - -- OLD HUNGARIAN CAPITAL LETTER US - '\x10cb2'# -> unI64 68850 - -- WARANG CITI CAPITAL LETTER NGAA - '\x118a0'# -> unI64 71872 - -- WARANG CITI CAPITAL LETTER A - '\x118a1'# -> unI64 71873 - -- WARANG CITI CAPITAL LETTER WI - '\x118a2'# -> unI64 71874 - -- WARANG CITI CAPITAL LETTER YU - '\x118a3'# -> unI64 71875 - -- WARANG CITI CAPITAL LETTER YA - '\x118a4'# -> unI64 71876 - -- WARANG CITI CAPITAL LETTER YO - '\x118a5'# -> unI64 71877 - -- WARANG CITI CAPITAL LETTER II - '\x118a6'# -> unI64 71878 - -- WARANG CITI CAPITAL LETTER UU - '\x118a7'# -> unI64 71879 - -- WARANG CITI CAPITAL LETTER E - '\x118a8'# -> unI64 71880 - -- WARANG CITI CAPITAL LETTER O - '\x118a9'# -> unI64 71881 - -- WARANG CITI CAPITAL LETTER ANG - '\x118aa'# -> unI64 71882 - -- WARANG CITI CAPITAL LETTER GA - '\x118ab'# -> unI64 71883 - -- WARANG CITI CAPITAL LETTER KO - '\x118ac'# -> unI64 71884 - -- WARANG CITI CAPITAL LETTER ENY - '\x118ad'# -> unI64 71885 - -- WARANG CITI CAPITAL LETTER YUJ - '\x118ae'# -> unI64 71886 - -- WARANG CITI CAPITAL LETTER UC - '\x118af'# -> unI64 71887 - -- WARANG CITI CAPITAL LETTER ENN - '\x118b0'# -> unI64 71888 - -- WARANG CITI CAPITAL LETTER ODD - '\x118b1'# -> unI64 71889 - -- WARANG CITI CAPITAL LETTER TTE - '\x118b2'# -> unI64 71890 - -- WARANG CITI CAPITAL LETTER NUNG - '\x118b3'# -> unI64 71891 - -- WARANG CITI CAPITAL LETTER DA - '\x118b4'# -> unI64 71892 - -- WARANG CITI CAPITAL LETTER AT - '\x118b5'# -> unI64 71893 - -- WARANG CITI CAPITAL LETTER AM - '\x118b6'# -> unI64 71894 - -- WARANG CITI CAPITAL LETTER BU - '\x118b7'# -> unI64 71895 - -- WARANG CITI CAPITAL LETTER PU - '\x118b8'# -> unI64 71896 - -- WARANG CITI CAPITAL LETTER HIYO - '\x118b9'# -> unI64 71897 - -- WARANG CITI CAPITAL LETTER HOLO - '\x118ba'# -> unI64 71898 - -- WARANG CITI CAPITAL LETTER HORR - '\x118bb'# -> unI64 71899 - -- WARANG CITI CAPITAL LETTER HAR - '\x118bc'# -> unI64 71900 - -- WARANG CITI CAPITAL LETTER SSUU - '\x118bd'# -> unI64 71901 - -- WARANG CITI CAPITAL LETTER SII - '\x118be'# -> unI64 71902 - -- WARANG CITI CAPITAL LETTER VIYO - '\x118bf'# -> unI64 71903 - -- MEDEFAIDRIN CAPITAL LETTER M - '\x16e40'# -> unI64 93792 - -- MEDEFAIDRIN CAPITAL LETTER S - '\x16e41'# -> unI64 93793 - -- MEDEFAIDRIN CAPITAL LETTER V - '\x16e42'# -> unI64 93794 - -- MEDEFAIDRIN CAPITAL LETTER W - '\x16e43'# -> unI64 93795 - -- MEDEFAIDRIN CAPITAL LETTER ATIU - '\x16e44'# -> unI64 93796 - -- MEDEFAIDRIN CAPITAL LETTER Z - '\x16e45'# -> unI64 93797 - -- MEDEFAIDRIN CAPITAL LETTER KP - '\x16e46'# -> unI64 93798 - -- MEDEFAIDRIN CAPITAL LETTER P - '\x16e47'# -> unI64 93799 - -- MEDEFAIDRIN CAPITAL LETTER T - '\x16e48'# -> unI64 93800 - -- MEDEFAIDRIN CAPITAL LETTER G - '\x16e49'# -> unI64 93801 - -- MEDEFAIDRIN CAPITAL LETTER F - '\x16e4a'# -> unI64 93802 - -- MEDEFAIDRIN CAPITAL LETTER I - '\x16e4b'# -> unI64 93803 - -- MEDEFAIDRIN CAPITAL LETTER K - '\x16e4c'# -> unI64 93804 - -- MEDEFAIDRIN CAPITAL LETTER A - '\x16e4d'# -> unI64 93805 - -- MEDEFAIDRIN CAPITAL LETTER J - '\x16e4e'# -> unI64 93806 - -- MEDEFAIDRIN CAPITAL LETTER E - '\x16e4f'# -> unI64 93807 - -- MEDEFAIDRIN CAPITAL LETTER B - '\x16e50'# -> unI64 93808 - -- MEDEFAIDRIN CAPITAL LETTER C - '\x16e51'# -> unI64 93809 - -- MEDEFAIDRIN CAPITAL LETTER U - '\x16e52'# -> unI64 93810 - -- MEDEFAIDRIN CAPITAL LETTER YU - '\x16e53'# -> unI64 93811 - -- MEDEFAIDRIN CAPITAL LETTER L - '\x16e54'# -> unI64 93812 - -- MEDEFAIDRIN CAPITAL LETTER Q - '\x16e55'# -> unI64 93813 - -- MEDEFAIDRIN CAPITAL LETTER HP - '\x16e56'# -> unI64 93814 - -- MEDEFAIDRIN CAPITAL LETTER NY - '\x16e57'# -> unI64 93815 - -- MEDEFAIDRIN CAPITAL LETTER X - '\x16e58'# -> unI64 93816 - -- MEDEFAIDRIN CAPITAL LETTER D - '\x16e59'# -> unI64 93817 - -- MEDEFAIDRIN CAPITAL LETTER OE - '\x16e5a'# -> unI64 93818 - -- MEDEFAIDRIN CAPITAL LETTER N - '\x16e5b'# -> unI64 93819 - -- MEDEFAIDRIN CAPITAL LETTER R - '\x16e5c'# -> unI64 93820 - -- MEDEFAIDRIN CAPITAL LETTER O - '\x16e5d'# -> unI64 93821 - -- MEDEFAIDRIN CAPITAL LETTER AI - '\x16e5e'# -> unI64 93822 - -- MEDEFAIDRIN CAPITAL LETTER Y - '\x16e5f'# -> unI64 93823 - -- ADLAM CAPITAL LETTER ALIF - '\x1e900'# -> unI64 125218 - -- ADLAM CAPITAL LETTER DAALI - '\x1e901'# -> unI64 125219 - -- ADLAM CAPITAL LETTER LAAM - '\x1e902'# -> unI64 125220 - -- ADLAM CAPITAL LETTER MIIM - '\x1e903'# -> unI64 125221 - -- ADLAM CAPITAL LETTER BA - '\x1e904'# -> unI64 125222 - -- ADLAM CAPITAL LETTER SINNYIIYHE - '\x1e905'# -> unI64 125223 - -- ADLAM CAPITAL LETTER PE - '\x1e906'# -> unI64 125224 - -- ADLAM CAPITAL LETTER BHE - '\x1e907'# -> unI64 125225 - -- ADLAM CAPITAL LETTER RA - '\x1e908'# -> unI64 125226 - -- ADLAM CAPITAL LETTER E - '\x1e909'# -> unI64 125227 - -- ADLAM CAPITAL LETTER FA - '\x1e90a'# -> unI64 125228 - -- ADLAM CAPITAL LETTER I - '\x1e90b'# -> unI64 125229 - -- ADLAM CAPITAL LETTER O - '\x1e90c'# -> unI64 125230 - -- ADLAM CAPITAL LETTER DHA - '\x1e90d'# -> unI64 125231 - -- ADLAM CAPITAL LETTER YHE - '\x1e90e'# -> unI64 125232 - -- ADLAM CAPITAL LETTER WAW - '\x1e90f'# -> unI64 125233 - -- ADLAM CAPITAL LETTER NUN - '\x1e910'# -> unI64 125234 - -- ADLAM CAPITAL LETTER KAF - '\x1e911'# -> unI64 125235 - -- ADLAM CAPITAL LETTER YA - '\x1e912'# -> unI64 125236 - -- ADLAM CAPITAL LETTER U - '\x1e913'# -> unI64 125237 - -- ADLAM CAPITAL LETTER JIIM - '\x1e914'# -> unI64 125238 - -- ADLAM CAPITAL LETTER CHI - '\x1e915'# -> unI64 125239 - -- ADLAM CAPITAL LETTER HA - '\x1e916'# -> unI64 125240 - -- ADLAM CAPITAL LETTER QAAF - '\x1e917'# -> unI64 125241 - -- ADLAM CAPITAL LETTER GA - '\x1e918'# -> unI64 125242 - -- ADLAM CAPITAL LETTER NYA - '\x1e919'# -> unI64 125243 - -- ADLAM CAPITAL LETTER TU - '\x1e91a'# -> unI64 125244 - -- ADLAM CAPITAL LETTER NHA - '\x1e91b'# -> unI64 125245 - -- ADLAM CAPITAL LETTER VA - '\x1e91c'# -> unI64 125246 - -- ADLAM CAPITAL LETTER KHA - '\x1e91d'# -> unI64 125247 - -- ADLAM CAPITAL LETTER GBE - '\x1e91e'# -> unI64 125248 - -- ADLAM CAPITAL LETTER ZAL - '\x1e91f'# -> unI64 125249 - -- ADLAM CAPITAL LETTER KPO - '\x1e920'# -> unI64 125250 - -- ADLAM CAPITAL LETTER SHA - '\x1e921'# -> unI64 125251 - _ -> unI64 0 +-- AUTOMATICALLY GENERATED - DO NOT EDIT +-- Generated by scripts/CaseMapping.hs +-- CaseFolding-17.0.0.txt +-- Date: 2025-07-30, 23:54:36 GMT +-- SpecialCasing-17.0.0.txt +-- Date: 2025-07-31, 22:11:55 GMT + +{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +module Data.Text.Internal.Fusion.CaseMapping where +import GHC.Int +import GHC.Exts +import Data.Version (Version, makeVersion) +unicodeVersion :: Version +unicodeVersion = makeVersion [17, 0, 0] +unI64 :: Int64 -> _ {- unboxed Int64 -} +unI64 (I64# n) = n + +upperMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE upperMapping #-} +upperMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 174063699 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 146800710 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 153092166 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 159383622 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 321057542111302 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 334251681644614 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 176160851 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 176160851 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2856322357 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2831156548 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2795504964 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2808087876 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2831156558 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2812282180 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1931484936 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1931484937 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1931484938 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1931484939 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1931484940 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1931484941 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1931484942 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1931484943 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1931484936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1931484937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1931484938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1931484939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1931484940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1931484941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1931484942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1931484943 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1931484968 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1931484969 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1931484970 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1931484971 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1931484972 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1931484973 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1931484974 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1931484975 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1931484968 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1931484969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1931484970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1931484971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1931484972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1931484973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1931484974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1931484975 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1931485032 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1931485033 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1931485034 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1931485035 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1931485036 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1931485037 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1931485038 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1931485039 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1931485032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1931485033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1931485034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1931485035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1931485036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1931485037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1931485038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1931485039 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1931477905 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1931477905 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1931477911 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1931477911 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1931477929 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1931477929 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1931485114 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1931477894 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1931485130 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1931477897 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1931485178 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1931477903 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4050602585752465 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4050602585752471 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4050602585752489 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c5'# -> unI64 452 + '\x01c6'# -> unI64 452 + '\x01c8'# -> unI64 455 + '\x01c9'# -> unI64 455 + '\x01cb'# -> unI64 458 + '\x01cc'# -> unI64 458 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f2'# -> unI64 497 + '\x01f3'# -> unI64 497 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x10d0'# -> unI64 7312 + '\x10d1'# -> unI64 7313 + '\x10d2'# -> unI64 7314 + '\x10d3'# -> unI64 7315 + '\x10d4'# -> unI64 7316 + '\x10d5'# -> unI64 7317 + '\x10d6'# -> unI64 7318 + '\x10d7'# -> unI64 7319 + '\x10d8'# -> unI64 7320 + '\x10d9'# -> unI64 7321 + '\x10da'# -> unI64 7322 + '\x10db'# -> unI64 7323 + '\x10dc'# -> unI64 7324 + '\x10dd'# -> unI64 7325 + '\x10de'# -> unI64 7326 + '\x10df'# -> unI64 7327 + '\x10e0'# -> unI64 7328 + '\x10e1'# -> unI64 7329 + '\x10e2'# -> unI64 7330 + '\x10e3'# -> unI64 7331 + '\x10e4'# -> unI64 7332 + '\x10e5'# -> unI64 7333 + '\x10e6'# -> unI64 7334 + '\x10e7'# -> unI64 7335 + '\x10e8'# -> unI64 7336 + '\x10e9'# -> unI64 7337 + '\x10ea'# -> unI64 7338 + '\x10eb'# -> unI64 7339 + '\x10ec'# -> unI64 7340 + '\x10ed'# -> unI64 7341 + '\x10ee'# -> unI64 7342 + '\x10ef'# -> unI64 7343 + '\x10f0'# -> unI64 7344 + '\x10f1'# -> unI64 7345 + '\x10f2'# -> unI64 7346 + '\x10f3'# -> unI64 7347 + '\x10f4'# -> unI64 7348 + '\x10f5'# -> unI64 7349 + '\x10f6'# -> unI64 7350 + '\x10f7'# -> unI64 7351 + '\x10f8'# -> unI64 7352 + '\x10f9'# -> unI64 7353 + '\x10fa'# -> unI64 7354 + '\x10fd'# -> unI64 7357 + '\x10fe'# -> unI64 7358 + '\x10ff'# -> unI64 7359 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +lowerMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE lowerMapping #-} +lowerMapping = \case + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 8064 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 8065 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 8066 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 8067 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 8068 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 8069 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 8070 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 8071 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 8080 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 8081 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 8082 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 8083 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 8084 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 8085 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 8086 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 8087 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 8096 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 8097 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 8098 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 8099 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 8100 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 8101 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 8102 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 8103 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 8115 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 8131 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 8179 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c89'# -> unI64 7306 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1e9e'# -> unI64 223 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7cb'# -> unI64 612 + '\xa7cc'# -> unI64 42957 + '\xa7ce'# -> unI64 42959 + '\xa7d0'# -> unI64 42961 + '\xa7d2'# -> unI64 42963 + '\xa7d4'# -> unI64 42965 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7da'# -> unI64 42971 + '\xa7dc'# -> unI64 411 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x10d50'# -> unI64 68976 + '\x10d51'# -> unI64 68977 + '\x10d52'# -> unI64 68978 + '\x10d53'# -> unI64 68979 + '\x10d54'# -> unI64 68980 + '\x10d55'# -> unI64 68981 + '\x10d56'# -> unI64 68982 + '\x10d57'# -> unI64 68983 + '\x10d58'# -> unI64 68984 + '\x10d59'# -> unI64 68985 + '\x10d5a'# -> unI64 68986 + '\x10d5b'# -> unI64 68987 + '\x10d5c'# -> unI64 68988 + '\x10d5d'# -> unI64 68989 + '\x10d5e'# -> unI64 68990 + '\x10d5f'# -> unI64 68991 + '\x10d60'# -> unI64 68992 + '\x10d61'# -> unI64 68993 + '\x10d62'# -> unI64 68994 + '\x10d63'# -> unI64 68995 + '\x10d64'# -> unI64 68996 + '\x10d65'# -> unI64 68997 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x16ea0'# -> unI64 93883 + '\x16ea1'# -> unI64 93884 + '\x16ea2'# -> unI64 93885 + '\x16ea3'# -> unI64 93886 + '\x16ea4'# -> unI64 93887 + '\x16ea5'# -> unI64 93888 + '\x16ea6'# -> unI64 93889 + '\x16ea7'# -> unI64 93890 + '\x16ea8'# -> unI64 93891 + '\x16ea9'# -> unI64 93892 + '\x16eaa'# -> unI64 93893 + '\x16eab'# -> unI64 93894 + '\x16eac'# -> unI64 93895 + '\x16ead'# -> unI64 93896 + '\x16eae'# -> unI64 93897 + '\x16eaf'# -> unI64 93898 + '\x16eb0'# -> unI64 93899 + '\x16eb1'# -> unI64 93900 + '\x16eb2'# -> unI64 93901 + '\x16eb3'# -> unI64 93902 + '\x16eb4'# -> unI64 93903 + '\x16eb5'# -> unI64 93904 + '\x16eb6'# -> unI64 93905 + '\x16eb7'# -> unI64 93906 + '\x16eb8'# -> unI64 93907 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 +titleMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE titleMapping #-} +titleMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172563 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909574 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201030 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492486 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575494 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108806 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269715 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269715 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985653 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819844 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168260 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751172 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819854 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945476 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 8072 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 8073 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 8074 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 8075 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 8076 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 8077 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 8078 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 8079 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 8088 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 8089 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 8090 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 8091 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 8092 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 8093 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 8094 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 8095 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 8104 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 8105 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 8106 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 8107 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 8108 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 8109 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 8110 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 8111 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 8124 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 8140 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 8188 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1755324346 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1755317126 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1755324362 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1755317129 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1755324410 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1755317135 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 3681166678819729 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 3681166678819735 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 3681166678819753 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c4'# -> unI64 453 + '\x01c6'# -> unI64 453 + '\x01c7'# -> unI64 456 + '\x01c9'# -> unI64 456 + '\x01ca'# -> unI64 459 + '\x01cc'# -> unI64 459 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f1'# -> unI64 498 + '\x01f3'# -> unI64 498 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +foldMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE foldMapping #-} +foldMapping = \case + -- LATIN CAPITAL LETTER A + '\x0041'# -> unI64 97 + -- LATIN CAPITAL LETTER B + '\x0042'# -> unI64 98 + -- LATIN CAPITAL LETTER C + '\x0043'# -> unI64 99 + -- LATIN CAPITAL LETTER D + '\x0044'# -> unI64 100 + -- LATIN CAPITAL LETTER E + '\x0045'# -> unI64 101 + -- LATIN CAPITAL LETTER F + '\x0046'# -> unI64 102 + -- LATIN CAPITAL LETTER G + '\x0047'# -> unI64 103 + -- LATIN CAPITAL LETTER H + '\x0048'# -> unI64 104 + -- LATIN CAPITAL LETTER I + '\x0049'# -> unI64 105 + -- LATIN CAPITAL LETTER J + '\x004a'# -> unI64 106 + -- LATIN CAPITAL LETTER K + '\x004b'# -> unI64 107 + -- LATIN CAPITAL LETTER L + '\x004c'# -> unI64 108 + -- LATIN CAPITAL LETTER M + '\x004d'# -> unI64 109 + -- LATIN CAPITAL LETTER N + '\x004e'# -> unI64 110 + -- LATIN CAPITAL LETTER O + '\x004f'# -> unI64 111 + -- LATIN CAPITAL LETTER P + '\x0050'# -> unI64 112 + -- LATIN CAPITAL LETTER Q + '\x0051'# -> unI64 113 + -- LATIN CAPITAL LETTER R + '\x0052'# -> unI64 114 + -- LATIN CAPITAL LETTER S + '\x0053'# -> unI64 115 + -- LATIN CAPITAL LETTER T + '\x0054'# -> unI64 116 + -- LATIN CAPITAL LETTER U + '\x0055'# -> unI64 117 + -- LATIN CAPITAL LETTER V + '\x0056'# -> unI64 118 + -- LATIN CAPITAL LETTER W + '\x0057'# -> unI64 119 + -- LATIN CAPITAL LETTER X + '\x0058'# -> unI64 120 + -- LATIN CAPITAL LETTER Y + '\x0059'# -> unI64 121 + -- LATIN CAPITAL LETTER Z + '\x005a'# -> unI64 122 + -- MICRO SIGN + '\x00b5'# -> unI64 956 + -- LATIN CAPITAL LETTER A WITH GRAVE + '\x00c0'# -> unI64 224 + -- LATIN CAPITAL LETTER A WITH ACUTE + '\x00c1'# -> unI64 225 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX + '\x00c2'# -> unI64 226 + -- LATIN CAPITAL LETTER A WITH TILDE + '\x00c3'# -> unI64 227 + -- LATIN CAPITAL LETTER A WITH DIAERESIS + '\x00c4'# -> unI64 228 + -- LATIN CAPITAL LETTER A WITH RING ABOVE + '\x00c5'# -> unI64 229 + -- LATIN CAPITAL LETTER AE + '\x00c6'# -> unI64 230 + -- LATIN CAPITAL LETTER C WITH CEDILLA + '\x00c7'# -> unI64 231 + -- LATIN CAPITAL LETTER E WITH GRAVE + '\x00c8'# -> unI64 232 + -- LATIN CAPITAL LETTER E WITH ACUTE + '\x00c9'# -> unI64 233 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX + '\x00ca'# -> unI64 234 + -- LATIN CAPITAL LETTER E WITH DIAERESIS + '\x00cb'# -> unI64 235 + -- LATIN CAPITAL LETTER I WITH GRAVE + '\x00cc'# -> unI64 236 + -- LATIN CAPITAL LETTER I WITH ACUTE + '\x00cd'# -> unI64 237 + -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX + '\x00ce'# -> unI64 238 + -- LATIN CAPITAL LETTER I WITH DIAERESIS + '\x00cf'# -> unI64 239 + -- LATIN CAPITAL LETTER ETH + '\x00d0'# -> unI64 240 + -- LATIN CAPITAL LETTER N WITH TILDE + '\x00d1'# -> unI64 241 + -- LATIN CAPITAL LETTER O WITH GRAVE + '\x00d2'# -> unI64 242 + -- LATIN CAPITAL LETTER O WITH ACUTE + '\x00d3'# -> unI64 243 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX + '\x00d4'# -> unI64 244 + -- LATIN CAPITAL LETTER O WITH TILDE + '\x00d5'# -> unI64 245 + -- LATIN CAPITAL LETTER O WITH DIAERESIS + '\x00d6'# -> unI64 246 + -- LATIN CAPITAL LETTER O WITH STROKE + '\x00d8'# -> unI64 248 + -- LATIN CAPITAL LETTER U WITH GRAVE + '\x00d9'# -> unI64 249 + -- LATIN CAPITAL LETTER U WITH ACUTE + '\x00da'# -> unI64 250 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX + '\x00db'# -> unI64 251 + -- LATIN CAPITAL LETTER U WITH DIAERESIS + '\x00dc'# -> unI64 252 + -- LATIN CAPITAL LETTER Y WITH ACUTE + '\x00dd'# -> unI64 253 + -- LATIN CAPITAL LETTER THORN + '\x00de'# -> unI64 254 + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH MACRON + '\x0100'# -> unI64 257 + -- LATIN CAPITAL LETTER A WITH BREVE + '\x0102'# -> unI64 259 + -- LATIN CAPITAL LETTER A WITH OGONEK + '\x0104'# -> unI64 261 + -- LATIN CAPITAL LETTER C WITH ACUTE + '\x0106'# -> unI64 263 + -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX + '\x0108'# -> unI64 265 + -- LATIN CAPITAL LETTER C WITH DOT ABOVE + '\x010a'# -> unI64 267 + -- LATIN CAPITAL LETTER C WITH CARON + '\x010c'# -> unI64 269 + -- LATIN CAPITAL LETTER D WITH CARON + '\x010e'# -> unI64 271 + -- LATIN CAPITAL LETTER D WITH STROKE + '\x0110'# -> unI64 273 + -- LATIN CAPITAL LETTER E WITH MACRON + '\x0112'# -> unI64 275 + -- LATIN CAPITAL LETTER E WITH BREVE + '\x0114'# -> unI64 277 + -- LATIN CAPITAL LETTER E WITH DOT ABOVE + '\x0116'# -> unI64 279 + -- LATIN CAPITAL LETTER E WITH OGONEK + '\x0118'# -> unI64 281 + -- LATIN CAPITAL LETTER E WITH CARON + '\x011a'# -> unI64 283 + -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX + '\x011c'# -> unI64 285 + -- LATIN CAPITAL LETTER G WITH BREVE + '\x011e'# -> unI64 287 + -- LATIN CAPITAL LETTER G WITH DOT ABOVE + '\x0120'# -> unI64 289 + -- LATIN CAPITAL LETTER G WITH CEDILLA + '\x0122'# -> unI64 291 + -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX + '\x0124'# -> unI64 293 + -- LATIN CAPITAL LETTER H WITH STROKE + '\x0126'# -> unI64 295 + -- LATIN CAPITAL LETTER I WITH TILDE + '\x0128'# -> unI64 297 + -- LATIN CAPITAL LETTER I WITH MACRON + '\x012a'# -> unI64 299 + -- LATIN CAPITAL LETTER I WITH BREVE + '\x012c'# -> unI64 301 + -- LATIN CAPITAL LETTER I WITH OGONEK + '\x012e'# -> unI64 303 + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- LATIN CAPITAL LIGATURE IJ + '\x0132'# -> unI64 307 + -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX + '\x0134'# -> unI64 309 + -- LATIN CAPITAL LETTER K WITH CEDILLA + '\x0136'# -> unI64 311 + -- LATIN CAPITAL LETTER L WITH ACUTE + '\x0139'# -> unI64 314 + -- LATIN CAPITAL LETTER L WITH CEDILLA + '\x013b'# -> unI64 316 + -- LATIN CAPITAL LETTER L WITH CARON + '\x013d'# -> unI64 318 + -- LATIN CAPITAL LETTER L WITH MIDDLE DOT + '\x013f'# -> unI64 320 + -- LATIN CAPITAL LETTER L WITH STROKE + '\x0141'# -> unI64 322 + -- LATIN CAPITAL LETTER N WITH ACUTE + '\x0143'# -> unI64 324 + -- LATIN CAPITAL LETTER N WITH CEDILLA + '\x0145'# -> unI64 326 + -- LATIN CAPITAL LETTER N WITH CARON + '\x0147'# -> unI64 328 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 230687420 + -- LATIN CAPITAL LETTER ENG + '\x014a'# -> unI64 331 + -- LATIN CAPITAL LETTER O WITH MACRON + '\x014c'# -> unI64 333 + -- LATIN CAPITAL LETTER O WITH BREVE + '\x014e'# -> unI64 335 + -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + '\x0150'# -> unI64 337 + -- LATIN CAPITAL LIGATURE OE + '\x0152'# -> unI64 339 + -- LATIN CAPITAL LETTER R WITH ACUTE + '\x0154'# -> unI64 341 + -- LATIN CAPITAL LETTER R WITH CEDILLA + '\x0156'# -> unI64 343 + -- LATIN CAPITAL LETTER R WITH CARON + '\x0158'# -> unI64 345 + -- LATIN CAPITAL LETTER S WITH ACUTE + '\x015a'# -> unI64 347 + -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX + '\x015c'# -> unI64 349 + -- LATIN CAPITAL LETTER S WITH CEDILLA + '\x015e'# -> unI64 351 + -- LATIN CAPITAL LETTER S WITH CARON + '\x0160'# -> unI64 353 + -- LATIN CAPITAL LETTER T WITH CEDILLA + '\x0162'# -> unI64 355 + -- LATIN CAPITAL LETTER T WITH CARON + '\x0164'# -> unI64 357 + -- LATIN CAPITAL LETTER T WITH STROKE + '\x0166'# -> unI64 359 + -- LATIN CAPITAL LETTER U WITH TILDE + '\x0168'# -> unI64 361 + -- LATIN CAPITAL LETTER U WITH MACRON + '\x016a'# -> unI64 363 + -- LATIN CAPITAL LETTER U WITH BREVE + '\x016c'# -> unI64 365 + -- LATIN CAPITAL LETTER U WITH RING ABOVE + '\x016e'# -> unI64 367 + -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + '\x0170'# -> unI64 369 + -- LATIN CAPITAL LETTER U WITH OGONEK + '\x0172'# -> unI64 371 + -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX + '\x0174'# -> unI64 373 + -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + '\x0176'# -> unI64 375 + -- LATIN CAPITAL LETTER Y WITH DIAERESIS + '\x0178'# -> unI64 255 + -- LATIN CAPITAL LETTER Z WITH ACUTE + '\x0179'# -> unI64 378 + -- LATIN CAPITAL LETTER Z WITH DOT ABOVE + '\x017b'# -> unI64 380 + -- LATIN CAPITAL LETTER Z WITH CARON + '\x017d'# -> unI64 382 + -- LATIN SMALL LETTER LONG S + '\x017f'# -> unI64 115 + -- LATIN CAPITAL LETTER B WITH HOOK + '\x0181'# -> unI64 595 + -- LATIN CAPITAL LETTER B WITH TOPBAR + '\x0182'# -> unI64 387 + -- LATIN CAPITAL LETTER TONE SIX + '\x0184'# -> unI64 389 + -- LATIN CAPITAL LETTER OPEN O + '\x0186'# -> unI64 596 + -- LATIN CAPITAL LETTER C WITH HOOK + '\x0187'# -> unI64 392 + -- LATIN CAPITAL LETTER AFRICAN D + '\x0189'# -> unI64 598 + -- LATIN CAPITAL LETTER D WITH HOOK + '\x018a'# -> unI64 599 + -- LATIN CAPITAL LETTER D WITH TOPBAR + '\x018b'# -> unI64 396 + -- LATIN CAPITAL LETTER REVERSED E + '\x018e'# -> unI64 477 + -- LATIN CAPITAL LETTER SCHWA + '\x018f'# -> unI64 601 + -- LATIN CAPITAL LETTER OPEN E + '\x0190'# -> unI64 603 + -- LATIN CAPITAL LETTER F WITH HOOK + '\x0191'# -> unI64 402 + -- LATIN CAPITAL LETTER G WITH HOOK + '\x0193'# -> unI64 608 + -- LATIN CAPITAL LETTER GAMMA + '\x0194'# -> unI64 611 + -- LATIN CAPITAL LETTER IOTA + '\x0196'# -> unI64 617 + -- LATIN CAPITAL LETTER I WITH STROKE + '\x0197'# -> unI64 616 + -- LATIN CAPITAL LETTER K WITH HOOK + '\x0198'# -> unI64 409 + -- LATIN CAPITAL LETTER TURNED M + '\x019c'# -> unI64 623 + -- LATIN CAPITAL LETTER N WITH LEFT HOOK + '\x019d'# -> unI64 626 + -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE + '\x019f'# -> unI64 629 + -- LATIN CAPITAL LETTER O WITH HORN + '\x01a0'# -> unI64 417 + -- LATIN CAPITAL LETTER OI + '\x01a2'# -> unI64 419 + -- LATIN CAPITAL LETTER P WITH HOOK + '\x01a4'# -> unI64 421 + -- LATIN LETTER YR + '\x01a6'# -> unI64 640 + -- LATIN CAPITAL LETTER TONE TWO + '\x01a7'# -> unI64 424 + -- LATIN CAPITAL LETTER ESH + '\x01a9'# -> unI64 643 + -- LATIN CAPITAL LETTER T WITH HOOK + '\x01ac'# -> unI64 429 + -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + '\x01ae'# -> unI64 648 + -- LATIN CAPITAL LETTER U WITH HORN + '\x01af'# -> unI64 432 + -- LATIN CAPITAL LETTER UPSILON + '\x01b1'# -> unI64 650 + -- LATIN CAPITAL LETTER V WITH HOOK + '\x01b2'# -> unI64 651 + -- LATIN CAPITAL LETTER Y WITH HOOK + '\x01b3'# -> unI64 436 + -- LATIN CAPITAL LETTER Z WITH STROKE + '\x01b5'# -> unI64 438 + -- LATIN CAPITAL LETTER EZH + '\x01b7'# -> unI64 658 + -- LATIN CAPITAL LETTER EZH REVERSED + '\x01b8'# -> unI64 441 + -- LATIN CAPITAL LETTER TONE FIVE + '\x01bc'# -> unI64 445 + -- LATIN CAPITAL LETTER DZ WITH CARON + '\x01c4'# -> unI64 454 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + '\x01c5'# -> unI64 454 + -- LATIN CAPITAL LETTER LJ + '\x01c7'# -> unI64 457 + -- LATIN CAPITAL LETTER L WITH SMALL LETTER J + '\x01c8'# -> unI64 457 + -- LATIN CAPITAL LETTER NJ + '\x01ca'# -> unI64 460 + -- LATIN CAPITAL LETTER N WITH SMALL LETTER J + '\x01cb'# -> unI64 460 + -- LATIN CAPITAL LETTER A WITH CARON + '\x01cd'# -> unI64 462 + -- LATIN CAPITAL LETTER I WITH CARON + '\x01cf'# -> unI64 464 + -- LATIN CAPITAL LETTER O WITH CARON + '\x01d1'# -> unI64 466 + -- LATIN CAPITAL LETTER U WITH CARON + '\x01d3'# -> unI64 468 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + '\x01d5'# -> unI64 470 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + '\x01d7'# -> unI64 472 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + '\x01d9'# -> unI64 474 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + '\x01db'# -> unI64 476 + -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + '\x01de'# -> unI64 479 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + '\x01e0'# -> unI64 481 + -- LATIN CAPITAL LETTER AE WITH MACRON + '\x01e2'# -> unI64 483 + -- LATIN CAPITAL LETTER G WITH STROKE + '\x01e4'# -> unI64 485 + -- LATIN CAPITAL LETTER G WITH CARON + '\x01e6'# -> unI64 487 + -- LATIN CAPITAL LETTER K WITH CARON + '\x01e8'# -> unI64 489 + -- LATIN CAPITAL LETTER O WITH OGONEK + '\x01ea'# -> unI64 491 + -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + '\x01ec'# -> unI64 493 + -- LATIN CAPITAL LETTER EZH WITH CARON + '\x01ee'# -> unI64 495 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778666 + -- LATIN CAPITAL LETTER DZ + '\x01f1'# -> unI64 499 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z + '\x01f2'# -> unI64 499 + -- LATIN CAPITAL LETTER G WITH ACUTE + '\x01f4'# -> unI64 501 + -- LATIN CAPITAL LETTER HWAIR + '\x01f6'# -> unI64 405 + -- LATIN CAPITAL LETTER WYNN + '\x01f7'# -> unI64 447 + -- LATIN CAPITAL LETTER N WITH GRAVE + '\x01f8'# -> unI64 505 + -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + '\x01fa'# -> unI64 507 + -- LATIN CAPITAL LETTER AE WITH ACUTE + '\x01fc'# -> unI64 509 + -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + '\x01fe'# -> unI64 511 + -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + '\x0200'# -> unI64 513 + -- LATIN CAPITAL LETTER A WITH INVERTED BREVE + '\x0202'# -> unI64 515 + -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + '\x0204'# -> unI64 517 + -- LATIN CAPITAL LETTER E WITH INVERTED BREVE + '\x0206'# -> unI64 519 + -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + '\x0208'# -> unI64 521 + -- LATIN CAPITAL LETTER I WITH INVERTED BREVE + '\x020a'# -> unI64 523 + -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + '\x020c'# -> unI64 525 + -- LATIN CAPITAL LETTER O WITH INVERTED BREVE + '\x020e'# -> unI64 527 + -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + '\x0210'# -> unI64 529 + -- LATIN CAPITAL LETTER R WITH INVERTED BREVE + '\x0212'# -> unI64 531 + -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + '\x0214'# -> unI64 533 + -- LATIN CAPITAL LETTER U WITH INVERTED BREVE + '\x0216'# -> unI64 535 + -- LATIN CAPITAL LETTER S WITH COMMA BELOW + '\x0218'# -> unI64 537 + -- LATIN CAPITAL LETTER T WITH COMMA BELOW + '\x021a'# -> unI64 539 + -- LATIN CAPITAL LETTER YOGH + '\x021c'# -> unI64 541 + -- LATIN CAPITAL LETTER H WITH CARON + '\x021e'# -> unI64 543 + -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + '\x0220'# -> unI64 414 + -- LATIN CAPITAL LETTER OU + '\x0222'# -> unI64 547 + -- LATIN CAPITAL LETTER Z WITH HOOK + '\x0224'# -> unI64 549 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE + '\x0226'# -> unI64 551 + -- LATIN CAPITAL LETTER E WITH CEDILLA + '\x0228'# -> unI64 553 + -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + '\x022a'# -> unI64 555 + -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON + '\x022c'# -> unI64 557 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE + '\x022e'# -> unI64 559 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + '\x0230'# -> unI64 561 + -- LATIN CAPITAL LETTER Y WITH MACRON + '\x0232'# -> unI64 563 + -- LATIN CAPITAL LETTER A WITH STROKE + '\x023a'# -> unI64 11365 + -- LATIN CAPITAL LETTER C WITH STROKE + '\x023b'# -> unI64 572 + -- LATIN CAPITAL LETTER L WITH BAR + '\x023d'# -> unI64 410 + -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + '\x023e'# -> unI64 11366 + -- LATIN CAPITAL LETTER GLOTTAL STOP + '\x0241'# -> unI64 578 + -- LATIN CAPITAL LETTER B WITH STROKE + '\x0243'# -> unI64 384 + -- LATIN CAPITAL LETTER U BAR + '\x0244'# -> unI64 649 + -- LATIN CAPITAL LETTER TURNED V + '\x0245'# -> unI64 652 + -- LATIN CAPITAL LETTER E WITH STROKE + '\x0246'# -> unI64 583 + -- LATIN CAPITAL LETTER J WITH STROKE + '\x0248'# -> unI64 585 + -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL + '\x024a'# -> unI64 587 + -- LATIN CAPITAL LETTER R WITH STROKE + '\x024c'# -> unI64 589 + -- LATIN CAPITAL LETTER Y WITH STROKE + '\x024e'# -> unI64 591 + -- COMBINING GREEK YPOGEGRAMMENI + '\x0345'# -> unI64 953 + -- GREEK CAPITAL LETTER HETA + '\x0370'# -> unI64 881 + -- GREEK CAPITAL LETTER ARCHAIC SAMPI + '\x0372'# -> unI64 883 + -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA + '\x0376'# -> unI64 887 + -- GREEK CAPITAL LETTER YOT + '\x037f'# -> unI64 1011 + -- GREEK CAPITAL LETTER ALPHA WITH TONOS + '\x0386'# -> unI64 940 + -- GREEK CAPITAL LETTER EPSILON WITH TONOS + '\x0388'# -> unI64 941 + -- GREEK CAPITAL LETTER ETA WITH TONOS + '\x0389'# -> unI64 942 + -- GREEK CAPITAL LETTER IOTA WITH TONOS + '\x038a'# -> unI64 943 + -- GREEK CAPITAL LETTER OMICRON WITH TONOS + '\x038c'# -> unI64 972 + -- GREEK CAPITAL LETTER UPSILON WITH TONOS + '\x038e'# -> unI64 973 + -- GREEK CAPITAL LETTER OMEGA WITH TONOS + '\x038f'# -> unI64 974 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429881 + -- GREEK CAPITAL LETTER ALPHA + '\x0391'# -> unI64 945 + -- GREEK CAPITAL LETTER BETA + '\x0392'# -> unI64 946 + -- GREEK CAPITAL LETTER GAMMA + '\x0393'# -> unI64 947 + -- GREEK CAPITAL LETTER DELTA + '\x0394'# -> unI64 948 + -- GREEK CAPITAL LETTER EPSILON + '\x0395'# -> unI64 949 + -- GREEK CAPITAL LETTER ZETA + '\x0396'# -> unI64 950 + -- GREEK CAPITAL LETTER ETA + '\x0397'# -> unI64 951 + -- GREEK CAPITAL LETTER THETA + '\x0398'# -> unI64 952 + -- GREEK CAPITAL LETTER IOTA + '\x0399'# -> unI64 953 + -- GREEK CAPITAL LETTER KAPPA + '\x039a'# -> unI64 954 + -- GREEK CAPITAL LETTER LAMDA + '\x039b'# -> unI64 955 + -- GREEK CAPITAL LETTER MU + '\x039c'# -> unI64 956 + -- GREEK CAPITAL LETTER NU + '\x039d'# -> unI64 957 + -- GREEK CAPITAL LETTER XI + '\x039e'# -> unI64 958 + -- GREEK CAPITAL LETTER OMICRON + '\x039f'# -> unI64 959 + -- GREEK CAPITAL LETTER PI + '\x03a0'# -> unI64 960 + -- GREEK CAPITAL LETTER RHO + '\x03a1'# -> unI64 961 + -- GREEK CAPITAL LETTER SIGMA + '\x03a3'# -> unI64 963 + -- GREEK CAPITAL LETTER TAU + '\x03a4'# -> unI64 964 + -- GREEK CAPITAL LETTER UPSILON + '\x03a5'# -> unI64 965 + -- GREEK CAPITAL LETTER PHI + '\x03a6'# -> unI64 966 + -- GREEK CAPITAL LETTER CHI + '\x03a7'# -> unI64 967 + -- GREEK CAPITAL LETTER PSI + '\x03a8'# -> unI64 968 + -- GREEK CAPITAL LETTER OMEGA + '\x03a9'# -> unI64 969 + -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + '\x03aa'# -> unI64 970 + -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + '\x03ab'# -> unI64 971 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER FINAL SIGMA + '\x03c2'# -> unI64 963 + -- GREEK CAPITAL KAI SYMBOL + '\x03cf'# -> unI64 983 + -- GREEK BETA SYMBOL + '\x03d0'# -> unI64 946 + -- GREEK THETA SYMBOL + '\x03d1'# -> unI64 952 + -- GREEK PHI SYMBOL + '\x03d5'# -> unI64 966 + -- GREEK PI SYMBOL + '\x03d6'# -> unI64 960 + -- GREEK LETTER ARCHAIC KOPPA + '\x03d8'# -> unI64 985 + -- GREEK LETTER STIGMA + '\x03da'# -> unI64 987 + -- GREEK LETTER DIGAMMA + '\x03dc'# -> unI64 989 + -- GREEK LETTER KOPPA + '\x03de'# -> unI64 991 + -- GREEK LETTER SAMPI + '\x03e0'# -> unI64 993 + -- COPTIC CAPITAL LETTER SHEI + '\x03e2'# -> unI64 995 + -- COPTIC CAPITAL LETTER FEI + '\x03e4'# -> unI64 997 + -- COPTIC CAPITAL LETTER KHEI + '\x03e6'# -> unI64 999 + -- COPTIC CAPITAL LETTER HORI + '\x03e8'# -> unI64 1001 + -- COPTIC CAPITAL LETTER GANGIA + '\x03ea'# -> unI64 1003 + -- COPTIC CAPITAL LETTER SHIMA + '\x03ec'# -> unI64 1005 + -- COPTIC CAPITAL LETTER DEI + '\x03ee'# -> unI64 1007 + -- GREEK KAPPA SYMBOL + '\x03f0'# -> unI64 954 + -- GREEK RHO SYMBOL + '\x03f1'# -> unI64 961 + -- GREEK CAPITAL THETA SYMBOL + '\x03f4'# -> unI64 952 + -- GREEK LUNATE EPSILON SYMBOL + '\x03f5'# -> unI64 949 + -- GREEK CAPITAL LETTER SHO + '\x03f7'# -> unI64 1016 + -- GREEK CAPITAL LUNATE SIGMA SYMBOL + '\x03f9'# -> unI64 1010 + -- GREEK CAPITAL LETTER SAN + '\x03fa'# -> unI64 1019 + -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL + '\x03fd'# -> unI64 891 + -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL + '\x03fe'# -> unI64 892 + -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + '\x03ff'# -> unI64 893 + -- CYRILLIC CAPITAL LETTER IE WITH GRAVE + '\x0400'# -> unI64 1104 + -- CYRILLIC CAPITAL LETTER IO + '\x0401'# -> unI64 1105 + -- CYRILLIC CAPITAL LETTER DJE + '\x0402'# -> unI64 1106 + -- CYRILLIC CAPITAL LETTER GJE + '\x0403'# -> unI64 1107 + -- CYRILLIC CAPITAL LETTER UKRAINIAN IE + '\x0404'# -> unI64 1108 + -- CYRILLIC CAPITAL LETTER DZE + '\x0405'# -> unI64 1109 + -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + '\x0406'# -> unI64 1110 + -- CYRILLIC CAPITAL LETTER YI + '\x0407'# -> unI64 1111 + -- CYRILLIC CAPITAL LETTER JE + '\x0408'# -> unI64 1112 + -- CYRILLIC CAPITAL LETTER LJE + '\x0409'# -> unI64 1113 + -- CYRILLIC CAPITAL LETTER NJE + '\x040a'# -> unI64 1114 + -- CYRILLIC CAPITAL LETTER TSHE + '\x040b'# -> unI64 1115 + -- CYRILLIC CAPITAL LETTER KJE + '\x040c'# -> unI64 1116 + -- CYRILLIC CAPITAL LETTER I WITH GRAVE + '\x040d'# -> unI64 1117 + -- CYRILLIC CAPITAL LETTER SHORT U + '\x040e'# -> unI64 1118 + -- CYRILLIC CAPITAL LETTER DZHE + '\x040f'# -> unI64 1119 + -- CYRILLIC CAPITAL LETTER A + '\x0410'# -> unI64 1072 + -- CYRILLIC CAPITAL LETTER BE + '\x0411'# -> unI64 1073 + -- CYRILLIC CAPITAL LETTER VE + '\x0412'# -> unI64 1074 + -- CYRILLIC CAPITAL LETTER GHE + '\x0413'# -> unI64 1075 + -- CYRILLIC CAPITAL LETTER DE + '\x0414'# -> unI64 1076 + -- CYRILLIC CAPITAL LETTER IE + '\x0415'# -> unI64 1077 + -- CYRILLIC CAPITAL LETTER ZHE + '\x0416'# -> unI64 1078 + -- CYRILLIC CAPITAL LETTER ZE + '\x0417'# -> unI64 1079 + -- CYRILLIC CAPITAL LETTER I + '\x0418'# -> unI64 1080 + -- CYRILLIC CAPITAL LETTER SHORT I + '\x0419'# -> unI64 1081 + -- CYRILLIC CAPITAL LETTER KA + '\x041a'# -> unI64 1082 + -- CYRILLIC CAPITAL LETTER EL + '\x041b'# -> unI64 1083 + -- CYRILLIC CAPITAL LETTER EM + '\x041c'# -> unI64 1084 + -- CYRILLIC CAPITAL LETTER EN + '\x041d'# -> unI64 1085 + -- CYRILLIC CAPITAL LETTER O + '\x041e'# -> unI64 1086 + -- CYRILLIC CAPITAL LETTER PE + '\x041f'# -> unI64 1087 + -- CYRILLIC CAPITAL LETTER ER + '\x0420'# -> unI64 1088 + -- CYRILLIC CAPITAL LETTER ES + '\x0421'# -> unI64 1089 + -- CYRILLIC CAPITAL LETTER TE + '\x0422'# -> unI64 1090 + -- CYRILLIC CAPITAL LETTER U + '\x0423'# -> unI64 1091 + -- CYRILLIC CAPITAL LETTER EF + '\x0424'# -> unI64 1092 + -- CYRILLIC CAPITAL LETTER HA + '\x0425'# -> unI64 1093 + -- CYRILLIC CAPITAL LETTER TSE + '\x0426'# -> unI64 1094 + -- CYRILLIC CAPITAL LETTER CHE + '\x0427'# -> unI64 1095 + -- CYRILLIC CAPITAL LETTER SHA + '\x0428'# -> unI64 1096 + -- CYRILLIC CAPITAL LETTER SHCHA + '\x0429'# -> unI64 1097 + -- CYRILLIC CAPITAL LETTER HARD SIGN + '\x042a'# -> unI64 1098 + -- CYRILLIC CAPITAL LETTER YERU + '\x042b'# -> unI64 1099 + -- CYRILLIC CAPITAL LETTER SOFT SIGN + '\x042c'# -> unI64 1100 + -- CYRILLIC CAPITAL LETTER E + '\x042d'# -> unI64 1101 + -- CYRILLIC CAPITAL LETTER YU + '\x042e'# -> unI64 1102 + -- CYRILLIC CAPITAL LETTER YA + '\x042f'# -> unI64 1103 + -- CYRILLIC CAPITAL LETTER OMEGA + '\x0460'# -> unI64 1121 + -- CYRILLIC CAPITAL LETTER YAT + '\x0462'# -> unI64 1123 + -- CYRILLIC CAPITAL LETTER IOTIFIED E + '\x0464'# -> unI64 1125 + -- CYRILLIC CAPITAL LETTER LITTLE YUS + '\x0466'# -> unI64 1127 + -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS + '\x0468'# -> unI64 1129 + -- CYRILLIC CAPITAL LETTER BIG YUS + '\x046a'# -> unI64 1131 + -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS + '\x046c'# -> unI64 1133 + -- CYRILLIC CAPITAL LETTER KSI + '\x046e'# -> unI64 1135 + -- CYRILLIC CAPITAL LETTER PSI + '\x0470'# -> unI64 1137 + -- CYRILLIC CAPITAL LETTER FITA + '\x0472'# -> unI64 1139 + -- CYRILLIC CAPITAL LETTER IZHITSA + '\x0474'# -> unI64 1141 + -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + '\x0476'# -> unI64 1143 + -- CYRILLIC CAPITAL LETTER UK + '\x0478'# -> unI64 1145 + -- CYRILLIC CAPITAL LETTER ROUND OMEGA + '\x047a'# -> unI64 1147 + -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO + '\x047c'# -> unI64 1149 + -- CYRILLIC CAPITAL LETTER OT + '\x047e'# -> unI64 1151 + -- CYRILLIC CAPITAL LETTER KOPPA + '\x0480'# -> unI64 1153 + -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL + '\x048a'# -> unI64 1163 + -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN + '\x048c'# -> unI64 1165 + -- CYRILLIC CAPITAL LETTER ER WITH TICK + '\x048e'# -> unI64 1167 + -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN + '\x0490'# -> unI64 1169 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE + '\x0492'# -> unI64 1171 + -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK + '\x0494'# -> unI64 1173 + -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + '\x0496'# -> unI64 1175 + -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + '\x0498'# -> unI64 1177 + -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER + '\x049a'# -> unI64 1179 + -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + '\x049c'# -> unI64 1181 + -- CYRILLIC CAPITAL LETTER KA WITH STROKE + '\x049e'# -> unI64 1183 + -- CYRILLIC CAPITAL LETTER BASHKIR KA + '\x04a0'# -> unI64 1185 + -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER + '\x04a2'# -> unI64 1187 + -- CYRILLIC CAPITAL LIGATURE EN GHE + '\x04a4'# -> unI64 1189 + -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK + '\x04a6'# -> unI64 1191 + -- CYRILLIC CAPITAL LETTER ABKHASIAN HA + '\x04a8'# -> unI64 1193 + -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER + '\x04aa'# -> unI64 1195 + -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER + '\x04ac'# -> unI64 1197 + -- CYRILLIC CAPITAL LETTER STRAIGHT U + '\x04ae'# -> unI64 1199 + -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + '\x04b0'# -> unI64 1201 + -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER + '\x04b2'# -> unI64 1203 + -- CYRILLIC CAPITAL LIGATURE TE TSE + '\x04b4'# -> unI64 1205 + -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + '\x04b6'# -> unI64 1207 + -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + '\x04b8'# -> unI64 1209 + -- CYRILLIC CAPITAL LETTER SHHA + '\x04ba'# -> unI64 1211 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE + '\x04bc'# -> unI64 1213 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER + '\x04be'# -> unI64 1215 + -- CYRILLIC LETTER PALOCHKA + '\x04c0'# -> unI64 1231 + -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE + '\x04c1'# -> unI64 1218 + -- CYRILLIC CAPITAL LETTER KA WITH HOOK + '\x04c3'# -> unI64 1220 + -- CYRILLIC CAPITAL LETTER EL WITH TAIL + '\x04c5'# -> unI64 1222 + -- CYRILLIC CAPITAL LETTER EN WITH HOOK + '\x04c7'# -> unI64 1224 + -- CYRILLIC CAPITAL LETTER EN WITH TAIL + '\x04c9'# -> unI64 1226 + -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE + '\x04cb'# -> unI64 1228 + -- CYRILLIC CAPITAL LETTER EM WITH TAIL + '\x04cd'# -> unI64 1230 + -- CYRILLIC CAPITAL LETTER A WITH BREVE + '\x04d0'# -> unI64 1233 + -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS + '\x04d2'# -> unI64 1235 + -- CYRILLIC CAPITAL LIGATURE A IE + '\x04d4'# -> unI64 1237 + -- CYRILLIC CAPITAL LETTER IE WITH BREVE + '\x04d6'# -> unI64 1239 + -- CYRILLIC CAPITAL LETTER SCHWA + '\x04d8'# -> unI64 1241 + -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + '\x04da'# -> unI64 1243 + -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + '\x04dc'# -> unI64 1245 + -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + '\x04de'# -> unI64 1247 + -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE + '\x04e0'# -> unI64 1249 + -- CYRILLIC CAPITAL LETTER I WITH MACRON + '\x04e2'# -> unI64 1251 + -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS + '\x04e4'# -> unI64 1253 + -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS + '\x04e6'# -> unI64 1255 + -- CYRILLIC CAPITAL LETTER BARRED O + '\x04e8'# -> unI64 1257 + -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + '\x04ea'# -> unI64 1259 + -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS + '\x04ec'# -> unI64 1261 + -- CYRILLIC CAPITAL LETTER U WITH MACRON + '\x04ee'# -> unI64 1263 + -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS + '\x04f0'# -> unI64 1265 + -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + '\x04f2'# -> unI64 1267 + -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + '\x04f4'# -> unI64 1269 + -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER + '\x04f6'# -> unI64 1271 + -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + '\x04f8'# -> unI64 1273 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK + '\x04fa'# -> unI64 1275 + -- CYRILLIC CAPITAL LETTER HA WITH HOOK + '\x04fc'# -> unI64 1277 + -- CYRILLIC CAPITAL LETTER HA WITH STROKE + '\x04fe'# -> unI64 1279 + -- CYRILLIC CAPITAL LETTER KOMI DE + '\x0500'# -> unI64 1281 + -- CYRILLIC CAPITAL LETTER KOMI DJE + '\x0502'# -> unI64 1283 + -- CYRILLIC CAPITAL LETTER KOMI ZJE + '\x0504'# -> unI64 1285 + -- CYRILLIC CAPITAL LETTER KOMI DZJE + '\x0506'# -> unI64 1287 + -- CYRILLIC CAPITAL LETTER KOMI LJE + '\x0508'# -> unI64 1289 + -- CYRILLIC CAPITAL LETTER KOMI NJE + '\x050a'# -> unI64 1291 + -- CYRILLIC CAPITAL LETTER KOMI SJE + '\x050c'# -> unI64 1293 + -- CYRILLIC CAPITAL LETTER KOMI TJE + '\x050e'# -> unI64 1295 + -- CYRILLIC CAPITAL LETTER REVERSED ZE + '\x0510'# -> unI64 1297 + -- CYRILLIC CAPITAL LETTER EL WITH HOOK + '\x0512'# -> unI64 1299 + -- CYRILLIC CAPITAL LETTER LHA + '\x0514'# -> unI64 1301 + -- CYRILLIC CAPITAL LETTER RHA + '\x0516'# -> unI64 1303 + -- CYRILLIC CAPITAL LETTER YAE + '\x0518'# -> unI64 1305 + -- CYRILLIC CAPITAL LETTER QA + '\x051a'# -> unI64 1307 + -- CYRILLIC CAPITAL LETTER WE + '\x051c'# -> unI64 1309 + -- CYRILLIC CAPITAL LETTER ALEUT KA + '\x051e'# -> unI64 1311 + -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK + '\x0520'# -> unI64 1313 + -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK + '\x0522'# -> unI64 1315 + -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER + '\x0524'# -> unI64 1317 + -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER + '\x0526'# -> unI64 1319 + -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK + '\x0528'# -> unI64 1321 + -- CYRILLIC CAPITAL LETTER DZZHE + '\x052a'# -> unI64 1323 + -- CYRILLIC CAPITAL LETTER DCHE + '\x052c'# -> unI64 1325 + -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER + '\x052e'# -> unI64 1327 + -- ARMENIAN CAPITAL LETTER AYB + '\x0531'# -> unI64 1377 + -- ARMENIAN CAPITAL LETTER BEN + '\x0532'# -> unI64 1378 + -- ARMENIAN CAPITAL LETTER GIM + '\x0533'# -> unI64 1379 + -- ARMENIAN CAPITAL LETTER DA + '\x0534'# -> unI64 1380 + -- ARMENIAN CAPITAL LETTER ECH + '\x0535'# -> unI64 1381 + -- ARMENIAN CAPITAL LETTER ZA + '\x0536'# -> unI64 1382 + -- ARMENIAN CAPITAL LETTER EH + '\x0537'# -> unI64 1383 + -- ARMENIAN CAPITAL LETTER ET + '\x0538'# -> unI64 1384 + -- ARMENIAN CAPITAL LETTER TO + '\x0539'# -> unI64 1385 + -- ARMENIAN CAPITAL LETTER ZHE + '\x053a'# -> unI64 1386 + -- ARMENIAN CAPITAL LETTER INI + '\x053b'# -> unI64 1387 + -- ARMENIAN CAPITAL LETTER LIWN + '\x053c'# -> unI64 1388 + -- ARMENIAN CAPITAL LETTER XEH + '\x053d'# -> unI64 1389 + -- ARMENIAN CAPITAL LETTER CA + '\x053e'# -> unI64 1390 + -- ARMENIAN CAPITAL LETTER KEN + '\x053f'# -> unI64 1391 + -- ARMENIAN CAPITAL LETTER HO + '\x0540'# -> unI64 1392 + -- ARMENIAN CAPITAL LETTER JA + '\x0541'# -> unI64 1393 + -- ARMENIAN CAPITAL LETTER GHAD + '\x0542'# -> unI64 1394 + -- ARMENIAN CAPITAL LETTER CHEH + '\x0543'# -> unI64 1395 + -- ARMENIAN CAPITAL LETTER MEN + '\x0544'# -> unI64 1396 + -- ARMENIAN CAPITAL LETTER YI + '\x0545'# -> unI64 1397 + -- ARMENIAN CAPITAL LETTER NOW + '\x0546'# -> unI64 1398 + -- ARMENIAN CAPITAL LETTER SHA + '\x0547'# -> unI64 1399 + -- ARMENIAN CAPITAL LETTER VO + '\x0548'# -> unI64 1400 + -- ARMENIAN CAPITAL LETTER CHA + '\x0549'# -> unI64 1401 + -- ARMENIAN CAPITAL LETTER PEH + '\x054a'# -> unI64 1402 + -- ARMENIAN CAPITAL LETTER JHEH + '\x054b'# -> unI64 1403 + -- ARMENIAN CAPITAL LETTER RA + '\x054c'# -> unI64 1404 + -- ARMENIAN CAPITAL LETTER SEH + '\x054d'# -> unI64 1405 + -- ARMENIAN CAPITAL LETTER VEW + '\x054e'# -> unI64 1406 + -- ARMENIAN CAPITAL LETTER TIWN + '\x054f'# -> unI64 1407 + -- ARMENIAN CAPITAL LETTER REH + '\x0550'# -> unI64 1408 + -- ARMENIAN CAPITAL LETTER CO + '\x0551'# -> unI64 1409 + -- ARMENIAN CAPITAL LETTER YIWN + '\x0552'# -> unI64 1410 + -- ARMENIAN CAPITAL LETTER PIWR + '\x0553'# -> unI64 1411 + -- ARMENIAN CAPITAL LETTER KEH + '\x0554'# -> unI64 1412 + -- ARMENIAN CAPITAL LETTER OH + '\x0555'# -> unI64 1413 + -- ARMENIAN CAPITAL LETTER FEH + '\x0556'# -> unI64 1414 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985701 + -- GEORGIAN CAPITAL LETTER AN + '\x10a0'# -> unI64 11520 + -- GEORGIAN CAPITAL LETTER BAN + '\x10a1'# -> unI64 11521 + -- GEORGIAN CAPITAL LETTER GAN + '\x10a2'# -> unI64 11522 + -- GEORGIAN CAPITAL LETTER DON + '\x10a3'# -> unI64 11523 + -- GEORGIAN CAPITAL LETTER EN + '\x10a4'# -> unI64 11524 + -- GEORGIAN CAPITAL LETTER VIN + '\x10a5'# -> unI64 11525 + -- GEORGIAN CAPITAL LETTER ZEN + '\x10a6'# -> unI64 11526 + -- GEORGIAN CAPITAL LETTER TAN + '\x10a7'# -> unI64 11527 + -- GEORGIAN CAPITAL LETTER IN + '\x10a8'# -> unI64 11528 + -- GEORGIAN CAPITAL LETTER KAN + '\x10a9'# -> unI64 11529 + -- GEORGIAN CAPITAL LETTER LAS + '\x10aa'# -> unI64 11530 + -- GEORGIAN CAPITAL LETTER MAN + '\x10ab'# -> unI64 11531 + -- GEORGIAN CAPITAL LETTER NAR + '\x10ac'# -> unI64 11532 + -- GEORGIAN CAPITAL LETTER ON + '\x10ad'# -> unI64 11533 + -- GEORGIAN CAPITAL LETTER PAR + '\x10ae'# -> unI64 11534 + -- GEORGIAN CAPITAL LETTER ZHAR + '\x10af'# -> unI64 11535 + -- GEORGIAN CAPITAL LETTER RAE + '\x10b0'# -> unI64 11536 + -- GEORGIAN CAPITAL LETTER SAN + '\x10b1'# -> unI64 11537 + -- GEORGIAN CAPITAL LETTER TAR + '\x10b2'# -> unI64 11538 + -- GEORGIAN CAPITAL LETTER UN + '\x10b3'# -> unI64 11539 + -- GEORGIAN CAPITAL LETTER PHAR + '\x10b4'# -> unI64 11540 + -- GEORGIAN CAPITAL LETTER KHAR + '\x10b5'# -> unI64 11541 + -- GEORGIAN CAPITAL LETTER GHAN + '\x10b6'# -> unI64 11542 + -- GEORGIAN CAPITAL LETTER QAR + '\x10b7'# -> unI64 11543 + -- GEORGIAN CAPITAL LETTER SHIN + '\x10b8'# -> unI64 11544 + -- GEORGIAN CAPITAL LETTER CHIN + '\x10b9'# -> unI64 11545 + -- GEORGIAN CAPITAL LETTER CAN + '\x10ba'# -> unI64 11546 + -- GEORGIAN CAPITAL LETTER JIL + '\x10bb'# -> unI64 11547 + -- GEORGIAN CAPITAL LETTER CIL + '\x10bc'# -> unI64 11548 + -- GEORGIAN CAPITAL LETTER CHAR + '\x10bd'# -> unI64 11549 + -- GEORGIAN CAPITAL LETTER XAN + '\x10be'# -> unI64 11550 + -- GEORGIAN CAPITAL LETTER JHAN + '\x10bf'# -> unI64 11551 + -- GEORGIAN CAPITAL LETTER HAE + '\x10c0'# -> unI64 11552 + -- GEORGIAN CAPITAL LETTER HE + '\x10c1'# -> unI64 11553 + -- GEORGIAN CAPITAL LETTER HIE + '\x10c2'# -> unI64 11554 + -- GEORGIAN CAPITAL LETTER WE + '\x10c3'# -> unI64 11555 + -- GEORGIAN CAPITAL LETTER HAR + '\x10c4'# -> unI64 11556 + -- GEORGIAN CAPITAL LETTER HOE + '\x10c5'# -> unI64 11557 + -- GEORGIAN CAPITAL LETTER YN + '\x10c7'# -> unI64 11559 + -- GEORGIAN CAPITAL LETTER AEN + '\x10cd'# -> unI64 11565 + -- CHEROKEE SMALL LETTER YE + '\x13f8'# -> unI64 5104 + -- CHEROKEE SMALL LETTER YI + '\x13f9'# -> unI64 5105 + -- CHEROKEE SMALL LETTER YO + '\x13fa'# -> unI64 5106 + -- CHEROKEE SMALL LETTER YU + '\x13fb'# -> unI64 5107 + -- CHEROKEE SMALL LETTER YV + '\x13fc'# -> unI64 5108 + -- CHEROKEE SMALL LETTER MV + '\x13fd'# -> unI64 5109 + -- CYRILLIC SMALL LETTER ROUNDED VE + '\x1c80'# -> unI64 1074 + -- CYRILLIC SMALL LETTER LONG-LEGGED DE + '\x1c81'# -> unI64 1076 + -- CYRILLIC SMALL LETTER NARROW O + '\x1c82'# -> unI64 1086 + -- CYRILLIC SMALL LETTER WIDE ES + '\x1c83'# -> unI64 1089 + -- CYRILLIC SMALL LETTER TALL TE + '\x1c84'# -> unI64 1090 + -- CYRILLIC SMALL LETTER THREE-LEGGED TE + '\x1c85'# -> unI64 1090 + -- CYRILLIC SMALL LETTER TALL HARD SIGN + '\x1c86'# -> unI64 1098 + -- CYRILLIC SMALL LETTER TALL YAT + '\x1c87'# -> unI64 1123 + -- CYRILLIC SMALL LETTER UNBLENDED UK + '\x1c88'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER TJE + '\x1c89'# -> unI64 7306 + -- GEORGIAN MTAVRULI CAPITAL LETTER AN + '\x1c90'# -> unI64 4304 + -- GEORGIAN MTAVRULI CAPITAL LETTER BAN + '\x1c91'# -> unI64 4305 + -- GEORGIAN MTAVRULI CAPITAL LETTER GAN + '\x1c92'# -> unI64 4306 + -- GEORGIAN MTAVRULI CAPITAL LETTER DON + '\x1c93'# -> unI64 4307 + -- GEORGIAN MTAVRULI CAPITAL LETTER EN + '\x1c94'# -> unI64 4308 + -- GEORGIAN MTAVRULI CAPITAL LETTER VIN + '\x1c95'# -> unI64 4309 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN + '\x1c96'# -> unI64 4310 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAN + '\x1c97'# -> unI64 4311 + -- GEORGIAN MTAVRULI CAPITAL LETTER IN + '\x1c98'# -> unI64 4312 + -- GEORGIAN MTAVRULI CAPITAL LETTER KAN + '\x1c99'# -> unI64 4313 + -- GEORGIAN MTAVRULI CAPITAL LETTER LAS + '\x1c9a'# -> unI64 4314 + -- GEORGIAN MTAVRULI CAPITAL LETTER MAN + '\x1c9b'# -> unI64 4315 + -- GEORGIAN MTAVRULI CAPITAL LETTER NAR + '\x1c9c'# -> unI64 4316 + -- GEORGIAN MTAVRULI CAPITAL LETTER ON + '\x1c9d'# -> unI64 4317 + -- GEORGIAN MTAVRULI CAPITAL LETTER PAR + '\x1c9e'# -> unI64 4318 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR + '\x1c9f'# -> unI64 4319 + -- GEORGIAN MTAVRULI CAPITAL LETTER RAE + '\x1ca0'# -> unI64 4320 + -- GEORGIAN MTAVRULI CAPITAL LETTER SAN + '\x1ca1'# -> unI64 4321 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAR + '\x1ca2'# -> unI64 4322 + -- GEORGIAN MTAVRULI CAPITAL LETTER UN + '\x1ca3'# -> unI64 4323 + -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR + '\x1ca4'# -> unI64 4324 + -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR + '\x1ca5'# -> unI64 4325 + -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN + '\x1ca6'# -> unI64 4326 + -- GEORGIAN MTAVRULI CAPITAL LETTER QAR + '\x1ca7'# -> unI64 4327 + -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN + '\x1ca8'# -> unI64 4328 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN + '\x1ca9'# -> unI64 4329 + -- GEORGIAN MTAVRULI CAPITAL LETTER CAN + '\x1caa'# -> unI64 4330 + -- GEORGIAN MTAVRULI CAPITAL LETTER JIL + '\x1cab'# -> unI64 4331 + -- GEORGIAN MTAVRULI CAPITAL LETTER CIL + '\x1cac'# -> unI64 4332 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR + '\x1cad'# -> unI64 4333 + -- GEORGIAN MTAVRULI CAPITAL LETTER XAN + '\x1cae'# -> unI64 4334 + -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN + '\x1caf'# -> unI64 4335 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAE + '\x1cb0'# -> unI64 4336 + -- GEORGIAN MTAVRULI CAPITAL LETTER HE + '\x1cb1'# -> unI64 4337 + -- GEORGIAN MTAVRULI CAPITAL LETTER HIE + '\x1cb2'# -> unI64 4338 + -- GEORGIAN MTAVRULI CAPITAL LETTER WE + '\x1cb3'# -> unI64 4339 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAR + '\x1cb4'# -> unI64 4340 + -- GEORGIAN MTAVRULI CAPITAL LETTER HOE + '\x1cb5'# -> unI64 4341 + -- GEORGIAN MTAVRULI CAPITAL LETTER FI + '\x1cb6'# -> unI64 4342 + -- GEORGIAN MTAVRULI CAPITAL LETTER YN + '\x1cb7'# -> unI64 4343 + -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI + '\x1cb8'# -> unI64 4344 + -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN + '\x1cb9'# -> unI64 4345 + -- GEORGIAN MTAVRULI CAPITAL LETTER AIN + '\x1cba'# -> unI64 4346 + -- GEORGIAN MTAVRULI CAPITAL LETTER AEN + '\x1cbd'# -> unI64 4349 + -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN + '\x1cbe'# -> unI64 4350 + -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + '\x1cbf'# -> unI64 4351 + -- LATIN CAPITAL LETTER A WITH RING BELOW + '\x1e00'# -> unI64 7681 + -- LATIN CAPITAL LETTER B WITH DOT ABOVE + '\x1e02'# -> unI64 7683 + -- LATIN CAPITAL LETTER B WITH DOT BELOW + '\x1e04'# -> unI64 7685 + -- LATIN CAPITAL LETTER B WITH LINE BELOW + '\x1e06'# -> unI64 7687 + -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + '\x1e08'# -> unI64 7689 + -- LATIN CAPITAL LETTER D WITH DOT ABOVE + '\x1e0a'# -> unI64 7691 + -- LATIN CAPITAL LETTER D WITH DOT BELOW + '\x1e0c'# -> unI64 7693 + -- LATIN CAPITAL LETTER D WITH LINE BELOW + '\x1e0e'# -> unI64 7695 + -- LATIN CAPITAL LETTER D WITH CEDILLA + '\x1e10'# -> unI64 7697 + -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + '\x1e12'# -> unI64 7699 + -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + '\x1e14'# -> unI64 7701 + -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + '\x1e16'# -> unI64 7703 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + '\x1e18'# -> unI64 7705 + -- LATIN CAPITAL LETTER E WITH TILDE BELOW + '\x1e1a'# -> unI64 7707 + -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + '\x1e1c'# -> unI64 7709 + -- LATIN CAPITAL LETTER F WITH DOT ABOVE + '\x1e1e'# -> unI64 7711 + -- LATIN CAPITAL LETTER G WITH MACRON + '\x1e20'# -> unI64 7713 + -- LATIN CAPITAL LETTER H WITH DOT ABOVE + '\x1e22'# -> unI64 7715 + -- LATIN CAPITAL LETTER H WITH DOT BELOW + '\x1e24'# -> unI64 7717 + -- LATIN CAPITAL LETTER H WITH DIAERESIS + '\x1e26'# -> unI64 7719 + -- LATIN CAPITAL LETTER H WITH CEDILLA + '\x1e28'# -> unI64 7721 + -- LATIN CAPITAL LETTER H WITH BREVE BELOW + '\x1e2a'# -> unI64 7723 + -- LATIN CAPITAL LETTER I WITH TILDE BELOW + '\x1e2c'# -> unI64 7725 + -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + '\x1e2e'# -> unI64 7727 + -- LATIN CAPITAL LETTER K WITH ACUTE + '\x1e30'# -> unI64 7729 + -- LATIN CAPITAL LETTER K WITH DOT BELOW + '\x1e32'# -> unI64 7731 + -- LATIN CAPITAL LETTER K WITH LINE BELOW + '\x1e34'# -> unI64 7733 + -- LATIN CAPITAL LETTER L WITH DOT BELOW + '\x1e36'# -> unI64 7735 + -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + '\x1e38'# -> unI64 7737 + -- LATIN CAPITAL LETTER L WITH LINE BELOW + '\x1e3a'# -> unI64 7739 + -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + '\x1e3c'# -> unI64 7741 + -- LATIN CAPITAL LETTER M WITH ACUTE + '\x1e3e'# -> unI64 7743 + -- LATIN CAPITAL LETTER M WITH DOT ABOVE + '\x1e40'# -> unI64 7745 + -- LATIN CAPITAL LETTER M WITH DOT BELOW + '\x1e42'# -> unI64 7747 + -- LATIN CAPITAL LETTER N WITH DOT ABOVE + '\x1e44'# -> unI64 7749 + -- LATIN CAPITAL LETTER N WITH DOT BELOW + '\x1e46'# -> unI64 7751 + -- LATIN CAPITAL LETTER N WITH LINE BELOW + '\x1e48'# -> unI64 7753 + -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + '\x1e4a'# -> unI64 7755 + -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + '\x1e4c'# -> unI64 7757 + -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + '\x1e4e'# -> unI64 7759 + -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + '\x1e50'# -> unI64 7761 + -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + '\x1e52'# -> unI64 7763 + -- LATIN CAPITAL LETTER P WITH ACUTE + '\x1e54'# -> unI64 7765 + -- LATIN CAPITAL LETTER P WITH DOT ABOVE + '\x1e56'# -> unI64 7767 + -- LATIN CAPITAL LETTER R WITH DOT ABOVE + '\x1e58'# -> unI64 7769 + -- LATIN CAPITAL LETTER R WITH DOT BELOW + '\x1e5a'# -> unI64 7771 + -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + '\x1e5c'# -> unI64 7773 + -- LATIN CAPITAL LETTER R WITH LINE BELOW + '\x1e5e'# -> unI64 7775 + -- LATIN CAPITAL LETTER S WITH DOT ABOVE + '\x1e60'# -> unI64 7777 + -- LATIN CAPITAL LETTER S WITH DOT BELOW + '\x1e62'# -> unI64 7779 + -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + '\x1e64'# -> unI64 7781 + -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + '\x1e66'# -> unI64 7783 + -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + '\x1e68'# -> unI64 7785 + -- LATIN CAPITAL LETTER T WITH DOT ABOVE + '\x1e6a'# -> unI64 7787 + -- LATIN CAPITAL LETTER T WITH DOT BELOW + '\x1e6c'# -> unI64 7789 + -- LATIN CAPITAL LETTER T WITH LINE BELOW + '\x1e6e'# -> unI64 7791 + -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + '\x1e70'# -> unI64 7793 + -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + '\x1e72'# -> unI64 7795 + -- LATIN CAPITAL LETTER U WITH TILDE BELOW + '\x1e74'# -> unI64 7797 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + '\x1e76'# -> unI64 7799 + -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + '\x1e78'# -> unI64 7801 + -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + '\x1e7a'# -> unI64 7803 + -- LATIN CAPITAL LETTER V WITH TILDE + '\x1e7c'# -> unI64 7805 + -- LATIN CAPITAL LETTER V WITH DOT BELOW + '\x1e7e'# -> unI64 7807 + -- LATIN CAPITAL LETTER W WITH GRAVE + '\x1e80'# -> unI64 7809 + -- LATIN CAPITAL LETTER W WITH ACUTE + '\x1e82'# -> unI64 7811 + -- LATIN CAPITAL LETTER W WITH DIAERESIS + '\x1e84'# -> unI64 7813 + -- LATIN CAPITAL LETTER W WITH DOT ABOVE + '\x1e86'# -> unI64 7815 + -- LATIN CAPITAL LETTER W WITH DOT BELOW + '\x1e88'# -> unI64 7817 + -- LATIN CAPITAL LETTER X WITH DOT ABOVE + '\x1e8a'# -> unI64 7819 + -- LATIN CAPITAL LETTER X WITH DIAERESIS + '\x1e8c'# -> unI64 7821 + -- LATIN CAPITAL LETTER Y WITH DOT ABOVE + '\x1e8e'# -> unI64 7823 + -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + '\x1e90'# -> unI64 7825 + -- LATIN CAPITAL LETTER Z WITH DOT BELOW + '\x1e92'# -> unI64 7827 + -- LATIN CAPITAL LETTER Z WITH LINE BELOW + '\x1e94'# -> unI64 7829 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373288 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390068 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584375 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584377 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200801 + -- LATIN SMALL LETTER LONG S WITH DOT ABOVE + '\x1e9b'# -> unI64 7777 + -- LATIN CAPITAL LETTER SHARP S + '\x1e9e'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH DOT BELOW + '\x1ea0'# -> unI64 7841 + -- LATIN CAPITAL LETTER A WITH HOOK ABOVE + '\x1ea2'# -> unI64 7843 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + '\x1ea4'# -> unI64 7845 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + '\x1ea6'# -> unI64 7847 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ea8'# -> unI64 7849 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + '\x1eaa'# -> unI64 7851 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + '\x1eac'# -> unI64 7853 + -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + '\x1eae'# -> unI64 7855 + -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + '\x1eb0'# -> unI64 7857 + -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + '\x1eb2'# -> unI64 7859 + -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE + '\x1eb4'# -> unI64 7861 + -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + '\x1eb6'# -> unI64 7863 + -- LATIN CAPITAL LETTER E WITH DOT BELOW + '\x1eb8'# -> unI64 7865 + -- LATIN CAPITAL LETTER E WITH HOOK ABOVE + '\x1eba'# -> unI64 7867 + -- LATIN CAPITAL LETTER E WITH TILDE + '\x1ebc'# -> unI64 7869 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + '\x1ebe'# -> unI64 7871 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + '\x1ec0'# -> unI64 7873 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ec2'# -> unI64 7875 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + '\x1ec4'# -> unI64 7877 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + '\x1ec6'# -> unI64 7879 + -- LATIN CAPITAL LETTER I WITH HOOK ABOVE + '\x1ec8'# -> unI64 7881 + -- LATIN CAPITAL LETTER I WITH DOT BELOW + '\x1eca'# -> unI64 7883 + -- LATIN CAPITAL LETTER O WITH DOT BELOW + '\x1ecc'# -> unI64 7885 + -- LATIN CAPITAL LETTER O WITH HOOK ABOVE + '\x1ece'# -> unI64 7887 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + '\x1ed0'# -> unI64 7889 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + '\x1ed2'# -> unI64 7891 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ed4'# -> unI64 7893 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + '\x1ed6'# -> unI64 7895 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + '\x1ed8'# -> unI64 7897 + -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE + '\x1eda'# -> unI64 7899 + -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE + '\x1edc'# -> unI64 7901 + -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + '\x1ede'# -> unI64 7903 + -- LATIN CAPITAL LETTER O WITH HORN AND TILDE + '\x1ee0'# -> unI64 7905 + -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + '\x1ee2'# -> unI64 7907 + -- LATIN CAPITAL LETTER U WITH DOT BELOW + '\x1ee4'# -> unI64 7909 + -- LATIN CAPITAL LETTER U WITH HOOK ABOVE + '\x1ee6'# -> unI64 7911 + -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE + '\x1ee8'# -> unI64 7913 + -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE + '\x1eea'# -> unI64 7915 + -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + '\x1eec'# -> unI64 7917 + -- LATIN CAPITAL LETTER U WITH HORN AND TILDE + '\x1eee'# -> unI64 7919 + -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + '\x1ef0'# -> unI64 7921 + -- LATIN CAPITAL LETTER Y WITH GRAVE + '\x1ef2'# -> unI64 7923 + -- LATIN CAPITAL LETTER Y WITH DOT BELOW + '\x1ef4'# -> unI64 7925 + -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE + '\x1ef6'# -> unI64 7927 + -- LATIN CAPITAL LETTER Y WITH TILDE + '\x1ef8'# -> unI64 7929 + -- LATIN CAPITAL LETTER MIDDLE-WELSH LL + '\x1efa'# -> unI64 7931 + -- LATIN CAPITAL LETTER MIDDLE-WELSH V + '\x1efc'# -> unI64 7933 + -- LATIN CAPITAL LETTER Y WITH LOOP + '\x1efe'# -> unI64 7935 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI + '\x1f08'# -> unI64 7936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA + '\x1f09'# -> unI64 7937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA + '\x1f0a'# -> unI64 7938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA + '\x1f0b'# -> unI64 7939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA + '\x1f0c'# -> unI64 7940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA + '\x1f0d'# -> unI64 7941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI + '\x1f0e'# -> unI64 7942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + '\x1f0f'# -> unI64 7943 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI + '\x1f18'# -> unI64 7952 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA + '\x1f19'# -> unI64 7953 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA + '\x1f1a'# -> unI64 7954 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA + '\x1f1b'# -> unI64 7955 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA + '\x1f1c'# -> unI64 7956 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + '\x1f1d'# -> unI64 7957 + -- GREEK CAPITAL LETTER ETA WITH PSILI + '\x1f28'# -> unI64 7968 + -- GREEK CAPITAL LETTER ETA WITH DASIA + '\x1f29'# -> unI64 7969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA + '\x1f2a'# -> unI64 7970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA + '\x1f2b'# -> unI64 7971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA + '\x1f2c'# -> unI64 7972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA + '\x1f2d'# -> unI64 7973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI + '\x1f2e'# -> unI64 7974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + '\x1f2f'# -> unI64 7975 + -- GREEK CAPITAL LETTER IOTA WITH PSILI + '\x1f38'# -> unI64 7984 + -- GREEK CAPITAL LETTER IOTA WITH DASIA + '\x1f39'# -> unI64 7985 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA + '\x1f3a'# -> unI64 7986 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA + '\x1f3b'# -> unI64 7987 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA + '\x1f3c'# -> unI64 7988 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA + '\x1f3d'# -> unI64 7989 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI + '\x1f3e'# -> unI64 7990 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + '\x1f3f'# -> unI64 7991 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI + '\x1f48'# -> unI64 8000 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA + '\x1f49'# -> unI64 8001 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA + '\x1f4a'# -> unI64 8002 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA + '\x1f4b'# -> unI64 8003 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA + '\x1f4c'# -> unI64 8004 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + '\x1f4d'# -> unI64 8005 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459589 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987461 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498565 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720325 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA + '\x1f59'# -> unI64 8017 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + '\x1f5b'# -> unI64 8019 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + '\x1f5d'# -> unI64 8021 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + '\x1f5f'# -> unI64 8023 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI + '\x1f68'# -> unI64 8032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA + '\x1f69'# -> unI64 8033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA + '\x1f6a'# -> unI64 8034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA + '\x1f6b'# -> unI64 8035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA + '\x1f6c'# -> unI64 8036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA + '\x1f6d'# -> unI64 8037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI + '\x1f6e'# -> unI64 8038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + '\x1f6f'# -> unI64 8039 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1998593792 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1998593793 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1998593794 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1998593795 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1998593796 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1998593797 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1998593798 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1998593799 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1998593792 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1998593793 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1998593794 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1998593795 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1998593796 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1998593797 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1998593798 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1998593799 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1998593824 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1998593825 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1998593826 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1998593827 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1998593828 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1998593829 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1998593830 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1998593831 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1998593824 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1998593825 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1998593826 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1998593827 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1998593828 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1998593829 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1998593830 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1998593831 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1998593888 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1998593889 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1998593890 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1998593891 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1998593892 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1998593893 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1998593894 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1998593895 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1998593888 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1998593889 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1998593890 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1998593891 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1998593892 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1998593893 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1998593894 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1998593895 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1998593904 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1998586801 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1998586796 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025713 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4191340074107825 + -- GREEK CAPITAL LETTER ALPHA WITH VRACHY + '\x1fb8'# -> unI64 8112 + -- GREEK CAPITAL LETTER ALPHA WITH MACRON + '\x1fb9'# -> unI64 8113 + -- GREEK CAPITAL LETTER ALPHA WITH VARIA + '\x1fba'# -> unI64 8048 + -- GREEK CAPITAL LETTER ALPHA WITH OXIA + '\x1fbb'# -> unI64 8049 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1998586801 + -- GREEK PROSGEGRAMMENI + '\x1fbe'# -> unI64 953 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1998593908 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1998586807 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1998586798 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025719 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4191340074107831 + -- GREEK CAPITAL LETTER EPSILON WITH VARIA + '\x1fc8'# -> unI64 8050 + -- GREEK CAPITAL LETTER EPSILON WITH OXIA + '\x1fc9'# -> unI64 8051 + -- GREEK CAPITAL LETTER ETA WITH VARIA + '\x1fca'# -> unI64 8052 + -- GREEK CAPITAL LETTER ETA WITH OXIA + '\x1fcb'# -> unI64 8053 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1998586807 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918777 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025721 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651641 + -- GREEK CAPITAL LETTER IOTA WITH VRACHY + '\x1fd8'# -> unI64 8144 + -- GREEK CAPITAL LETTER IOTA WITH MACRON + '\x1fd9'# -> unI64 8145 + -- GREEK CAPITAL LETTER IOTA WITH VARIA + '\x1fda'# -> unI64 8054 + -- GREEK CAPITAL LETTER IOTA WITH OXIA + '\x1fdb'# -> unI64 8055 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918789 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459585 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025733 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651653 + -- GREEK CAPITAL LETTER UPSILON WITH VRACHY + '\x1fe8'# -> unI64 8160 + -- GREEK CAPITAL LETTER UPSILON WITH MACRON + '\x1fe9'# -> unI64 8161 + -- GREEK CAPITAL LETTER UPSILON WITH VARIA + '\x1fea'# -> unI64 8058 + -- GREEK CAPITAL LETTER UPSILON WITH OXIA + '\x1feb'# -> unI64 8059 + -- GREEK CAPITAL LETTER RHO WITH DASIA + '\x1fec'# -> unI64 8165 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1998593916 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1998586825 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1998586830 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025737 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4191340074107849 + -- GREEK CAPITAL LETTER OMICRON WITH VARIA + '\x1ff8'# -> unI64 8056 + -- GREEK CAPITAL LETTER OMICRON WITH OXIA + '\x1ff9'# -> unI64 8057 + -- GREEK CAPITAL LETTER OMEGA WITH VARIA + '\x1ffa'# -> unI64 8060 + -- GREEK CAPITAL LETTER OMEGA WITH OXIA + '\x1ffb'# -> unI64 8061 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1998586825 + -- OHM SIGN + '\x2126'# -> unI64 969 + -- KELVIN SIGN + '\x212a'# -> unI64 107 + -- ANGSTROM SIGN + '\x212b'# -> unI64 229 + -- TURNED CAPITAL F + '\x2132'# -> unI64 8526 + -- ROMAN NUMERAL ONE + '\x2160'# -> unI64 8560 + -- ROMAN NUMERAL TWO + '\x2161'# -> unI64 8561 + -- ROMAN NUMERAL THREE + '\x2162'# -> unI64 8562 + -- ROMAN NUMERAL FOUR + '\x2163'# -> unI64 8563 + -- ROMAN NUMERAL FIVE + '\x2164'# -> unI64 8564 + -- ROMAN NUMERAL SIX + '\x2165'# -> unI64 8565 + -- ROMAN NUMERAL SEVEN + '\x2166'# -> unI64 8566 + -- ROMAN NUMERAL EIGHT + '\x2167'# -> unI64 8567 + -- ROMAN NUMERAL NINE + '\x2168'# -> unI64 8568 + -- ROMAN NUMERAL TEN + '\x2169'# -> unI64 8569 + -- ROMAN NUMERAL ELEVEN + '\x216a'# -> unI64 8570 + -- ROMAN NUMERAL TWELVE + '\x216b'# -> unI64 8571 + -- ROMAN NUMERAL FIFTY + '\x216c'# -> unI64 8572 + -- ROMAN NUMERAL ONE HUNDRED + '\x216d'# -> unI64 8573 + -- ROMAN NUMERAL FIVE HUNDRED + '\x216e'# -> unI64 8574 + -- ROMAN NUMERAL ONE THOUSAND + '\x216f'# -> unI64 8575 + -- ROMAN NUMERAL REVERSED ONE HUNDRED + '\x2183'# -> unI64 8580 + -- CIRCLED LATIN CAPITAL LETTER A + '\x24b6'# -> unI64 9424 + -- CIRCLED LATIN CAPITAL LETTER B + '\x24b7'# -> unI64 9425 + -- CIRCLED LATIN CAPITAL LETTER C + '\x24b8'# -> unI64 9426 + -- CIRCLED LATIN CAPITAL LETTER D + '\x24b9'# -> unI64 9427 + -- CIRCLED LATIN CAPITAL LETTER E + '\x24ba'# -> unI64 9428 + -- CIRCLED LATIN CAPITAL LETTER F + '\x24bb'# -> unI64 9429 + -- CIRCLED LATIN CAPITAL LETTER G + '\x24bc'# -> unI64 9430 + -- CIRCLED LATIN CAPITAL LETTER H + '\x24bd'# -> unI64 9431 + -- CIRCLED LATIN CAPITAL LETTER I + '\x24be'# -> unI64 9432 + -- CIRCLED LATIN CAPITAL LETTER J + '\x24bf'# -> unI64 9433 + -- CIRCLED LATIN CAPITAL LETTER K + '\x24c0'# -> unI64 9434 + -- CIRCLED LATIN CAPITAL LETTER L + '\x24c1'# -> unI64 9435 + -- CIRCLED LATIN CAPITAL LETTER M + '\x24c2'# -> unI64 9436 + -- CIRCLED LATIN CAPITAL LETTER N + '\x24c3'# -> unI64 9437 + -- CIRCLED LATIN CAPITAL LETTER O + '\x24c4'# -> unI64 9438 + -- CIRCLED LATIN CAPITAL LETTER P + '\x24c5'# -> unI64 9439 + -- CIRCLED LATIN CAPITAL LETTER Q + '\x24c6'# -> unI64 9440 + -- CIRCLED LATIN CAPITAL LETTER R + '\x24c7'# -> unI64 9441 + -- CIRCLED LATIN CAPITAL LETTER S + '\x24c8'# -> unI64 9442 + -- CIRCLED LATIN CAPITAL LETTER T + '\x24c9'# -> unI64 9443 + -- CIRCLED LATIN CAPITAL LETTER U + '\x24ca'# -> unI64 9444 + -- CIRCLED LATIN CAPITAL LETTER V + '\x24cb'# -> unI64 9445 + -- CIRCLED LATIN CAPITAL LETTER W + '\x24cc'# -> unI64 9446 + -- CIRCLED LATIN CAPITAL LETTER X + '\x24cd'# -> unI64 9447 + -- CIRCLED LATIN CAPITAL LETTER Y + '\x24ce'# -> unI64 9448 + -- CIRCLED LATIN CAPITAL LETTER Z + '\x24cf'# -> unI64 9449 + -- GLAGOLITIC CAPITAL LETTER AZU + '\x2c00'# -> unI64 11312 + -- GLAGOLITIC CAPITAL LETTER BUKY + '\x2c01'# -> unI64 11313 + -- GLAGOLITIC CAPITAL LETTER VEDE + '\x2c02'# -> unI64 11314 + -- GLAGOLITIC CAPITAL LETTER GLAGOLI + '\x2c03'# -> unI64 11315 + -- GLAGOLITIC CAPITAL LETTER DOBRO + '\x2c04'# -> unI64 11316 + -- GLAGOLITIC CAPITAL LETTER YESTU + '\x2c05'# -> unI64 11317 + -- GLAGOLITIC CAPITAL LETTER ZHIVETE + '\x2c06'# -> unI64 11318 + -- GLAGOLITIC CAPITAL LETTER DZELO + '\x2c07'# -> unI64 11319 + -- GLAGOLITIC CAPITAL LETTER ZEMLJA + '\x2c08'# -> unI64 11320 + -- GLAGOLITIC CAPITAL LETTER IZHE + '\x2c09'# -> unI64 11321 + -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE + '\x2c0a'# -> unI64 11322 + -- GLAGOLITIC CAPITAL LETTER I + '\x2c0b'# -> unI64 11323 + -- GLAGOLITIC CAPITAL LETTER DJERVI + '\x2c0c'# -> unI64 11324 + -- GLAGOLITIC CAPITAL LETTER KAKO + '\x2c0d'# -> unI64 11325 + -- GLAGOLITIC CAPITAL LETTER LJUDIJE + '\x2c0e'# -> unI64 11326 + -- GLAGOLITIC CAPITAL LETTER MYSLITE + '\x2c0f'# -> unI64 11327 + -- GLAGOLITIC CAPITAL LETTER NASHI + '\x2c10'# -> unI64 11328 + -- GLAGOLITIC CAPITAL LETTER ONU + '\x2c11'# -> unI64 11329 + -- GLAGOLITIC CAPITAL LETTER POKOJI + '\x2c12'# -> unI64 11330 + -- GLAGOLITIC CAPITAL LETTER RITSI + '\x2c13'# -> unI64 11331 + -- GLAGOLITIC CAPITAL LETTER SLOVO + '\x2c14'# -> unI64 11332 + -- GLAGOLITIC CAPITAL LETTER TVRIDO + '\x2c15'# -> unI64 11333 + -- GLAGOLITIC CAPITAL LETTER UKU + '\x2c16'# -> unI64 11334 + -- GLAGOLITIC CAPITAL LETTER FRITU + '\x2c17'# -> unI64 11335 + -- GLAGOLITIC CAPITAL LETTER HERU + '\x2c18'# -> unI64 11336 + -- GLAGOLITIC CAPITAL LETTER OTU + '\x2c19'# -> unI64 11337 + -- GLAGOLITIC CAPITAL LETTER PE + '\x2c1a'# -> unI64 11338 + -- GLAGOLITIC CAPITAL LETTER SHTA + '\x2c1b'# -> unI64 11339 + -- GLAGOLITIC CAPITAL LETTER TSI + '\x2c1c'# -> unI64 11340 + -- GLAGOLITIC CAPITAL LETTER CHRIVI + '\x2c1d'# -> unI64 11341 + -- GLAGOLITIC CAPITAL LETTER SHA + '\x2c1e'# -> unI64 11342 + -- GLAGOLITIC CAPITAL LETTER YERU + '\x2c1f'# -> unI64 11343 + -- GLAGOLITIC CAPITAL LETTER YERI + '\x2c20'# -> unI64 11344 + -- GLAGOLITIC CAPITAL LETTER YATI + '\x2c21'# -> unI64 11345 + -- GLAGOLITIC CAPITAL LETTER SPIDERY HA + '\x2c22'# -> unI64 11346 + -- GLAGOLITIC CAPITAL LETTER YU + '\x2c23'# -> unI64 11347 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS + '\x2c24'# -> unI64 11348 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL + '\x2c25'# -> unI64 11349 + -- GLAGOLITIC CAPITAL LETTER YO + '\x2c26'# -> unI64 11350 + -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS + '\x2c27'# -> unI64 11351 + -- GLAGOLITIC CAPITAL LETTER BIG YUS + '\x2c28'# -> unI64 11352 + -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS + '\x2c29'# -> unI64 11353 + -- GLAGOLITIC CAPITAL LETTER FITA + '\x2c2a'# -> unI64 11354 + -- GLAGOLITIC CAPITAL LETTER IZHITSA + '\x2c2b'# -> unI64 11355 + -- GLAGOLITIC CAPITAL LETTER SHTAPIC + '\x2c2c'# -> unI64 11356 + -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A + '\x2c2d'# -> unI64 11357 + -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE + '\x2c2e'# -> unI64 11358 + -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI + '\x2c2f'# -> unI64 11359 + -- LATIN CAPITAL LETTER L WITH DOUBLE BAR + '\x2c60'# -> unI64 11361 + -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE + '\x2c62'# -> unI64 619 + -- LATIN CAPITAL LETTER P WITH STROKE + '\x2c63'# -> unI64 7549 + -- LATIN CAPITAL LETTER R WITH TAIL + '\x2c64'# -> unI64 637 + -- LATIN CAPITAL LETTER H WITH DESCENDER + '\x2c67'# -> unI64 11368 + -- LATIN CAPITAL LETTER K WITH DESCENDER + '\x2c69'# -> unI64 11370 + -- LATIN CAPITAL LETTER Z WITH DESCENDER + '\x2c6b'# -> unI64 11372 + -- LATIN CAPITAL LETTER ALPHA + '\x2c6d'# -> unI64 593 + -- LATIN CAPITAL LETTER M WITH HOOK + '\x2c6e'# -> unI64 625 + -- LATIN CAPITAL LETTER TURNED A + '\x2c6f'# -> unI64 592 + -- LATIN CAPITAL LETTER TURNED ALPHA + '\x2c70'# -> unI64 594 + -- LATIN CAPITAL LETTER W WITH HOOK + '\x2c72'# -> unI64 11379 + -- LATIN CAPITAL LETTER HALF H + '\x2c75'# -> unI64 11382 + -- LATIN CAPITAL LETTER S WITH SWASH TAIL + '\x2c7e'# -> unI64 575 + -- LATIN CAPITAL LETTER Z WITH SWASH TAIL + '\x2c7f'# -> unI64 576 + -- COPTIC CAPITAL LETTER ALFA + '\x2c80'# -> unI64 11393 + -- COPTIC CAPITAL LETTER VIDA + '\x2c82'# -> unI64 11395 + -- COPTIC CAPITAL LETTER GAMMA + '\x2c84'# -> unI64 11397 + -- COPTIC CAPITAL LETTER DALDA + '\x2c86'# -> unI64 11399 + -- COPTIC CAPITAL LETTER EIE + '\x2c88'# -> unI64 11401 + -- COPTIC CAPITAL LETTER SOU + '\x2c8a'# -> unI64 11403 + -- COPTIC CAPITAL LETTER ZATA + '\x2c8c'# -> unI64 11405 + -- COPTIC CAPITAL LETTER HATE + '\x2c8e'# -> unI64 11407 + -- COPTIC CAPITAL LETTER THETHE + '\x2c90'# -> unI64 11409 + -- COPTIC CAPITAL LETTER IAUDA + '\x2c92'# -> unI64 11411 + -- COPTIC CAPITAL LETTER KAPA + '\x2c94'# -> unI64 11413 + -- COPTIC CAPITAL LETTER LAULA + '\x2c96'# -> unI64 11415 + -- COPTIC CAPITAL LETTER MI + '\x2c98'# -> unI64 11417 + -- COPTIC CAPITAL LETTER NI + '\x2c9a'# -> unI64 11419 + -- COPTIC CAPITAL LETTER KSI + '\x2c9c'# -> unI64 11421 + -- COPTIC CAPITAL LETTER O + '\x2c9e'# -> unI64 11423 + -- COPTIC CAPITAL LETTER PI + '\x2ca0'# -> unI64 11425 + -- COPTIC CAPITAL LETTER RO + '\x2ca2'# -> unI64 11427 + -- COPTIC CAPITAL LETTER SIMA + '\x2ca4'# -> unI64 11429 + -- COPTIC CAPITAL LETTER TAU + '\x2ca6'# -> unI64 11431 + -- COPTIC CAPITAL LETTER UA + '\x2ca8'# -> unI64 11433 + -- COPTIC CAPITAL LETTER FI + '\x2caa'# -> unI64 11435 + -- COPTIC CAPITAL LETTER KHI + '\x2cac'# -> unI64 11437 + -- COPTIC CAPITAL LETTER PSI + '\x2cae'# -> unI64 11439 + -- COPTIC CAPITAL LETTER OOU + '\x2cb0'# -> unI64 11441 + -- COPTIC CAPITAL LETTER DIALECT-P ALEF + '\x2cb2'# -> unI64 11443 + -- COPTIC CAPITAL LETTER OLD COPTIC AIN + '\x2cb4'# -> unI64 11445 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE + '\x2cb6'# -> unI64 11447 + -- COPTIC CAPITAL LETTER DIALECT-P KAPA + '\x2cb8'# -> unI64 11449 + -- COPTIC CAPITAL LETTER DIALECT-P NI + '\x2cba'# -> unI64 11451 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI + '\x2cbc'# -> unI64 11453 + -- COPTIC CAPITAL LETTER OLD COPTIC OOU + '\x2cbe'# -> unI64 11455 + -- COPTIC CAPITAL LETTER SAMPI + '\x2cc0'# -> unI64 11457 + -- COPTIC CAPITAL LETTER CROSSED SHEI + '\x2cc2'# -> unI64 11459 + -- COPTIC CAPITAL LETTER OLD COPTIC SHEI + '\x2cc4'# -> unI64 11461 + -- COPTIC CAPITAL LETTER OLD COPTIC ESH + '\x2cc6'# -> unI64 11463 + -- COPTIC CAPITAL LETTER AKHMIMIC KHEI + '\x2cc8'# -> unI64 11465 + -- COPTIC CAPITAL LETTER DIALECT-P HORI + '\x2cca'# -> unI64 11467 + -- COPTIC CAPITAL LETTER OLD COPTIC HORI + '\x2ccc'# -> unI64 11469 + -- COPTIC CAPITAL LETTER OLD COPTIC HA + '\x2cce'# -> unI64 11471 + -- COPTIC CAPITAL LETTER L-SHAPED HA + '\x2cd0'# -> unI64 11473 + -- COPTIC CAPITAL LETTER OLD COPTIC HEI + '\x2cd2'# -> unI64 11475 + -- COPTIC CAPITAL LETTER OLD COPTIC HAT + '\x2cd4'# -> unI64 11477 + -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA + '\x2cd6'# -> unI64 11479 + -- COPTIC CAPITAL LETTER OLD COPTIC DJA + '\x2cd8'# -> unI64 11481 + -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA + '\x2cda'# -> unI64 11483 + -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA + '\x2cdc'# -> unI64 11485 + -- COPTIC CAPITAL LETTER OLD NUBIAN NGI + '\x2cde'# -> unI64 11487 + -- COPTIC CAPITAL LETTER OLD NUBIAN NYI + '\x2ce0'# -> unI64 11489 + -- COPTIC CAPITAL LETTER OLD NUBIAN WAU + '\x2ce2'# -> unI64 11491 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI + '\x2ceb'# -> unI64 11500 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA + '\x2ced'# -> unI64 11502 + -- COPTIC CAPITAL LETTER BOHAIRIC KHEI + '\x2cf2'# -> unI64 11507 + -- CYRILLIC CAPITAL LETTER ZEMLYA + '\xa640'# -> unI64 42561 + -- CYRILLIC CAPITAL LETTER DZELO + '\xa642'# -> unI64 42563 + -- CYRILLIC CAPITAL LETTER REVERSED DZE + '\xa644'# -> unI64 42565 + -- CYRILLIC CAPITAL LETTER IOTA + '\xa646'# -> unI64 42567 + -- CYRILLIC CAPITAL LETTER DJERV + '\xa648'# -> unI64 42569 + -- CYRILLIC CAPITAL LETTER MONOGRAPH UK + '\xa64a'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER BROAD OMEGA + '\xa64c'# -> unI64 42573 + -- CYRILLIC CAPITAL LETTER NEUTRAL YER + '\xa64e'# -> unI64 42575 + -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER + '\xa650'# -> unI64 42577 + -- CYRILLIC CAPITAL LETTER IOTIFIED YAT + '\xa652'# -> unI64 42579 + -- CYRILLIC CAPITAL LETTER REVERSED YU + '\xa654'# -> unI64 42581 + -- CYRILLIC CAPITAL LETTER IOTIFIED A + '\xa656'# -> unI64 42583 + -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS + '\xa658'# -> unI64 42585 + -- CYRILLIC CAPITAL LETTER BLENDED YUS + '\xa65a'# -> unI64 42587 + -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS + '\xa65c'# -> unI64 42589 + -- CYRILLIC CAPITAL LETTER YN + '\xa65e'# -> unI64 42591 + -- CYRILLIC CAPITAL LETTER REVERSED TSE + '\xa660'# -> unI64 42593 + -- CYRILLIC CAPITAL LETTER SOFT DE + '\xa662'# -> unI64 42595 + -- CYRILLIC CAPITAL LETTER SOFT EL + '\xa664'# -> unI64 42597 + -- CYRILLIC CAPITAL LETTER SOFT EM + '\xa666'# -> unI64 42599 + -- CYRILLIC CAPITAL LETTER MONOCULAR O + '\xa668'# -> unI64 42601 + -- CYRILLIC CAPITAL LETTER BINOCULAR O + '\xa66a'# -> unI64 42603 + -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O + '\xa66c'# -> unI64 42605 + -- CYRILLIC CAPITAL LETTER DWE + '\xa680'# -> unI64 42625 + -- CYRILLIC CAPITAL LETTER DZWE + '\xa682'# -> unI64 42627 + -- CYRILLIC CAPITAL LETTER ZHWE + '\xa684'# -> unI64 42629 + -- CYRILLIC CAPITAL LETTER CCHE + '\xa686'# -> unI64 42631 + -- CYRILLIC CAPITAL LETTER DZZE + '\xa688'# -> unI64 42633 + -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK + '\xa68a'# -> unI64 42635 + -- CYRILLIC CAPITAL LETTER TWE + '\xa68c'# -> unI64 42637 + -- CYRILLIC CAPITAL LETTER TSWE + '\xa68e'# -> unI64 42639 + -- CYRILLIC CAPITAL LETTER TSSE + '\xa690'# -> unI64 42641 + -- CYRILLIC CAPITAL LETTER TCHE + '\xa692'# -> unI64 42643 + -- CYRILLIC CAPITAL LETTER HWE + '\xa694'# -> unI64 42645 + -- CYRILLIC CAPITAL LETTER SHWE + '\xa696'# -> unI64 42647 + -- CYRILLIC CAPITAL LETTER DOUBLE O + '\xa698'# -> unI64 42649 + -- CYRILLIC CAPITAL LETTER CROSSED O + '\xa69a'# -> unI64 42651 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF + '\xa722'# -> unI64 42787 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN + '\xa724'# -> unI64 42789 + -- LATIN CAPITAL LETTER HENG + '\xa726'# -> unI64 42791 + -- LATIN CAPITAL LETTER TZ + '\xa728'# -> unI64 42793 + -- LATIN CAPITAL LETTER TRESILLO + '\xa72a'# -> unI64 42795 + -- LATIN CAPITAL LETTER CUATRILLO + '\xa72c'# -> unI64 42797 + -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA + '\xa72e'# -> unI64 42799 + -- LATIN CAPITAL LETTER AA + '\xa732'# -> unI64 42803 + -- LATIN CAPITAL LETTER AO + '\xa734'# -> unI64 42805 + -- LATIN CAPITAL LETTER AU + '\xa736'# -> unI64 42807 + -- LATIN CAPITAL LETTER AV + '\xa738'# -> unI64 42809 + -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR + '\xa73a'# -> unI64 42811 + -- LATIN CAPITAL LETTER AY + '\xa73c'# -> unI64 42813 + -- LATIN CAPITAL LETTER REVERSED C WITH DOT + '\xa73e'# -> unI64 42815 + -- LATIN CAPITAL LETTER K WITH STROKE + '\xa740'# -> unI64 42817 + -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + '\xa742'# -> unI64 42819 + -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + '\xa744'# -> unI64 42821 + -- LATIN CAPITAL LETTER BROKEN L + '\xa746'# -> unI64 42823 + -- LATIN CAPITAL LETTER L WITH HIGH STROKE + '\xa748'# -> unI64 42825 + -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + '\xa74a'# -> unI64 42827 + -- LATIN CAPITAL LETTER O WITH LOOP + '\xa74c'# -> unI64 42829 + -- LATIN CAPITAL LETTER OO + '\xa74e'# -> unI64 42831 + -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + '\xa750'# -> unI64 42833 + -- LATIN CAPITAL LETTER P WITH FLOURISH + '\xa752'# -> unI64 42835 + -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + '\xa754'# -> unI64 42837 + -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + '\xa756'# -> unI64 42839 + -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + '\xa758'# -> unI64 42841 + -- LATIN CAPITAL LETTER R ROTUNDA + '\xa75a'# -> unI64 42843 + -- LATIN CAPITAL LETTER RUM ROTUNDA + '\xa75c'# -> unI64 42845 + -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + '\xa75e'# -> unI64 42847 + -- LATIN CAPITAL LETTER VY + '\xa760'# -> unI64 42849 + -- LATIN CAPITAL LETTER VISIGOTHIC Z + '\xa762'# -> unI64 42851 + -- LATIN CAPITAL LETTER THORN WITH STROKE + '\xa764'# -> unI64 42853 + -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER + '\xa766'# -> unI64 42855 + -- LATIN CAPITAL LETTER VEND + '\xa768'# -> unI64 42857 + -- LATIN CAPITAL LETTER ET + '\xa76a'# -> unI64 42859 + -- LATIN CAPITAL LETTER IS + '\xa76c'# -> unI64 42861 + -- LATIN CAPITAL LETTER CON + '\xa76e'# -> unI64 42863 + -- LATIN CAPITAL LETTER INSULAR D + '\xa779'# -> unI64 42874 + -- LATIN CAPITAL LETTER INSULAR F + '\xa77b'# -> unI64 42876 + -- LATIN CAPITAL LETTER INSULAR G + '\xa77d'# -> unI64 7545 + -- LATIN CAPITAL LETTER TURNED INSULAR G + '\xa77e'# -> unI64 42879 + -- LATIN CAPITAL LETTER TURNED L + '\xa780'# -> unI64 42881 + -- LATIN CAPITAL LETTER INSULAR R + '\xa782'# -> unI64 42883 + -- LATIN CAPITAL LETTER INSULAR S + '\xa784'# -> unI64 42885 + -- LATIN CAPITAL LETTER INSULAR T + '\xa786'# -> unI64 42887 + -- LATIN CAPITAL LETTER SALTILLO + '\xa78b'# -> unI64 42892 + -- LATIN CAPITAL LETTER TURNED H + '\xa78d'# -> unI64 613 + -- LATIN CAPITAL LETTER N WITH DESCENDER + '\xa790'# -> unI64 42897 + -- LATIN CAPITAL LETTER C WITH BAR + '\xa792'# -> unI64 42899 + -- LATIN CAPITAL LETTER B WITH FLOURISH + '\xa796'# -> unI64 42903 + -- LATIN CAPITAL LETTER F WITH STROKE + '\xa798'# -> unI64 42905 + -- LATIN CAPITAL LETTER VOLAPUK AE + '\xa79a'# -> unI64 42907 + -- LATIN CAPITAL LETTER VOLAPUK OE + '\xa79c'# -> unI64 42909 + -- LATIN CAPITAL LETTER VOLAPUK UE + '\xa79e'# -> unI64 42911 + -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + '\xa7a0'# -> unI64 42913 + -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + '\xa7a2'# -> unI64 42915 + -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + '\xa7a4'# -> unI64 42917 + -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + '\xa7a6'# -> unI64 42919 + -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + '\xa7a8'# -> unI64 42921 + -- LATIN CAPITAL LETTER H WITH HOOK + '\xa7aa'# -> unI64 614 + -- LATIN CAPITAL LETTER REVERSED OPEN E + '\xa7ab'# -> unI64 604 + -- LATIN CAPITAL LETTER SCRIPT G + '\xa7ac'# -> unI64 609 + -- LATIN CAPITAL LETTER L WITH BELT + '\xa7ad'# -> unI64 620 + -- LATIN CAPITAL LETTER SMALL CAPITAL I + '\xa7ae'# -> unI64 618 + -- LATIN CAPITAL LETTER TURNED K + '\xa7b0'# -> unI64 670 + -- LATIN CAPITAL LETTER TURNED T + '\xa7b1'# -> unI64 647 + -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL + '\xa7b2'# -> unI64 669 + -- LATIN CAPITAL LETTER CHI + '\xa7b3'# -> unI64 43859 + -- LATIN CAPITAL LETTER BETA + '\xa7b4'# -> unI64 42933 + -- LATIN CAPITAL LETTER OMEGA + '\xa7b6'# -> unI64 42935 + -- LATIN CAPITAL LETTER U WITH STROKE + '\xa7b8'# -> unI64 42937 + -- LATIN CAPITAL LETTER GLOTTAL A + '\xa7ba'# -> unI64 42939 + -- LATIN CAPITAL LETTER GLOTTAL I + '\xa7bc'# -> unI64 42941 + -- LATIN CAPITAL LETTER GLOTTAL U + '\xa7be'# -> unI64 42943 + -- LATIN CAPITAL LETTER OLD POLISH O + '\xa7c0'# -> unI64 42945 + -- LATIN CAPITAL LETTER ANGLICANA W + '\xa7c2'# -> unI64 42947 + -- LATIN CAPITAL LETTER C WITH PALATAL HOOK + '\xa7c4'# -> unI64 42900 + -- LATIN CAPITAL LETTER S WITH HOOK + '\xa7c5'# -> unI64 642 + -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK + '\xa7c6'# -> unI64 7566 + -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + '\xa7c7'# -> unI64 42952 + -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + '\xa7c9'# -> unI64 42954 + -- LATIN CAPITAL LETTER RAMS HORN + '\xa7cb'# -> unI64 612 + -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE + '\xa7cc'# -> unI64 42957 + -- LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE + '\xa7ce'# -> unI64 42959 + -- LATIN CAPITAL LETTER CLOSED INSULAR G + '\xa7d0'# -> unI64 42961 + -- LATIN CAPITAL LETTER DOUBLE THORN + '\xa7d2'# -> unI64 42963 + -- LATIN CAPITAL LETTER DOUBLE WYNN + '\xa7d4'# -> unI64 42965 + -- LATIN CAPITAL LETTER MIDDLE SCOTS S + '\xa7d6'# -> unI64 42967 + -- LATIN CAPITAL LETTER SIGMOID S + '\xa7d8'# -> unI64 42969 + -- LATIN CAPITAL LETTER LAMBDA + '\xa7da'# -> unI64 42971 + -- LATIN CAPITAL LETTER LAMBDA WITH STROKE + '\xa7dc'# -> unI64 411 + -- LATIN CAPITAL LETTER REVERSED HALF H + '\xa7f5'# -> unI64 42998 + -- CHEROKEE SMALL LETTER A + '\xab70'# -> unI64 5024 + -- CHEROKEE SMALL LETTER E + '\xab71'# -> unI64 5025 + -- CHEROKEE SMALL LETTER I + '\xab72'# -> unI64 5026 + -- CHEROKEE SMALL LETTER O + '\xab73'# -> unI64 5027 + -- CHEROKEE SMALL LETTER U + '\xab74'# -> unI64 5028 + -- CHEROKEE SMALL LETTER V + '\xab75'# -> unI64 5029 + -- CHEROKEE SMALL LETTER GA + '\xab76'# -> unI64 5030 + -- CHEROKEE SMALL LETTER KA + '\xab77'# -> unI64 5031 + -- CHEROKEE SMALL LETTER GE + '\xab78'# -> unI64 5032 + -- CHEROKEE SMALL LETTER GI + '\xab79'# -> unI64 5033 + -- CHEROKEE SMALL LETTER GO + '\xab7a'# -> unI64 5034 + -- CHEROKEE SMALL LETTER GU + '\xab7b'# -> unI64 5035 + -- CHEROKEE SMALL LETTER GV + '\xab7c'# -> unI64 5036 + -- CHEROKEE SMALL LETTER HA + '\xab7d'# -> unI64 5037 + -- CHEROKEE SMALL LETTER HE + '\xab7e'# -> unI64 5038 + -- CHEROKEE SMALL LETTER HI + '\xab7f'# -> unI64 5039 + -- CHEROKEE SMALL LETTER HO + '\xab80'# -> unI64 5040 + -- CHEROKEE SMALL LETTER HU + '\xab81'# -> unI64 5041 + -- CHEROKEE SMALL LETTER HV + '\xab82'# -> unI64 5042 + -- CHEROKEE SMALL LETTER LA + '\xab83'# -> unI64 5043 + -- CHEROKEE SMALL LETTER LE + '\xab84'# -> unI64 5044 + -- CHEROKEE SMALL LETTER LI + '\xab85'# -> unI64 5045 + -- CHEROKEE SMALL LETTER LO + '\xab86'# -> unI64 5046 + -- CHEROKEE SMALL LETTER LU + '\xab87'# -> unI64 5047 + -- CHEROKEE SMALL LETTER LV + '\xab88'# -> unI64 5048 + -- CHEROKEE SMALL LETTER MA + '\xab89'# -> unI64 5049 + -- CHEROKEE SMALL LETTER ME + '\xab8a'# -> unI64 5050 + -- CHEROKEE SMALL LETTER MI + '\xab8b'# -> unI64 5051 + -- CHEROKEE SMALL LETTER MO + '\xab8c'# -> unI64 5052 + -- CHEROKEE SMALL LETTER MU + '\xab8d'# -> unI64 5053 + -- CHEROKEE SMALL LETTER NA + '\xab8e'# -> unI64 5054 + -- CHEROKEE SMALL LETTER HNA + '\xab8f'# -> unI64 5055 + -- CHEROKEE SMALL LETTER NAH + '\xab90'# -> unI64 5056 + -- CHEROKEE SMALL LETTER NE + '\xab91'# -> unI64 5057 + -- CHEROKEE SMALL LETTER NI + '\xab92'# -> unI64 5058 + -- CHEROKEE SMALL LETTER NO + '\xab93'# -> unI64 5059 + -- CHEROKEE SMALL LETTER NU + '\xab94'# -> unI64 5060 + -- CHEROKEE SMALL LETTER NV + '\xab95'# -> unI64 5061 + -- CHEROKEE SMALL LETTER QUA + '\xab96'# -> unI64 5062 + -- CHEROKEE SMALL LETTER QUE + '\xab97'# -> unI64 5063 + -- CHEROKEE SMALL LETTER QUI + '\xab98'# -> unI64 5064 + -- CHEROKEE SMALL LETTER QUO + '\xab99'# -> unI64 5065 + -- CHEROKEE SMALL LETTER QUU + '\xab9a'# -> unI64 5066 + -- CHEROKEE SMALL LETTER QUV + '\xab9b'# -> unI64 5067 + -- CHEROKEE SMALL LETTER SA + '\xab9c'# -> unI64 5068 + -- CHEROKEE SMALL LETTER S + '\xab9d'# -> unI64 5069 + -- CHEROKEE SMALL LETTER SE + '\xab9e'# -> unI64 5070 + -- CHEROKEE SMALL LETTER SI + '\xab9f'# -> unI64 5071 + -- CHEROKEE SMALL LETTER SO + '\xaba0'# -> unI64 5072 + -- CHEROKEE SMALL LETTER SU + '\xaba1'# -> unI64 5073 + -- CHEROKEE SMALL LETTER SV + '\xaba2'# -> unI64 5074 + -- CHEROKEE SMALL LETTER DA + '\xaba3'# -> unI64 5075 + -- CHEROKEE SMALL LETTER TA + '\xaba4'# -> unI64 5076 + -- CHEROKEE SMALL LETTER DE + '\xaba5'# -> unI64 5077 + -- CHEROKEE SMALL LETTER TE + '\xaba6'# -> unI64 5078 + -- CHEROKEE SMALL LETTER DI + '\xaba7'# -> unI64 5079 + -- CHEROKEE SMALL LETTER TI + '\xaba8'# -> unI64 5080 + -- CHEROKEE SMALL LETTER DO + '\xaba9'# -> unI64 5081 + -- CHEROKEE SMALL LETTER DU + '\xabaa'# -> unI64 5082 + -- CHEROKEE SMALL LETTER DV + '\xabab'# -> unI64 5083 + -- CHEROKEE SMALL LETTER DLA + '\xabac'# -> unI64 5084 + -- CHEROKEE SMALL LETTER TLA + '\xabad'# -> unI64 5085 + -- CHEROKEE SMALL LETTER TLE + '\xabae'# -> unI64 5086 + -- CHEROKEE SMALL LETTER TLI + '\xabaf'# -> unI64 5087 + -- CHEROKEE SMALL LETTER TLO + '\xabb0'# -> unI64 5088 + -- CHEROKEE SMALL LETTER TLU + '\xabb1'# -> unI64 5089 + -- CHEROKEE SMALL LETTER TLV + '\xabb2'# -> unI64 5090 + -- CHEROKEE SMALL LETTER TSA + '\xabb3'# -> unI64 5091 + -- CHEROKEE SMALL LETTER TSE + '\xabb4'# -> unI64 5092 + -- CHEROKEE SMALL LETTER TSI + '\xabb5'# -> unI64 5093 + -- CHEROKEE SMALL LETTER TSO + '\xabb6'# -> unI64 5094 + -- CHEROKEE SMALL LETTER TSU + '\xabb7'# -> unI64 5095 + -- CHEROKEE SMALL LETTER TSV + '\xabb8'# -> unI64 5096 + -- CHEROKEE SMALL LETTER WA + '\xabb9'# -> unI64 5097 + -- CHEROKEE SMALL LETTER WE + '\xabba'# -> unI64 5098 + -- CHEROKEE SMALL LETTER WI + '\xabbb'# -> unI64 5099 + -- CHEROKEE SMALL LETTER WO + '\xabbc'# -> unI64 5100 + -- CHEROKEE SMALL LETTER WU + '\xabbd'# -> unI64 5101 + -- CHEROKEE SMALL LETTER WV + '\xabbe'# -> unI64 5102 + -- CHEROKEE SMALL LETTER YA + '\xabbf'# -> unI64 5103 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909606 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201062 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492518 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575526 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108838 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269747 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269747 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819892 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168308 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751220 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819902 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945524 + -- FULLWIDTH LATIN CAPITAL LETTER A + '\xff21'# -> unI64 65345 + -- FULLWIDTH LATIN CAPITAL LETTER B + '\xff22'# -> unI64 65346 + -- FULLWIDTH LATIN CAPITAL LETTER C + '\xff23'# -> unI64 65347 + -- FULLWIDTH LATIN CAPITAL LETTER D + '\xff24'# -> unI64 65348 + -- FULLWIDTH LATIN CAPITAL LETTER E + '\xff25'# -> unI64 65349 + -- FULLWIDTH LATIN CAPITAL LETTER F + '\xff26'# -> unI64 65350 + -- FULLWIDTH LATIN CAPITAL LETTER G + '\xff27'# -> unI64 65351 + -- FULLWIDTH LATIN CAPITAL LETTER H + '\xff28'# -> unI64 65352 + -- FULLWIDTH LATIN CAPITAL LETTER I + '\xff29'# -> unI64 65353 + -- FULLWIDTH LATIN CAPITAL LETTER J + '\xff2a'# -> unI64 65354 + -- FULLWIDTH LATIN CAPITAL LETTER K + '\xff2b'# -> unI64 65355 + -- FULLWIDTH LATIN CAPITAL LETTER L + '\xff2c'# -> unI64 65356 + -- FULLWIDTH LATIN CAPITAL LETTER M + '\xff2d'# -> unI64 65357 + -- FULLWIDTH LATIN CAPITAL LETTER N + '\xff2e'# -> unI64 65358 + -- FULLWIDTH LATIN CAPITAL LETTER O + '\xff2f'# -> unI64 65359 + -- FULLWIDTH LATIN CAPITAL LETTER P + '\xff30'# -> unI64 65360 + -- FULLWIDTH LATIN CAPITAL LETTER Q + '\xff31'# -> unI64 65361 + -- FULLWIDTH LATIN CAPITAL LETTER R + '\xff32'# -> unI64 65362 + -- FULLWIDTH LATIN CAPITAL LETTER S + '\xff33'# -> unI64 65363 + -- FULLWIDTH LATIN CAPITAL LETTER T + '\xff34'# -> unI64 65364 + -- FULLWIDTH LATIN CAPITAL LETTER U + '\xff35'# -> unI64 65365 + -- FULLWIDTH LATIN CAPITAL LETTER V + '\xff36'# -> unI64 65366 + -- FULLWIDTH LATIN CAPITAL LETTER W + '\xff37'# -> unI64 65367 + -- FULLWIDTH LATIN CAPITAL LETTER X + '\xff38'# -> unI64 65368 + -- FULLWIDTH LATIN CAPITAL LETTER Y + '\xff39'# -> unI64 65369 + -- FULLWIDTH LATIN CAPITAL LETTER Z + '\xff3a'# -> unI64 65370 + -- DESERET CAPITAL LETTER LONG I + '\x10400'# -> unI64 66600 + -- DESERET CAPITAL LETTER LONG E + '\x10401'# -> unI64 66601 + -- DESERET CAPITAL LETTER LONG A + '\x10402'# -> unI64 66602 + -- DESERET CAPITAL LETTER LONG AH + '\x10403'# -> unI64 66603 + -- DESERET CAPITAL LETTER LONG O + '\x10404'# -> unI64 66604 + -- DESERET CAPITAL LETTER LONG OO + '\x10405'# -> unI64 66605 + -- DESERET CAPITAL LETTER SHORT I + '\x10406'# -> unI64 66606 + -- DESERET CAPITAL LETTER SHORT E + '\x10407'# -> unI64 66607 + -- DESERET CAPITAL LETTER SHORT A + '\x10408'# -> unI64 66608 + -- DESERET CAPITAL LETTER SHORT AH + '\x10409'# -> unI64 66609 + -- DESERET CAPITAL LETTER SHORT O + '\x1040a'# -> unI64 66610 + -- DESERET CAPITAL LETTER SHORT OO + '\x1040b'# -> unI64 66611 + -- DESERET CAPITAL LETTER AY + '\x1040c'# -> unI64 66612 + -- DESERET CAPITAL LETTER OW + '\x1040d'# -> unI64 66613 + -- DESERET CAPITAL LETTER WU + '\x1040e'# -> unI64 66614 + -- DESERET CAPITAL LETTER YEE + '\x1040f'# -> unI64 66615 + -- DESERET CAPITAL LETTER H + '\x10410'# -> unI64 66616 + -- DESERET CAPITAL LETTER PEE + '\x10411'# -> unI64 66617 + -- DESERET CAPITAL LETTER BEE + '\x10412'# -> unI64 66618 + -- DESERET CAPITAL LETTER TEE + '\x10413'# -> unI64 66619 + -- DESERET CAPITAL LETTER DEE + '\x10414'# -> unI64 66620 + -- DESERET CAPITAL LETTER CHEE + '\x10415'# -> unI64 66621 + -- DESERET CAPITAL LETTER JEE + '\x10416'# -> unI64 66622 + -- DESERET CAPITAL LETTER KAY + '\x10417'# -> unI64 66623 + -- DESERET CAPITAL LETTER GAY + '\x10418'# -> unI64 66624 + -- DESERET CAPITAL LETTER EF + '\x10419'# -> unI64 66625 + -- DESERET CAPITAL LETTER VEE + '\x1041a'# -> unI64 66626 + -- DESERET CAPITAL LETTER ETH + '\x1041b'# -> unI64 66627 + -- DESERET CAPITAL LETTER THEE + '\x1041c'# -> unI64 66628 + -- DESERET CAPITAL LETTER ES + '\x1041d'# -> unI64 66629 + -- DESERET CAPITAL LETTER ZEE + '\x1041e'# -> unI64 66630 + -- DESERET CAPITAL LETTER ESH + '\x1041f'# -> unI64 66631 + -- DESERET CAPITAL LETTER ZHEE + '\x10420'# -> unI64 66632 + -- DESERET CAPITAL LETTER ER + '\x10421'# -> unI64 66633 + -- DESERET CAPITAL LETTER EL + '\x10422'# -> unI64 66634 + -- DESERET CAPITAL LETTER EM + '\x10423'# -> unI64 66635 + -- DESERET CAPITAL LETTER EN + '\x10424'# -> unI64 66636 + -- DESERET CAPITAL LETTER ENG + '\x10425'# -> unI64 66637 + -- DESERET CAPITAL LETTER OI + '\x10426'# -> unI64 66638 + -- DESERET CAPITAL LETTER EW + '\x10427'# -> unI64 66639 + -- OSAGE CAPITAL LETTER A + '\x104b0'# -> unI64 66776 + -- OSAGE CAPITAL LETTER AI + '\x104b1'# -> unI64 66777 + -- OSAGE CAPITAL LETTER AIN + '\x104b2'# -> unI64 66778 + -- OSAGE CAPITAL LETTER AH + '\x104b3'# -> unI64 66779 + -- OSAGE CAPITAL LETTER BRA + '\x104b4'# -> unI64 66780 + -- OSAGE CAPITAL LETTER CHA + '\x104b5'# -> unI64 66781 + -- OSAGE CAPITAL LETTER EHCHA + '\x104b6'# -> unI64 66782 + -- OSAGE CAPITAL LETTER E + '\x104b7'# -> unI64 66783 + -- OSAGE CAPITAL LETTER EIN + '\x104b8'# -> unI64 66784 + -- OSAGE CAPITAL LETTER HA + '\x104b9'# -> unI64 66785 + -- OSAGE CAPITAL LETTER HYA + '\x104ba'# -> unI64 66786 + -- OSAGE CAPITAL LETTER I + '\x104bb'# -> unI64 66787 + -- OSAGE CAPITAL LETTER KA + '\x104bc'# -> unI64 66788 + -- OSAGE CAPITAL LETTER EHKA + '\x104bd'# -> unI64 66789 + -- OSAGE CAPITAL LETTER KYA + '\x104be'# -> unI64 66790 + -- OSAGE CAPITAL LETTER LA + '\x104bf'# -> unI64 66791 + -- OSAGE CAPITAL LETTER MA + '\x104c0'# -> unI64 66792 + -- OSAGE CAPITAL LETTER NA + '\x104c1'# -> unI64 66793 + -- OSAGE CAPITAL LETTER O + '\x104c2'# -> unI64 66794 + -- OSAGE CAPITAL LETTER OIN + '\x104c3'# -> unI64 66795 + -- OSAGE CAPITAL LETTER PA + '\x104c4'# -> unI64 66796 + -- OSAGE CAPITAL LETTER EHPA + '\x104c5'# -> unI64 66797 + -- OSAGE CAPITAL LETTER SA + '\x104c6'# -> unI64 66798 + -- OSAGE CAPITAL LETTER SHA + '\x104c7'# -> unI64 66799 + -- OSAGE CAPITAL LETTER TA + '\x104c8'# -> unI64 66800 + -- OSAGE CAPITAL LETTER EHTA + '\x104c9'# -> unI64 66801 + -- OSAGE CAPITAL LETTER TSA + '\x104ca'# -> unI64 66802 + -- OSAGE CAPITAL LETTER EHTSA + '\x104cb'# -> unI64 66803 + -- OSAGE CAPITAL LETTER TSHA + '\x104cc'# -> unI64 66804 + -- OSAGE CAPITAL LETTER DHA + '\x104cd'# -> unI64 66805 + -- OSAGE CAPITAL LETTER U + '\x104ce'# -> unI64 66806 + -- OSAGE CAPITAL LETTER WA + '\x104cf'# -> unI64 66807 + -- OSAGE CAPITAL LETTER KHA + '\x104d0'# -> unI64 66808 + -- OSAGE CAPITAL LETTER GHA + '\x104d1'# -> unI64 66809 + -- OSAGE CAPITAL LETTER ZA + '\x104d2'# -> unI64 66810 + -- OSAGE CAPITAL LETTER ZHA + '\x104d3'# -> unI64 66811 + -- VITHKUQI CAPITAL LETTER A + '\x10570'# -> unI64 66967 + -- VITHKUQI CAPITAL LETTER BBE + '\x10571'# -> unI64 66968 + -- VITHKUQI CAPITAL LETTER BE + '\x10572'# -> unI64 66969 + -- VITHKUQI CAPITAL LETTER CE + '\x10573'# -> unI64 66970 + -- VITHKUQI CAPITAL LETTER CHE + '\x10574'# -> unI64 66971 + -- VITHKUQI CAPITAL LETTER DE + '\x10575'# -> unI64 66972 + -- VITHKUQI CAPITAL LETTER DHE + '\x10576'# -> unI64 66973 + -- VITHKUQI CAPITAL LETTER EI + '\x10577'# -> unI64 66974 + -- VITHKUQI CAPITAL LETTER E + '\x10578'# -> unI64 66975 + -- VITHKUQI CAPITAL LETTER FE + '\x10579'# -> unI64 66976 + -- VITHKUQI CAPITAL LETTER GA + '\x1057a'# -> unI64 66977 + -- VITHKUQI CAPITAL LETTER HA + '\x1057c'# -> unI64 66979 + -- VITHKUQI CAPITAL LETTER HHA + '\x1057d'# -> unI64 66980 + -- VITHKUQI CAPITAL LETTER I + '\x1057e'# -> unI64 66981 + -- VITHKUQI CAPITAL LETTER IJE + '\x1057f'# -> unI64 66982 + -- VITHKUQI CAPITAL LETTER JE + '\x10580'# -> unI64 66983 + -- VITHKUQI CAPITAL LETTER KA + '\x10581'# -> unI64 66984 + -- VITHKUQI CAPITAL LETTER LA + '\x10582'# -> unI64 66985 + -- VITHKUQI CAPITAL LETTER LLA + '\x10583'# -> unI64 66986 + -- VITHKUQI CAPITAL LETTER ME + '\x10584'# -> unI64 66987 + -- VITHKUQI CAPITAL LETTER NE + '\x10585'# -> unI64 66988 + -- VITHKUQI CAPITAL LETTER NJE + '\x10586'# -> unI64 66989 + -- VITHKUQI CAPITAL LETTER O + '\x10587'# -> unI64 66990 + -- VITHKUQI CAPITAL LETTER PE + '\x10588'# -> unI64 66991 + -- VITHKUQI CAPITAL LETTER QA + '\x10589'# -> unI64 66992 + -- VITHKUQI CAPITAL LETTER RE + '\x1058a'# -> unI64 66993 + -- VITHKUQI CAPITAL LETTER SE + '\x1058c'# -> unI64 66995 + -- VITHKUQI CAPITAL LETTER SHE + '\x1058d'# -> unI64 66996 + -- VITHKUQI CAPITAL LETTER TE + '\x1058e'# -> unI64 66997 + -- VITHKUQI CAPITAL LETTER THE + '\x1058f'# -> unI64 66998 + -- VITHKUQI CAPITAL LETTER U + '\x10590'# -> unI64 66999 + -- VITHKUQI CAPITAL LETTER VE + '\x10591'# -> unI64 67000 + -- VITHKUQI CAPITAL LETTER XE + '\x10592'# -> unI64 67001 + -- VITHKUQI CAPITAL LETTER Y + '\x10594'# -> unI64 67003 + -- VITHKUQI CAPITAL LETTER ZE + '\x10595'# -> unI64 67004 + -- OLD HUNGARIAN CAPITAL LETTER A + '\x10c80'# -> unI64 68800 + -- OLD HUNGARIAN CAPITAL LETTER AA + '\x10c81'# -> unI64 68801 + -- OLD HUNGARIAN CAPITAL LETTER EB + '\x10c82'# -> unI64 68802 + -- OLD HUNGARIAN CAPITAL LETTER AMB + '\x10c83'# -> unI64 68803 + -- OLD HUNGARIAN CAPITAL LETTER EC + '\x10c84'# -> unI64 68804 + -- OLD HUNGARIAN CAPITAL LETTER ENC + '\x10c85'# -> unI64 68805 + -- OLD HUNGARIAN CAPITAL LETTER ECS + '\x10c86'# -> unI64 68806 + -- OLD HUNGARIAN CAPITAL LETTER ED + '\x10c87'# -> unI64 68807 + -- OLD HUNGARIAN CAPITAL LETTER AND + '\x10c88'# -> unI64 68808 + -- OLD HUNGARIAN CAPITAL LETTER E + '\x10c89'# -> unI64 68809 + -- OLD HUNGARIAN CAPITAL LETTER CLOSE E + '\x10c8a'# -> unI64 68810 + -- OLD HUNGARIAN CAPITAL LETTER EE + '\x10c8b'# -> unI64 68811 + -- OLD HUNGARIAN CAPITAL LETTER EF + '\x10c8c'# -> unI64 68812 + -- OLD HUNGARIAN CAPITAL LETTER EG + '\x10c8d'# -> unI64 68813 + -- OLD HUNGARIAN CAPITAL LETTER EGY + '\x10c8e'# -> unI64 68814 + -- OLD HUNGARIAN CAPITAL LETTER EH + '\x10c8f'# -> unI64 68815 + -- OLD HUNGARIAN CAPITAL LETTER I + '\x10c90'# -> unI64 68816 + -- OLD HUNGARIAN CAPITAL LETTER II + '\x10c91'# -> unI64 68817 + -- OLD HUNGARIAN CAPITAL LETTER EJ + '\x10c92'# -> unI64 68818 + -- OLD HUNGARIAN CAPITAL LETTER EK + '\x10c93'# -> unI64 68819 + -- OLD HUNGARIAN CAPITAL LETTER AK + '\x10c94'# -> unI64 68820 + -- OLD HUNGARIAN CAPITAL LETTER UNK + '\x10c95'# -> unI64 68821 + -- OLD HUNGARIAN CAPITAL LETTER EL + '\x10c96'# -> unI64 68822 + -- OLD HUNGARIAN CAPITAL LETTER ELY + '\x10c97'# -> unI64 68823 + -- OLD HUNGARIAN CAPITAL LETTER EM + '\x10c98'# -> unI64 68824 + -- OLD HUNGARIAN CAPITAL LETTER EN + '\x10c99'# -> unI64 68825 + -- OLD HUNGARIAN CAPITAL LETTER ENY + '\x10c9a'# -> unI64 68826 + -- OLD HUNGARIAN CAPITAL LETTER O + '\x10c9b'# -> unI64 68827 + -- OLD HUNGARIAN CAPITAL LETTER OO + '\x10c9c'# -> unI64 68828 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE + '\x10c9d'# -> unI64 68829 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE + '\x10c9e'# -> unI64 68830 + -- OLD HUNGARIAN CAPITAL LETTER OEE + '\x10c9f'# -> unI64 68831 + -- OLD HUNGARIAN CAPITAL LETTER EP + '\x10ca0'# -> unI64 68832 + -- OLD HUNGARIAN CAPITAL LETTER EMP + '\x10ca1'# -> unI64 68833 + -- OLD HUNGARIAN CAPITAL LETTER ER + '\x10ca2'# -> unI64 68834 + -- OLD HUNGARIAN CAPITAL LETTER SHORT ER + '\x10ca3'# -> unI64 68835 + -- OLD HUNGARIAN CAPITAL LETTER ES + '\x10ca4'# -> unI64 68836 + -- OLD HUNGARIAN CAPITAL LETTER ESZ + '\x10ca5'# -> unI64 68837 + -- OLD HUNGARIAN CAPITAL LETTER ET + '\x10ca6'# -> unI64 68838 + -- OLD HUNGARIAN CAPITAL LETTER ENT + '\x10ca7'# -> unI64 68839 + -- OLD HUNGARIAN CAPITAL LETTER ETY + '\x10ca8'# -> unI64 68840 + -- OLD HUNGARIAN CAPITAL LETTER ECH + '\x10ca9'# -> unI64 68841 + -- OLD HUNGARIAN CAPITAL LETTER U + '\x10caa'# -> unI64 68842 + -- OLD HUNGARIAN CAPITAL LETTER UU + '\x10cab'# -> unI64 68843 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE + '\x10cac'# -> unI64 68844 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE + '\x10cad'# -> unI64 68845 + -- OLD HUNGARIAN CAPITAL LETTER EV + '\x10cae'# -> unI64 68846 + -- OLD HUNGARIAN CAPITAL LETTER EZ + '\x10caf'# -> unI64 68847 + -- OLD HUNGARIAN CAPITAL LETTER EZS + '\x10cb0'# -> unI64 68848 + -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN + '\x10cb1'# -> unI64 68849 + -- OLD HUNGARIAN CAPITAL LETTER US + '\x10cb2'# -> unI64 68850 + -- GARAY CAPITAL LETTER A + '\x10d50'# -> unI64 68976 + -- GARAY CAPITAL LETTER CA + '\x10d51'# -> unI64 68977 + -- GARAY CAPITAL LETTER MA + '\x10d52'# -> unI64 68978 + -- GARAY CAPITAL LETTER KA + '\x10d53'# -> unI64 68979 + -- GARAY CAPITAL LETTER BA + '\x10d54'# -> unI64 68980 + -- GARAY CAPITAL LETTER JA + '\x10d55'# -> unI64 68981 + -- GARAY CAPITAL LETTER SA + '\x10d56'# -> unI64 68982 + -- GARAY CAPITAL LETTER WA + '\x10d57'# -> unI64 68983 + -- GARAY CAPITAL LETTER LA + '\x10d58'# -> unI64 68984 + -- GARAY CAPITAL LETTER GA + '\x10d59'# -> unI64 68985 + -- GARAY CAPITAL LETTER DA + '\x10d5a'# -> unI64 68986 + -- GARAY CAPITAL LETTER XA + '\x10d5b'# -> unI64 68987 + -- GARAY CAPITAL LETTER YA + '\x10d5c'# -> unI64 68988 + -- GARAY CAPITAL LETTER TA + '\x10d5d'# -> unI64 68989 + -- GARAY CAPITAL LETTER RA + '\x10d5e'# -> unI64 68990 + -- GARAY CAPITAL LETTER NYA + '\x10d5f'# -> unI64 68991 + -- GARAY CAPITAL LETTER FA + '\x10d60'# -> unI64 68992 + -- GARAY CAPITAL LETTER NA + '\x10d61'# -> unI64 68993 + -- GARAY CAPITAL LETTER PA + '\x10d62'# -> unI64 68994 + -- GARAY CAPITAL LETTER HA + '\x10d63'# -> unI64 68995 + -- GARAY CAPITAL LETTER OLD KA + '\x10d64'# -> unI64 68996 + -- GARAY CAPITAL LETTER OLD NA + '\x10d65'# -> unI64 68997 + -- WARANG CITI CAPITAL LETTER NGAA + '\x118a0'# -> unI64 71872 + -- WARANG CITI CAPITAL LETTER A + '\x118a1'# -> unI64 71873 + -- WARANG CITI CAPITAL LETTER WI + '\x118a2'# -> unI64 71874 + -- WARANG CITI CAPITAL LETTER YU + '\x118a3'# -> unI64 71875 + -- WARANG CITI CAPITAL LETTER YA + '\x118a4'# -> unI64 71876 + -- WARANG CITI CAPITAL LETTER YO + '\x118a5'# -> unI64 71877 + -- WARANG CITI CAPITAL LETTER II + '\x118a6'# -> unI64 71878 + -- WARANG CITI CAPITAL LETTER UU + '\x118a7'# -> unI64 71879 + -- WARANG CITI CAPITAL LETTER E + '\x118a8'# -> unI64 71880 + -- WARANG CITI CAPITAL LETTER O + '\x118a9'# -> unI64 71881 + -- WARANG CITI CAPITAL LETTER ANG + '\x118aa'# -> unI64 71882 + -- WARANG CITI CAPITAL LETTER GA + '\x118ab'# -> unI64 71883 + -- WARANG CITI CAPITAL LETTER KO + '\x118ac'# -> unI64 71884 + -- WARANG CITI CAPITAL LETTER ENY + '\x118ad'# -> unI64 71885 + -- WARANG CITI CAPITAL LETTER YUJ + '\x118ae'# -> unI64 71886 + -- WARANG CITI CAPITAL LETTER UC + '\x118af'# -> unI64 71887 + -- WARANG CITI CAPITAL LETTER ENN + '\x118b0'# -> unI64 71888 + -- WARANG CITI CAPITAL LETTER ODD + '\x118b1'# -> unI64 71889 + -- WARANG CITI CAPITAL LETTER TTE + '\x118b2'# -> unI64 71890 + -- WARANG CITI CAPITAL LETTER NUNG + '\x118b3'# -> unI64 71891 + -- WARANG CITI CAPITAL LETTER DA + '\x118b4'# -> unI64 71892 + -- WARANG CITI CAPITAL LETTER AT + '\x118b5'# -> unI64 71893 + -- WARANG CITI CAPITAL LETTER AM + '\x118b6'# -> unI64 71894 + -- WARANG CITI CAPITAL LETTER BU + '\x118b7'# -> unI64 71895 + -- WARANG CITI CAPITAL LETTER PU + '\x118b8'# -> unI64 71896 + -- WARANG CITI CAPITAL LETTER HIYO + '\x118b9'# -> unI64 71897 + -- WARANG CITI CAPITAL LETTER HOLO + '\x118ba'# -> unI64 71898 + -- WARANG CITI CAPITAL LETTER HORR + '\x118bb'# -> unI64 71899 + -- WARANG CITI CAPITAL LETTER HAR + '\x118bc'# -> unI64 71900 + -- WARANG CITI CAPITAL LETTER SSUU + '\x118bd'# -> unI64 71901 + -- WARANG CITI CAPITAL LETTER SII + '\x118be'# -> unI64 71902 + -- WARANG CITI CAPITAL LETTER VIYO + '\x118bf'# -> unI64 71903 + -- MEDEFAIDRIN CAPITAL LETTER M + '\x16e40'# -> unI64 93792 + -- MEDEFAIDRIN CAPITAL LETTER S + '\x16e41'# -> unI64 93793 + -- MEDEFAIDRIN CAPITAL LETTER V + '\x16e42'# -> unI64 93794 + -- MEDEFAIDRIN CAPITAL LETTER W + '\x16e43'# -> unI64 93795 + -- MEDEFAIDRIN CAPITAL LETTER ATIU + '\x16e44'# -> unI64 93796 + -- MEDEFAIDRIN CAPITAL LETTER Z + '\x16e45'# -> unI64 93797 + -- MEDEFAIDRIN CAPITAL LETTER KP + '\x16e46'# -> unI64 93798 + -- MEDEFAIDRIN CAPITAL LETTER P + '\x16e47'# -> unI64 93799 + -- MEDEFAIDRIN CAPITAL LETTER T + '\x16e48'# -> unI64 93800 + -- MEDEFAIDRIN CAPITAL LETTER G + '\x16e49'# -> unI64 93801 + -- MEDEFAIDRIN CAPITAL LETTER F + '\x16e4a'# -> unI64 93802 + -- MEDEFAIDRIN CAPITAL LETTER I + '\x16e4b'# -> unI64 93803 + -- MEDEFAIDRIN CAPITAL LETTER K + '\x16e4c'# -> unI64 93804 + -- MEDEFAIDRIN CAPITAL LETTER A + '\x16e4d'# -> unI64 93805 + -- MEDEFAIDRIN CAPITAL LETTER J + '\x16e4e'# -> unI64 93806 + -- MEDEFAIDRIN CAPITAL LETTER E + '\x16e4f'# -> unI64 93807 + -- MEDEFAIDRIN CAPITAL LETTER B + '\x16e50'# -> unI64 93808 + -- MEDEFAIDRIN CAPITAL LETTER C + '\x16e51'# -> unI64 93809 + -- MEDEFAIDRIN CAPITAL LETTER U + '\x16e52'# -> unI64 93810 + -- MEDEFAIDRIN CAPITAL LETTER YU + '\x16e53'# -> unI64 93811 + -- MEDEFAIDRIN CAPITAL LETTER L + '\x16e54'# -> unI64 93812 + -- MEDEFAIDRIN CAPITAL LETTER Q + '\x16e55'# -> unI64 93813 + -- MEDEFAIDRIN CAPITAL LETTER HP + '\x16e56'# -> unI64 93814 + -- MEDEFAIDRIN CAPITAL LETTER NY + '\x16e57'# -> unI64 93815 + -- MEDEFAIDRIN CAPITAL LETTER X + '\x16e58'# -> unI64 93816 + -- MEDEFAIDRIN CAPITAL LETTER D + '\x16e59'# -> unI64 93817 + -- MEDEFAIDRIN CAPITAL LETTER OE + '\x16e5a'# -> unI64 93818 + -- MEDEFAIDRIN CAPITAL LETTER N + '\x16e5b'# -> unI64 93819 + -- MEDEFAIDRIN CAPITAL LETTER R + '\x16e5c'# -> unI64 93820 + -- MEDEFAIDRIN CAPITAL LETTER O + '\x16e5d'# -> unI64 93821 + -- MEDEFAIDRIN CAPITAL LETTER AI + '\x16e5e'# -> unI64 93822 + -- MEDEFAIDRIN CAPITAL LETTER Y + '\x16e5f'# -> unI64 93823 + -- BERIA ERFE CAPITAL LETTER ARKAB + '\x16ea0'# -> unI64 93883 + -- BERIA ERFE CAPITAL LETTER BASIGNA + '\x16ea1'# -> unI64 93884 + -- BERIA ERFE CAPITAL LETTER DARBAI + '\x16ea2'# -> unI64 93885 + -- BERIA ERFE CAPITAL LETTER EH + '\x16ea3'# -> unI64 93886 + -- BERIA ERFE CAPITAL LETTER FITKO + '\x16ea4'# -> unI64 93887 + -- BERIA ERFE CAPITAL LETTER GOWAY + '\x16ea5'# -> unI64 93888 + -- BERIA ERFE CAPITAL LETTER HIRDEABO + '\x16ea6'# -> unI64 93889 + -- BERIA ERFE CAPITAL LETTER I + '\x16ea7'# -> unI64 93890 + -- BERIA ERFE CAPITAL LETTER DJAI + '\x16ea8'# -> unI64 93891 + -- BERIA ERFE CAPITAL LETTER KOBO + '\x16ea9'# -> unI64 93892 + -- BERIA ERFE CAPITAL LETTER LAKKO + '\x16eaa'# -> unI64 93893 + -- BERIA ERFE CAPITAL LETTER MERI + '\x16eab'# -> unI64 93894 + -- BERIA ERFE CAPITAL LETTER NINI + '\x16eac'# -> unI64 93895 + -- BERIA ERFE CAPITAL LETTER GNA + '\x16ead'# -> unI64 93896 + -- BERIA ERFE CAPITAL LETTER NGAY + '\x16eae'# -> unI64 93897 + -- BERIA ERFE CAPITAL LETTER OI + '\x16eaf'# -> unI64 93898 + -- BERIA ERFE CAPITAL LETTER PI + '\x16eb0'# -> unI64 93899 + -- BERIA ERFE CAPITAL LETTER ERIGO + '\x16eb1'# -> unI64 93900 + -- BERIA ERFE CAPITAL LETTER ERIGO TAMURA + '\x16eb2'# -> unI64 93901 + -- BERIA ERFE CAPITAL LETTER SERI + '\x16eb3'# -> unI64 93902 + -- BERIA ERFE CAPITAL LETTER SHEP + '\x16eb4'# -> unI64 93903 + -- BERIA ERFE CAPITAL LETTER TATASOUE + '\x16eb5'# -> unI64 93904 + -- BERIA ERFE CAPITAL LETTER UI + '\x16eb6'# -> unI64 93905 + -- BERIA ERFE CAPITAL LETTER WASSE + '\x16eb7'# -> unI64 93906 + -- BERIA ERFE CAPITAL LETTER AY + '\x16eb8'# -> unI64 93907 + -- ADLAM CAPITAL LETTER ALIF + '\x1e900'# -> unI64 125218 + -- ADLAM CAPITAL LETTER DAALI + '\x1e901'# -> unI64 125219 + -- ADLAM CAPITAL LETTER LAAM + '\x1e902'# -> unI64 125220 + -- ADLAM CAPITAL LETTER MIIM + '\x1e903'# -> unI64 125221 + -- ADLAM CAPITAL LETTER BA + '\x1e904'# -> unI64 125222 + -- ADLAM CAPITAL LETTER SINNYIIYHE + '\x1e905'# -> unI64 125223 + -- ADLAM CAPITAL LETTER PE + '\x1e906'# -> unI64 125224 + -- ADLAM CAPITAL LETTER BHE + '\x1e907'# -> unI64 125225 + -- ADLAM CAPITAL LETTER RA + '\x1e908'# -> unI64 125226 + -- ADLAM CAPITAL LETTER E + '\x1e909'# -> unI64 125227 + -- ADLAM CAPITAL LETTER FA + '\x1e90a'# -> unI64 125228 + -- ADLAM CAPITAL LETTER I + '\x1e90b'# -> unI64 125229 + -- ADLAM CAPITAL LETTER O + '\x1e90c'# -> unI64 125230 + -- ADLAM CAPITAL LETTER DHA + '\x1e90d'# -> unI64 125231 + -- ADLAM CAPITAL LETTER YHE + '\x1e90e'# -> unI64 125232 + -- ADLAM CAPITAL LETTER WAW + '\x1e90f'# -> unI64 125233 + -- ADLAM CAPITAL LETTER NUN + '\x1e910'# -> unI64 125234 + -- ADLAM CAPITAL LETTER KAF + '\x1e911'# -> unI64 125235 + -- ADLAM CAPITAL LETTER YA + '\x1e912'# -> unI64 125236 + -- ADLAM CAPITAL LETTER U + '\x1e913'# -> unI64 125237 + -- ADLAM CAPITAL LETTER JIIM + '\x1e914'# -> unI64 125238 + -- ADLAM CAPITAL LETTER CHI + '\x1e915'# -> unI64 125239 + -- ADLAM CAPITAL LETTER HA + '\x1e916'# -> unI64 125240 + -- ADLAM CAPITAL LETTER QAAF + '\x1e917'# -> unI64 125241 + -- ADLAM CAPITAL LETTER GA + '\x1e918'# -> unI64 125242 + -- ADLAM CAPITAL LETTER NYA + '\x1e919'# -> unI64 125243 + -- ADLAM CAPITAL LETTER TU + '\x1e91a'# -> unI64 125244 + -- ADLAM CAPITAL LETTER NHA + '\x1e91b'# -> unI64 125245 + -- ADLAM CAPITAL LETTER VA + '\x1e91c'# -> unI64 125246 + -- ADLAM CAPITAL LETTER KHA + '\x1e91d'# -> unI64 125247 + -- ADLAM CAPITAL LETTER GBE + '\x1e91e'# -> unI64 125248 + -- ADLAM CAPITAL LETTER ZAL + '\x1e91f'# -> unI64 125249 + -- ADLAM CAPITAL LETTER KPO + '\x1e920'# -> unI64 125250 + -- ADLAM CAPITAL LETTER SHA + '\x1e921'# -> unI64 125251 + _ -> unI64 0 diff --git a/src/Data/Text/Internal/Fusion/Common.hs b/src/Data/Text/Internal/Fusion/Common.hs --- a/src/Data/Text/Internal/Fusion/Common.hs +++ b/src/Data/Text/Internal/Fusion/Common.hs @@ -77,6 +77,7 @@ , foldl1' , foldr , foldr1 + , foldlM' -- ** Special folds , concat @@ -688,6 +689,20 @@ Skip s' -> loop_foldl1' z s' Yield x s' -> loop_foldl1' (f z x) s' {-# INLINE [0] foldl1' #-} + +-- | A monadic version of 'foldl'. +-- +-- __Properties__ +-- +-- @ 'foldlM'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldlM'' f z0 @ +foldlM' :: P.Monad m => (b -> Char -> m b) -> b -> Stream Char -> m b +foldlM' f z0 (Stream next s0 _len) = loop_foldlM' z0 s0 + where + loop_foldlM' !z !s = case next s of + Done -> P.pure z + Skip s' -> loop_foldlM' z s' + Yield x s' -> f z x P.>>= \z' -> loop_foldlM' z' s' +{-# INLINE [0] foldlM' #-} -- | 'foldr', applied to a binary operator, a starting value (typically the -- right-identity of the operator), and a stream, reduces the stream using the diff --git a/src/Data/Text/Internal/IO.hs b/src/Data/Text/Internal/IO.hs --- a/src/Data/Text/Internal/IO.hs +++ b/src/Data/Text/Internal/IO.hs @@ -1,166 +1,308 @@ -{-# LANGUAGE BangPatterns, RecordWildCards #-} --- | --- Module : Data.Text.Internal.IO --- Copyright : (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Simon Marlow --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Low-level support for text I\/O. - -module Data.Text.Internal.IO - ( - hGetLineWith - , readChunk - ) where - -import qualified Control.Exception as E -import Data.IORef (readIORef, writeIORef) -import Data.Text (Text) -import Data.Text.Internal.Fusion (unstream) -import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) -import Data.Text.Internal.Fusion.Size (exactSize, maxSize) -import Data.Text.Unsafe (inlinePerformIO) -import Foreign.Storable (peekElemOff) -import GHC.IO.Buffer (Buffer(..), CharBuffer, RawCharBuffer, bufferAdjustL, - bufferElems, charSize, isEmptyBuffer, readCharBuf, - withRawBuffer, writeCharBuf) -import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_) -import GHC.IO.Handle.Types (Handle__(..), Newline(..)) -import System.IO (Handle) -import System.IO.Error (isEOFError) -import qualified Data.Text as T - --- | Read a single line of input from a handle, constructing a list of --- decoded chunks as we go. When we're done, transform them into the --- destination type. -hGetLineWith :: ([Text] -> t) -> Handle -> IO t -hGetLineWith f h = wantReadableHandle_ "hGetLine" h go - where - go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] - -hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] -hGetLineLoop hh@Handle__{..} = go where - go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do - let findEOL raw r | r == w = return (False, w) - | otherwise = do - (c,r') <- readCharBuf raw r - if c == '\n' - then return (True, r) - else findEOL raw r' - (eol, off) <- findEOL raw0 r0 - (t,r') <- if haInputNL == CRLF - then unpack_nl raw0 r0 off - else do t <- unpack raw0 r0 off - return (t,off) - if eol - then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) - return $ reverse (t:ts) - else do - let buf1 = bufferAdjustL r' buf - maybe_buf <- maybeFillReadBuffer hh buf1 - case maybe_buf of - -- Nothing indicates we caught an EOF, and we may have a - -- partial line to return. - Nothing -> do - -- we reached EOF. There might be a lone \r left - -- in the buffer, so check for that and - -- append it to the line if necessary. - let pre | isEmptyBuffer buf1 = T.empty - | otherwise = T.singleton '\r' - writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } - let str = reverse . filter (not . T.null) $ pre:t:ts - if null str - then ioe_EOF - else return str - Just new_buf -> go (t:ts) new_buf - --- This function is lifted almost verbatim from GHC.IO.Handle.Text. -maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) -maybeFillReadBuffer handle_ buf - = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> - if isEOFError e - then return Nothing - else ioError e - -unpack :: RawCharBuffer -> Int -> Int -> IO Text -unpack !buf !r !w - | charSize /= 4 = sizeError "unpack" - | r >= w = return T.empty - | otherwise = withRawBuffer buf go - where - go pbuf = return $! unstream (Stream next r (exactSize (w-r))) - where - next !i | i >= w = Done - | otherwise = Yield (ix i) (i+1) - ix i = inlinePerformIO $ peekElemOff pbuf i - -unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) -unpack_nl !buf !r !w - | charSize /= 4 = sizeError "unpack_nl" - | r >= w = return (T.empty, 0) - | otherwise = withRawBuffer buf $ go - where - go pbuf = do - let !t = unstream (Stream next r (maxSize (w-r))) - w' = w - 1 - return $ if ix w' == '\r' - then (t,w') - else (t,w) - where - next !i | i >= w = Done - | c == '\r' = let i' = i + 1 - in if i' < w - then if ix i' == '\n' - then Yield '\n' (i+2) - else Yield '\n' i' - else Done - | otherwise = Yield c (i+1) - where c = ix i - ix i = inlinePerformIO $ peekElemOff pbuf i - --- This function is completely lifted from GHC.IO.Handle.Text. -getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer -getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = - case bufferElems buf of - -- buffer empty: read some more - 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf - - -- if the buffer has a single '\r' in it and we're doing newline - -- translation: read some more - 1 | haInputNL == CRLF -> do - (c,_) <- readCharBuf bufRaw bufL - if c == '\r' - then do -- shuffle the '\r' to the beginning. This is only safe - -- if we're about to call readTextDevice, otherwise it - -- would mess up flushCharBuffer. - -- See [note Buffer Flushing], GHC.IO.Handle.Types - _ <- writeCharBuf bufRaw 0 '\r' - let buf' = buf{ bufL=0, bufR=1 } - readTextDevice handle_ buf' - else do - return buf - - -- buffer has some chars in it already: just return it - _otherwise -> {-# SCC "otherwise" #-} return buf - --- | Read a single chunk of strict text from a buffer. Used by both --- the strict and lazy implementations of hGetContents. -readChunk :: Handle__ -> CharBuffer -> IO Text -readChunk hh@Handle__{..} buf = do - buf'@Buffer{..} <- getSomeCharacters hh buf - (t,r) <- if haInputNL == CRLF - then unpack_nl bufRaw bufL bufR - else do t <- unpack bufRaw bufL bufR - return (t,bufR) - writeIORef haCharBuffer (bufferAdjustL r buf') - return t - -sizeError :: String -> a -sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size" +{-# LANGUAGE BangPatterns, RecordWildCards #-} +{-# LANGUAGE MagicHash #-} +-- | +-- Module : Data.Text.Internal.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Low-level support for text I\/O. + +module Data.Text.Internal.IO + ( + hGetLineWith + , readChunk + , hPutStream + , hPutStr + , hPutStrLn + ) where + +import qualified Control.Exception as E +import qualified Data.ByteString as B +import Data.ByteString.Builder (hPutBuilder, charUtf8) +import Data.IORef (readIORef, writeIORef) +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, streamLn, unstream) +import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size (exactSize, maxSize) +import Data.Text.Unsafe (inlinePerformIO) +import Foreign.Storable (peekElemOff) +import GHC.Exts (reallyUnsafePtrEquality#, isTrue#) +import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer, + bufferAdjustL, bufferElems, charSize, emptyBuffer, + isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer, + writeCharBuf) +import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_, + wantWritableHandle) +import GHC.IO.Handle.Text (commitBuffer') +import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..)) +import System.IO (Handle, hPutChar, utf8) +import System.IO.Error (isEOFError) +import qualified Data.Text as T + +-- | Read a single line of input from a handle, constructing a list of +-- decoded chunks as we go. When we're done, transform them into the +-- destination type. +hGetLineWith :: ([Text] -> t) -> Handle -> IO t +hGetLineWith f h = wantReadableHandle_ "hGetLine" h go + where + go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] + +hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] +hGetLineLoop hh@Handle__{..} = go where + go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do + let findEOL raw r | r == w = return (False, w) + | otherwise = do + (c,r') <- readCharBuf raw r + if c == '\n' + then return (True, r) + else findEOL raw r' + (eol, off) <- findEOL raw0 r0 + (t,r') <- if haInputNL == CRLF + then unpack_nl raw0 r0 off + else do t <- unpack raw0 r0 off + return (t,off) + if eol + then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) + return $ reverse (t:ts) + else do + let buf1 = bufferAdjustL r' buf + maybe_buf <- maybeFillReadBuffer hh buf1 + case maybe_buf of + -- Nothing indicates we caught an EOF, and we may have a + -- partial line to return. + Nothing -> do + -- we reached EOF. There might be a lone \r left + -- in the buffer, so check for that and + -- append it to the line if necessary. + let pre | isEmptyBuffer buf1 = T.empty + | otherwise = T.singleton '\r' + writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } + let str = reverse . filter (not . T.null) $ pre:t:ts + if null str + then ioe_EOF + else return str + Just new_buf -> go (t:ts) new_buf + +-- This function is lifted almost verbatim from GHC.IO.Handle.Text. +maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) +maybeFillReadBuffer handle_ buf + = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> + if isEOFError e + then return Nothing + else ioError e + +unpack :: RawCharBuffer -> Int -> Int -> IO Text +unpack !buf !r !w + | charSize /= 4 = sizeError "unpack" + | r >= w = return T.empty + | otherwise = withRawBuffer buf go + where + go pbuf = return $! unstream (Stream next r (exactSize (w-r))) + where + next !i | i >= w = Done + | otherwise = Yield (ix i) (i+1) + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- Variant of 'unpack' with CRLF decoding. If there is a trailing '\r', leave it in the buffer. +unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) +unpack_nl !buf !r !w + | charSize /= 4 = sizeError "unpack_nl" + | r >= w = return (T.empty, 0) + | otherwise = withRawBuffer buf $ go + where + go pbuf = do + let !t = unstream (Stream next r (maxSize (w-r))) + w' = w - 1 + return $ if ix w' == '\r' + then (t,w') + else (t,w) + where + next !i | i >= w = Done + | c == '\r' = let i' = i + 1 + in if i' < w + then if ix i' == '\n' + then Yield '\n' (i+2) + else Yield '\r' i' + else Done + | otherwise = Yield c (i+1) + where c = ix i + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer +getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = + case bufferElems buf of + -- buffer empty: read some more + 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf + + -- if the buffer has a single '\r' in it and we're doing newline + -- translation: read some more + 1 | haInputNL == CRLF -> do + (c,_) <- readCharBuf bufRaw bufL + if c == '\r' + then do -- shuffle the '\r' to the beginning. This is only safe + -- if we're about to call readTextDevice, otherwise it + -- would mess up flushCharBuffer. + -- See [note Buffer Flushing], GHC.IO.Handle.Types + _ <- writeCharBuf bufRaw 0 '\r' + let buf' = buf{ bufL=0, bufR=1 } + readTextDevice handle_ buf' + else do + return buf + + -- buffer has some chars in it already: just return it + _otherwise -> {-# SCC "otherwise" #-} return buf + +-- | Read a single chunk of strict text from a buffer. Used by both +-- the strict and lazy implementations of hGetContents. +readChunk :: Handle__ -> CharBuffer -> IO Text +readChunk hh@Handle__{..} buf = do + buf'@Buffer{..} <- getSomeCharacters hh buf + (t,r) <- if haInputNL == CRLF + then unpack_nl bufRaw bufL bufR + else do t <- unpack bufRaw bufL bufR + return (t,bufR) + writeIORef haCharBuffer (bufferAdjustL r buf') + return t + +-- | Print a @Stream Char@. +hPutStream :: Handle -> Stream Char -> IO () +hPutStream h str = hPutStreamOrUtf8 h str Nothing + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8) + where + putUtf8 = B.hPutStr h (encodeUtf8 t) + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8) + where + -- Not using B.hPutStrLn because it's not necessarily atomic: + -- https://github.com/haskell/bytestring/issues/200 + putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n') + +-- | 'hPutStream' with an optional special case when the output encoding is +-- UTF-8 and without newline conversion. +hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO () +-- This function is modified from GHC.IO.Handle.Text. +hPutStreamOrUtf8 h str mPutUtf8 = do + (buffer_mode, nl, isUtf8) <- + wantWritableHandle "hPutStr" h $ \h_ -> do + bmode <- getSpareBuffer h_ + return (bmode, haOutputNL h_, eqUTF8 h_) + case buffer_mode of + _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8 + (NoBuffering, _) -> hPutChars h str + (LineBuffering, buf) -> writeLines h nl buf str + (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str + + where + -- If the encoding is UTF-8, it's most likely pointer-equal to + -- 'System.IO.utf8', letting us avoid a String comparison. + -- If it is somehow UTF-8 but not pointer-equal to 'utf8', + -- we will just take a slower branch, but the result is still correct. + eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec +{-# INLINE hPutStreamOrUtf8 #-} + +hPutChars :: Handle -> Stream Char -> IO () +hPutChars h (Stream next0 s0 _len) = loop s0 + where + loop !s = case next0 s of + Done -> return () + Skip s' -> loop s' + Yield x s' -> hPutChar h x >> loop s' + +-- The following functions are largely lifted from GHC.IO.Handle.Text, +-- but adapted to a coinductive stream of data instead of an inductive +-- list. +-- +-- We have several variations of more or less the same code for +-- performance reasons. Splitting the original buffered write +-- function into line- and block-oriented versions gave us a 2.1x +-- performance improvement. Lifting out the raw/cooked newline +-- handling gave a few more percent on top. + +writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO () +writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' -> do + n' <- if nl == CRLF + then do n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' + else writeCharBuf' raw len n x + commit n' True{-needs flush-} False >>= outer s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO () +writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + -- Leave room for two characters for CRLF decoding + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' && isCRLF -> do + n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' >>= inner s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +-- | Only modifies the raw buffer and not the buffer attributes +writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int +writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $ + writeCharBuf bufRaw n c + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) +getSpareBuffer Handle__{haCharBuffer=ref, + haBuffers=spare_ref, + haBufferMode=mode} + = do + case mode of + NoBuffering -> return (mode, error "no buffer!") + _ -> do + bufs <- readIORef spare_ref + buf <- readIORef ref + case bufs of + BufferListCons b rest -> do + writeIORef spare_ref rest + return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) + BufferListNil -> do + new_buf <- newCharBuffer (bufSize buf) WriteBuffer + return (mode, new_buf) + + +-- This function is modified from GHC.Internal.IO.Handle.Text. +commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool + -> IO CharBuffer +commitBuffer hdl !raw !sz !count flush release = + wantWritableHandle "commitAndReleaseBuffer" hdl $ + commitBuffer' raw sz count flush release +{-# INLINE commitBuffer #-} + +sizeError :: String -> a +sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size" diff --git a/src/Data/Text/Internal/Lazy.hs b/src/Data/Text/Internal/Lazy.hs --- a/src/Data/Text/Internal/Lazy.hs +++ b/src/Data/Text/Internal/Lazy.hs @@ -1,147 +1,151 @@ -{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} -{-# OPTIONS_HADDOCK not-home #-} - --- | --- Module : Data.Text.Internal.Lazy --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- A module containing private 'Text' internals. This exposes the --- 'Text' representation and low level construction functions. --- Modules which extend the 'Text' system may need to use this module. - -module Data.Text.Internal.Lazy - ( - Text(..) - , LazyText - , chunk - , empty - , foldrChunks - , foldlChunks - -- * Data type invariant and abstraction functions - - -- $invariant - , strictInvariant - , lazyInvariant - , showStructure - - -- * Chunk allocation sizes - , defaultChunkSize - , smallChunkSize - , chunkOverhead - - , equal - ) where - -import Data.Bits (shiftL) -import Data.Text () -import Data.Typeable (Typeable) -import Foreign.Storable (sizeOf) -import qualified Data.Text.Array as A -import qualified Data.Text.Internal as T -import qualified Data.Text as T - -data Text = Empty - | Chunk {-# UNPACK #-} !T.Text Text - deriving (Typeable) - --- | Type synonym for the lazy flavour of 'Text'. -type LazyText = Text - --- $invariant --- --- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or --- consists of non-null 'T.Text's. All functions must preserve this, --- and the QC properties must check this. - --- | Check the invariant strictly. -strictInvariant :: Text -> Bool -strictInvariant Empty = True -strictInvariant x@(Chunk (T.Text _ _ len) cs) - | len > 0 = strictInvariant cs - | otherwise = error $ "Data.Text.Lazy: invariant violation: " - ++ showStructure x - --- | Check the invariant lazily. -lazyInvariant :: Text -> Text -lazyInvariant Empty = Empty -lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) - | len > 0 = Chunk c (lazyInvariant cs) - | otherwise = error $ "Data.Text.Lazy: invariant violation: " - ++ showStructure x - --- | Display the internal structure of a lazy 'Text'. -showStructure :: Text -> String -showStructure Empty = "Empty" -showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" -showStructure (Chunk t ts) = - "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" - --- | Smart constructor for 'Chunk'. Guarantees the data type invariant. -chunk :: T.Text -> Text -> Text -{-# INLINE [0] chunk #-} -chunk t ts | T.null t = ts - | otherwise = Chunk t ts - -{-# RULES -"TEXT chunk/text" forall arr off len. - chunk (T.text arr off len) = chunk (T.Text arr off len) -"TEXT chunk/empty" forall ts. - chunk T.empty ts = ts -#-} - --- | Smart constructor for 'Empty'. -empty :: Text -{-# INLINE [0] empty #-} -empty = Empty - --- | Consume the chunks of a lazy 'Text' with a natural right fold. -foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a -foldrChunks f z = go - where go Empty = z - go (Chunk c cs) = f c (go cs) -{-# INLINE foldrChunks #-} - --- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, --- accumulating left fold. -foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a -foldlChunks f z = go z - where go !a Empty = a - go !a (Chunk c cs) = go (f a c) cs -{-# INLINE foldlChunks #-} - --- | Currently set to 16 KiB, less the memory management overhead. -defaultChunkSize :: Int -defaultChunkSize = 16384 - chunkOverhead -{-# INLINE defaultChunkSize #-} - --- | Currently set to 128 bytes, less the memory management overhead. -smallChunkSize :: Int -smallChunkSize = 128 - chunkOverhead -{-# INLINE smallChunkSize #-} - --- | The memory management overhead. Currently this is tuned for GHC only. -chunkOverhead :: Int -chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 -{-# INLINE chunkOverhead #-} - -equal :: Text -> Text -> Bool -equal Empty Empty = True -equal Empty _ = False -equal _ Empty = False -equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = - case compare lenA lenB of - LT -> A.equal arrA offA arrB offB lenA && - as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs - EQ -> A.equal arrA offA arrB offB lenA && - as `equal` bs - GT -> A.equal arrA offA arrB offB lenB && - Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs +{-# LANGUAGE BangPatterns #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal.Lazy +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. + +module Data.Text.Internal.Lazy + ( + Text(..) + , LazyText + , chunk + , empty + , foldrChunks + , foldlChunks + -- * Data type invariant and abstraction functions + + -- $invariant + , strictInvariant + , lazyInvariant + , showStructure + + -- * Chunk allocation sizes + , defaultChunkSize + , smallChunkSize + , chunkOverhead + + , equal + ) where + +import Data.Bits (shiftL) +import Data.Text () +import Foreign.Storable (sizeOf) +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text as T + +data Text = Empty + -- ^ Empty text. + -- + -- @since 2.1.2 + | Chunk {-# UNPACK #-} !T.Text Text + -- ^ Chunks must be non-empty, this invariant is not checked. + +-- | Type synonym for the lazy flavour of 'Text'. +-- +-- @since 2.1.1 +type LazyText = Text + +-- $invariant +-- +-- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or +-- consists of non-null 'T.Text's. All functions must preserve this, +-- and the QC properties must check this. + +-- | Check the invariant strictly. +strictInvariant :: Text -> Bool +strictInvariant Empty = True +strictInvariant x@(Chunk (T.Text _ _ len) cs) + | len > 0 = strictInvariant cs + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Check the invariant lazily. +lazyInvariant :: Text -> Text +lazyInvariant Empty = Empty +lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) + | len > 0 = Chunk c (lazyInvariant cs) + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Display the internal structure of a lazy 'Text'. +showStructure :: Text -> String +showStructure Empty = "Empty" +showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" +showStructure (Chunk t ts) = + "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" + +-- | Smart constructor for 'Chunk'. Guarantees the data type invariant. +chunk :: T.Text -> Text -> Text +{-# INLINE [0] chunk #-} +chunk t ts | T.null t = ts + | otherwise = Chunk t ts + +{-# RULES +"TEXT chunk/text" forall arr off len. + chunk (T.text arr off len) = chunk (T.Text arr off len) +"TEXT chunk/empty" forall ts. + chunk T.empty ts = ts +#-} + +-- | Smart constructor for 'Empty'. +empty :: Text +{-# INLINE [0] empty #-} +empty = Empty + +-- | Consume the chunks of a lazy 'Text' with a natural right fold. +foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a +foldrChunks f z = go + where go Empty = z + go (Chunk c cs) = f c (go cs) +{-# INLINE foldrChunks #-} + +-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, +-- accumulating left fold. +foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a +foldlChunks f z = go z + where go !a Empty = a + go !a (Chunk c cs) = go (f a c) cs +{-# INLINE foldlChunks #-} + +-- | Currently set to 16 KiB, less the memory management overhead. +defaultChunkSize :: Int +defaultChunkSize = 16384 - chunkOverhead +{-# INLINE defaultChunkSize #-} + +-- | Currently set to 128 bytes, less the memory management overhead. +smallChunkSize :: Int +smallChunkSize = 128 - chunkOverhead +{-# INLINE smallChunkSize #-} + +-- | The memory management overhead. Currently this is tuned for GHC only. +chunkOverhead :: Int +chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 +{-# INLINE chunkOverhead #-} + +equal :: Text -> Text -> Bool +equal Empty Empty = True +equal Empty _ = False +equal _ Empty = False +equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + case compare lenA lenB of + LT -> A.equal arrA offA arrB offB lenA && + as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs + EQ -> A.equal arrA offA arrB offB lenA && + as `equal` bs + GT -> A.equal arrA offA arrB offB lenB && + Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs diff --git a/src/Data/Text/Internal/Lazy/Fusion.hs b/src/Data/Text/Internal/Lazy/Fusion.hs --- a/src/Data/Text/Internal/Lazy/Fusion.hs +++ b/src/Data/Text/Internal/Lazy/Fusion.hs @@ -17,6 +17,7 @@ module Data.Text.Internal.Lazy.Fusion ( stream + , streamLn , unstream , unstreamChunks , length @@ -47,14 +48,35 @@ HasCallStack => #endif Text -> Stream Char -stream text = Stream next (text :*: 0) unknownSize +stream = stream' False +{-# INLINE [0] stream #-} + +-- | /O(n)/ @'streamLn' t = 'stream' (t <> \'\\n\')@ +-- +-- @since 2.1.2 +streamLn :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Stream Char +streamLn = stream' True + +-- | Shared implementation of 'stream' and 'streamLn'. +stream' :: +#if defined(ASSERTS) + HasCallStack => +#endif + Bool -> Text -> Stream Char +stream' addNl text = Stream next (text :*: 0) unknownSize where - next (Empty :*: _) = Done + next (Empty :*: i) + | addNl && i <= 0 = Yield '\n' (Empty :*: 1) + | otherwise = Done next (txt@(Chunk t@(I.Text _ _ len) ts) :*: i) | i >= len = next (ts :*: 0) | otherwise = Yield c (txt :*: i+d) where Iter c d = iter t i -{-# INLINE [0] stream #-} +{-# INLINE [0] stream' #-} -- | /O(n)/ Convert a 'Stream Char' into a 'Text', using the given -- chunk size. diff --git a/src/Data/Text/Internal/StrictBuilder.hs b/src/Data/Text/Internal/StrictBuilder.hs --- a/src/Data/Text/Internal/StrictBuilder.hs +++ b/src/Data/Text/Internal/StrictBuilder.hs @@ -1,6 +1,7 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeSynonymInstances #-} -- | -- Module : Data.Text.Internal.Builder @@ -14,7 +15,8 @@ -- @since 2.0.2 module Data.Text.Internal.StrictBuilder - ( StrictBuilder(..) + ( StrictTextBuilder(..) + , StrictBuilder , toText , fromChar , fromText @@ -43,18 +45,24 @@ -- | A delayed representation of strict 'Text'. -- --- @since 2.0.2 -data StrictBuilder = StrictBuilder +-- @since 2.1.2 +data StrictTextBuilder = StrictTextBuilder { sbLength :: {-# UNPACK #-} !Int , sbWrite :: forall s. A.MArray s -> Int -> ST s () } +-- | A delayed representation of strict 'Text'. +-- +-- @since 2.0.2 +{-# DEPRECATED StrictBuilder "Use StrictTextBuilder instead" #-} +type StrictBuilder = StrictTextBuilder + -- | Use 'StrictBuilder' to build 'Text'. -- -- @since 2.0.2 -toText :: StrictBuilder -> Text -toText (StrictBuilder 0 _) = empty -toText (StrictBuilder n write) = runST (do +toText :: StrictTextBuilder -> Text +toText (StrictTextBuilder 0 _) = empty +toText (StrictTextBuilder n write) = runST (do dst <- A.new n write dst 0 arr <- A.unsafeFreeze dst @@ -63,21 +71,21 @@ -- | Concatenation of 'StrictBuilder' is right-biased: -- the right builder will be run first. This allows a builder to -- run tail-recursively when it was accumulated left-to-right. -instance Semigroup StrictBuilder where +instance Semigroup StrictTextBuilder where (<>) = appendRStrictBuilder -instance Monoid StrictBuilder where +instance Monoid StrictTextBuilder where mempty = emptyStrictBuilder mappend = (<>) -emptyStrictBuilder :: StrictBuilder -emptyStrictBuilder = StrictBuilder 0 (\_ _ -> pure ()) +emptyStrictBuilder :: StrictTextBuilder +emptyStrictBuilder = StrictTextBuilder 0 (\_ _ -> pure ()) -appendRStrictBuilder :: StrictBuilder -> StrictBuilder -> StrictBuilder -appendRStrictBuilder (StrictBuilder 0 _) b2 = b2 -appendRStrictBuilder b1 (StrictBuilder 0 _) = b1 -appendRStrictBuilder (StrictBuilder n1 write1) (StrictBuilder n2 write2) = - StrictBuilder (n1 + n2) (\dst ofs -> do +appendRStrictBuilder :: StrictTextBuilder -> StrictTextBuilder -> StrictTextBuilder +appendRStrictBuilder (StrictTextBuilder 0 _) b2 = b2 +appendRStrictBuilder b1 (StrictTextBuilder 0 _) = b1 +appendRStrictBuilder (StrictTextBuilder n1 write1) (StrictTextBuilder n2 write2) = + StrictTextBuilder (n1 + n2) (\dst ofs -> do write2 dst (ofs + n1) write1 dst ofs) @@ -91,16 +99,16 @@ -- Unsafe: This may not be valid UTF-8 text. -- -- @since 2.0.2 -unsafeFromByteString :: ByteString -> StrictBuilder +unsafeFromByteString :: ByteString -> StrictTextBuilder unsafeFromByteString bs = - StrictBuilder (B.length bs) (\dst ofs -> copyFromByteString dst ofs bs) + StrictTextBuilder (B.length bs) (\dst ofs -> copyFromByteString dst ofs bs) -- | -- @since 2.0.2 {-# INLINE fromChar #-} -fromChar :: Char -> StrictBuilder +fromChar :: Char -> StrictTextBuilder fromChar c = - StrictBuilder (utf8Length c) (\dst ofs -> void (Char.unsafeWrite dst ofs (safe c))) + StrictTextBuilder (utf8Length c) (\dst ofs -> void (Char.unsafeWrite dst ofs (safe c))) -- $unsafe -- For internal purposes, we abuse 'StrictBuilder' as a delayed 'Array' rather @@ -109,13 +117,13 @@ -- | Unsafe: This may not be valid UTF-8 text. -- -- @since 2.0.2 -unsafeFromWord8 :: Word8 -> StrictBuilder +unsafeFromWord8 :: Word8 -> StrictTextBuilder unsafeFromWord8 !w = - StrictBuilder 1 (\dst ofs -> A.unsafeWrite dst ofs w) + StrictTextBuilder 1 (\dst ofs -> A.unsafeWrite dst ofs w) -- | Copy 'Text' in a 'StrictBuilder' -- -- @since 2.0.2 -fromText :: Text -> StrictBuilder -fromText (Text src srcOfs n) = StrictBuilder n (\dst dstOfs -> +fromText :: Text -> StrictTextBuilder +fromText (Text src srcOfs n) = StrictTextBuilder n (\dst dstOfs -> A.copyI n dst dstOfs src srcOfs) diff --git a/src/Data/Text/Internal/Transformation.hs b/src/Data/Text/Internal/Transformation.hs --- a/src/Data/Text/Internal/Transformation.hs +++ b/src/Data/Text/Internal/Transformation.hs @@ -1,240 +1,340 @@ -{-# LANGUAGE BangPatterns, CPP, MagicHash #-} -{-# LANGUAGE Trustworthy #-} -{-# LANGUAGE UnliftedFFITypes #-} -{-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE PartialTypeSignatures #-} - -{-# OPTIONS_GHC -fno-warn-orphans #-} -{-# OPTIONS_GHC -Wno-partial-type-signatures #-} - --- | --- Module : Data.Text.Internal.Transformation --- Copyright : (c) 2008, 2009 Tom Harper, --- (c) 2009, 2010 Bryan O'Sullivan, --- (c) 2009 Duncan Coutts --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- This module holds functions shared between the strict and lazy implementations of @Text@ transformations. - -module Data.Text.Internal.Transformation - ( mapNonEmpty - , toCaseFoldNonEmpty - , toLowerNonEmpty - , toUpperNonEmpty - , filter_ - ) where - -import Prelude (Char, Bool(..), Int, - Ord(..), - Monad(..), pure, - (+), (-), ($), - not, return, otherwise) -import Data.Bits ((.&.), shiftR, shiftL) -import Control.Monad.ST (ST, runST) -import qualified Data.Text.Array as A -import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4) -import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping) -import Data.Text.Internal (Text(..), safe) -import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) -import qualified Prelude as P -import Data.Text.Unsafe (Iter(..), iterArray) -import Data.Word (Word8) -import qualified GHC.Exts as Exts -import GHC.Int (Int64(..)) - --- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. --- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. -mapNonEmpty :: (Char -> Char) -> Text -> Text -mapNonEmpty f = go - where - go (Text src o l) = runST $ do - marr <- A.new (l + 4) - outer marr (l + 4) o 0 - where - outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text - outer !dst !dstLen = inner - where - inner !srcOff !dstOff - | srcOff >= l + o = do - A.shrinkM dst dstOff - arr <- A.unsafeFreeze dst - return (Text arr 0 dstOff) - | dstOff + 4 > dstLen = do - let !dstLen' = dstLen + (l + o) - srcOff + 4 - dst' <- A.resizeM dst dstLen' - outer dst' dstLen' srcOff dstOff - | otherwise = do - let !(Iter c d) = iterArray src srcOff - d' <- unsafeWrite dst dstOff (safe (f c)) - inner (srcOff + d) (dstOff + d') -{-# INLINE mapNonEmpty #-} - -caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text -caseConvert ascii remap (Text src o l) = runST $ do - -- Case conversion a single code point may produce up to 3 code-points, - -- each up to 4 bytes, so 12 in total. - dst <- A.new (l + 12) - outer dst l o 0 - where - outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text - outer !dst !dstLen = inner - where - inner !srcOff !dstOff - | srcOff >= o + l = do - A.shrinkM dst dstOff - arr <- A.unsafeFreeze dst - return (Text arr 0 dstOff) - | dstOff + 12 > dstLen = do - -- Ensure to extend the buffer by at least 12 bytes. - let !dstLen' = dstLen + max 12 (l + o - srcOff) - dst' <- A.resizeM dst dstLen' - outer dst' dstLen' srcOff dstOff - -- If a character is to remain unchanged, no need to decode Char back into UTF8, - -- just copy bytes from input. - | otherwise = do - let m0 = A.unsafeIndex src srcOff - m1 = A.unsafeIndex src (srcOff + 1) - m2 = A.unsafeIndex src (srcOff + 2) - m3 = A.unsafeIndex src (srcOff + 3) - !d = utf8LengthByLeader m0 - case d of - 1 -> do - A.unsafeWrite dst dstOff (ascii m0) - inner (srcOff + 1) (dstOff + 1) - 2 -> do - let !(Exts.C# c) = chr2 m0 m1 - dstOff' <- case I64# (remap c) of - 0 -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - pure $ dstOff + 2 - i -> writeMapping i dstOff - inner (srcOff + 2) dstOff' - 3 -> do - let !(Exts.C# c) = chr3 m0 m1 m2 - dstOff' <- case I64# (remap c) of - 0 -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - pure $ dstOff + 3 - i -> writeMapping i dstOff - inner (srcOff + 3) dstOff' - _ -> do - let !(Exts.C# c) = chr4 m0 m1 m2 m3 - dstOff' <- case I64# (remap c) of - 0 -> do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - A.unsafeWrite dst (dstOff + 3) m3 - pure $ dstOff + 4 - i -> writeMapping i dstOff - inner (srcOff + 4) dstOff' - - writeMapping :: Int64 -> Int -> ST s Int - writeMapping 0 dstOff = pure dstOff - writeMapping i dstOff = do - let (ch, j) = chopOffChar i - d <- unsafeWrite dst dstOff ch - writeMapping j (dstOff + d) - - chopOffChar :: Int64 -> (Char, Int64) - chopOffChar ab = (chr a, ab `shiftR` 21) - where - chr (Exts.I# n) = Exts.C# (Exts.chr# n) - mask = (1 `shiftL` 21) - 1 - a = P.fromIntegral $ ab .&. mask -{-# INLINE caseConvert #-} - - --- | /O(n)/ Convert a string to folded case. --- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. -toCaseFoldNonEmpty :: Text -> Text -toCaseFoldNonEmpty = \xs -> caseConvert (\w -> if w - 65 <= 25 then w + 32 else w) foldMapping xs -{-# INLINE toCaseFoldNonEmpty #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. --- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. -toLowerNonEmpty :: Text -> Text -toLowerNonEmpty = \xs -> caseConvert (\w -> if w - 65 <= 25 then w + 32 else w) lowerMapping xs -{-# INLINE toLowerNonEmpty #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. --- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. -toUpperNonEmpty :: Text -> Text -toUpperNonEmpty = \xs -> caseConvert (\w -> if w - 97 <= 25 then w - 32 else w) upperMapping xs -{-# INLINE toUpperNonEmpty #-} - --- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@, --- calls the continuation with the @Text@ containing only the characters satisfying the predicate. -filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a -filter_ mkText p = go - where - go (Text src o l) = runST $ do - -- It's tempting to allocate l elements at once and avoid resizing. - -- However, this can be unacceptable in scenarios where a huge array - -- is filtered with a rare predicate, resulting in a much shorter buffer. - let !dstLen = min l 64 - dst <- A.new dstLen - outer dst dstLen o 0 - where - outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a - outer !dst !dstLen = inner - where - inner !srcOff !dstOff - | srcOff >= o + l = do - A.shrinkM dst dstOff - arr <- A.unsafeFreeze dst - return $ mkText arr 0 dstOff - | dstOff + 4 > dstLen = do - -- Double size of the buffer, unless it becomes longer than - -- source string. Ensure to extend it by least 4 bytes. - let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) - dst' <- A.resizeM dst dstLen' - outer dst' dstLen' srcOff dstOff - -- In case of success, filter writes exactly the same character - -- it just read (this is not a case for map, for example). - -- We leverage this fact below: no need to decode Char back into UTF8, - -- just copy bytes from input. - | otherwise = do - let m0 = A.unsafeIndex src srcOff - m1 = A.unsafeIndex src (srcOff + 1) - m2 = A.unsafeIndex src (srcOff + 2) - m3 = A.unsafeIndex src (srcOff + 3) - !d = utf8LengthByLeader m0 - case d of - 1 -> do - let !c = unsafeChr8 m0 - if not (p c) then inner (srcOff + 1) dstOff else do - A.unsafeWrite dst dstOff m0 - inner (srcOff + 1) (dstOff + 1) - 2 -> do - let !c = chr2 m0 m1 - if not (p c) then inner (srcOff + 2) dstOff else do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - inner (srcOff + 2) (dstOff + 2) - 3 -> do - let !c = chr3 m0 m1 m2 - if not (p c) then inner (srcOff + 3) dstOff else do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - inner (srcOff + 3) (dstOff + 3) - _ -> do - let !c = chr4 m0 m1 m2 m3 - if not (p c) then inner (srcOff + 4) dstOff else do - A.unsafeWrite dst dstOff m0 - A.unsafeWrite dst (dstOff + 1) m1 - A.unsafeWrite dst (dstOff + 2) m2 - A.unsafeWrite dst (dstOff + 3) m3 - inner (srcOff + 4) (dstOff + 4) -{-# INLINE filter_ #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text.Internal.Transformation +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- This module holds functions shared between the strict and lazy implementations of @Text@ transformations. + +module Data.Text.Internal.Transformation + ( mapNonEmpty + , toCaseFoldNonEmpty + , toLowerNonEmpty + , toUpperNonEmpty + , toTitleNonEmpty + , filter_ + ) where + +import Prelude (Char, Bool(..), Int, + Ord(..), + Monad(..), pure, + (+), (-), ($), (&&), (||), (==), + not, return, otherwise) +import Data.Bits ((.&.), shiftR, shiftL) +import Data.Char (isLetter, isSpace) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4) +import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping) +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iterArray) +import Data.Word (Word8) +import qualified GHC.Exts as Exts +import GHC.Int (Int64(..)) + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +mapNonEmpty :: (Char -> Char) -> Text -> Text +mapNonEmpty f = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + | otherwise = do + let !(Iter c d) = iterArray src srcOff + d' <- unsafeWrite dst dstOff (safe (f c)) + inner (srcOff + d) (dstOff + d') +{-# INLINE mapNonEmpty #-} + +caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text +caseConvert ascii remap (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff (ascii m0) + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' + +{-# INLINABLE caseConvert #-} + +writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int +writeMapping !_ 0 !dstOff = pure dstOff +writeMapping dst i dstOff = do + let (ch, j) = chopOffChar i + d <- unsafeWrite dst dstOff ch + writeMapping dst j (dstOff + d) + +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (Exts.I# n) = Exts.C# (Exts.chr# n) + mask = (1 `shiftL` 21) - 1 + a = P.fromIntegral $ ab .&. mask + +-- | /O(n)/ Convert a string to folded case. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toCaseFoldNonEmpty :: Text -> Text +toCaseFoldNonEmpty = \xs -> caseConvert asciiToLower foldMapping xs +{-# INLINE toCaseFoldNonEmpty #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toLowerNonEmpty :: Text -> Text +toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs +{-# INLINE toLowerNonEmpty #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toUpperNonEmpty :: Text -> Text +toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs +{-# INLINE toUpperNonEmpty #-} + +asciiToLower :: Word8 -> Word8 +asciiToLower w = if w - 65 <= 25 then w + 32 else w + +asciiToUpper :: Word8 -> Word8 +asciiToUpper w = if w - 97 <= 25 then w - 32 else w + +isAsciiLetter :: Word8 -> Bool +isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25 + +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) + +-- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toTitleNonEmpty :: Text -> Text +toTitleNonEmpty (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 False + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !mode + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff mode + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + + case d of + 1 -> do + let (mode', m0') = asciiAdvance mode m0 + A.unsafeWrite dst dstOff m0' + inner (srcOff + 1) (dstOff + 1) mode' + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' mode' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + isSpace3 ch + = m0 == 0xE1 && m1 == 0x9A && m2 == 0x80 + || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F) + || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80 + !(# mode', c' #) = advance isSpace3 mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' mode' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + !(# mode', c' #) = advance (\_ -> False) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' mode' + + asciiAdvance :: Bool -> Word8 -> (Bool, Word8) + asciiAdvance False w = (isAsciiLetter w, asciiToUpper w) + asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w) + + advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #) + advance _ False c = (# isLetter (Exts.C# c), titleMapping c #) + advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #) + {-# INLINE advance #-} + +-- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@, +-- calls the continuation with the @Text@ containing only the characters satisfying the predicate. +filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a +filter_ mkText p = go + where + go (Text src o l) = runST $ do + -- It's tempting to allocate l elements at once and avoid resizing. + -- However, this can be unacceptable in scenarios where a huge array + -- is filtered with a rare predicate, resulting in a much shorter buffer. + let !dstLen = min l 64 + dst <- A.new dstLen + outer dst dstLen o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return $ mkText arr 0 dstOff + | dstOff + 4 > dstLen = do + -- Double size of the buffer, unless it becomes longer than + -- source string. Ensure to extend it by least 4 bytes. + let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- In case of success, filter writes exactly the same character + -- it just read (this is not a case for map, for example). + -- We leverage this fact below: no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + let !c = unsafeChr8 m0 + if not (p c) then inner (srcOff + 1) dstOff else do + A.unsafeWrite dst dstOff m0 + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !c = chr2 m0 m1 + if not (p c) then inner (srcOff + 2) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + inner (srcOff + 2) (dstOff + 2) + 3 -> do + let !c = chr3 m0 m1 m2 + if not (p c) then inner (srcOff + 3) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + inner (srcOff + 3) (dstOff + 3) + _ -> do + let !c = chr4 m0 m1 m2 m3 + if not (p c) then inner (srcOff + 4) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + inner (srcOff + 4) (dstOff + 4) +{-# INLINE filter_ #-} diff --git a/src/Data/Text/Lazy.hs b/src/Data/Text/Lazy.hs --- a/src/Data/Text/Lazy.hs +++ b/src/Data/Text/Lazy.hs @@ -1,1777 +1,1911 @@ -{-# OPTIONS_GHC -fno-warn-orphans #-} -{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} -{-# LANGUAGE Trustworthy #-} -{-# LANGUAGE TemplateHaskellQuotes #-} -{-# LANGUAGE LambdaCase #-} - --- | --- Module : Data.Text.Lazy --- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : GHC --- --- A time and space-efficient implementation of Unicode text using --- lists of packed arrays. --- --- /Note/: Read below the synopsis for important notes on the use of --- this module. --- --- The representation used by this module is suitable for high --- performance use and for streaming large quantities of data. It --- provides a means to manipulate a large body of text without --- requiring that the entire content be resident in memory. --- --- Some operations, such as 'concat', 'append', 'reverse' and 'cons', --- have better time complexity than their "Data.Text" equivalents, due --- to the underlying representation being a list of chunks. For other --- operations, lazy 'Text's are usually within a few percent of strict --- ones, but often with better heap usage if used in a streaming --- fashion. For data larger than available memory, or if you have --- tight memory constraints, this module will be the only option. --- --- This module is intended to be imported @qualified@, to avoid name --- clashes with "Prelude" functions. eg. --- --- > import qualified Data.Text.Lazy as L - -module Data.Text.Lazy - ( - -- * Fusion - -- $fusion - - -- * Acceptable data - -- $replacement - - -- * Types - Text - , LazyText - - -- * Creation and elimination - , pack - , unpack - , singleton - , empty - , fromChunks - , toChunks - , toStrict - , fromStrict - , foldrChunks - , foldlChunks - - -- * Basic interface - , cons - , snoc - , append - , uncons - , unsnoc - , head - , last - , tail - , init - , null - , length - , compareLength - - -- * Transformations - , map - , intercalate - , intersperse - , transpose - , reverse - , replace - - -- ** Case conversion - -- $case - , toCaseFold - , toLower - , toUpper - , toTitle - - -- ** Justification - , justifyLeft - , justifyRight - , center - - -- * Folds - , foldl - , foldl' - , foldl1 - , foldl1' - , foldr - , foldr1 - - -- ** Special folds - , concat - , concatMap - , any - , all - , maximum - , minimum - , isAscii - - -- * Construction - - -- ** Scans - , scanl - , scanl1 - , scanr - , scanr1 - - -- ** Accumulating maps - , mapAccumL - , mapAccumR - - -- ** Generation and unfolding - , repeat - , replicate - , cycle - , iterate - , unfoldr - , unfoldrN - - -- * Substrings - - -- ** Breaking strings - , take - , takeEnd - , drop - , dropEnd - , takeWhile - , takeWhileEnd - , dropWhile - , dropWhileEnd - , dropAround - , strip - , stripStart - , stripEnd - , splitAt - , span - , spanM - , spanEndM - , breakOn - , breakOnEnd - , break - , group - , groupBy - , inits - , tails - - -- ** Breaking into many substrings - -- $split - , splitOn - , split - , chunksOf - -- , breakSubstring - - -- ** Breaking into lines and words - , lines - , words - , unlines - , unwords - - -- * Predicates - , isPrefixOf - , isSuffixOf - , isInfixOf - - -- ** View patterns - , stripPrefix - , stripSuffix - , commonPrefixes - - -- * Searching - , filter - , find - , elem - , breakOnAll - , partition - - -- , findSubstring - - -- * Indexing - , index - , count - - -- * Zipping and unzipping - , zip - , zipWith - - -- -* Ordered text - -- , sort - ) where - -import Prelude (Char, Bool(..), Maybe(..), String, - Eq, (==), Ord(..), Ordering(..), Read(..), Show(..), - Monad(..), pure, (<$>), - (&&), (+), (-), (.), ($), (++), - error, flip, fmap, fromIntegral, not, otherwise, quot) -import qualified Prelude as P -import Control.Arrow (first) -import Control.DeepSeq (NFData(..)) -import Data.Bits (finiteBitSize) -import Data.Int (Int64) -import qualified Data.List as L hiding (head, tail) -import Data.Char (isSpace) -import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, - Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) -import Data.Binary (Binary(get, put)) -import Data.List.NonEmpty (NonEmpty(..)) -import qualified Data.List.NonEmpty as NE -import Data.Monoid (Monoid(..)) -import Data.Semigroup (Semigroup(..)) -import Data.String (IsString(..)) -import qualified Data.Text as T -import qualified Data.Text.Array as A -import qualified Data.Text.Internal as T -import qualified Data.Text.Internal.Fusion.Common as S -import qualified Data.Text.Unsafe as T -import qualified Data.Text.Internal.Lazy.Fusion as S -import Data.Text.Internal.Fusion.Types (PairS(..)) -import Data.Text.Internal.Lazy.Fusion (stream, unstream) -import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, - foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText) -import Data.Text.Internal (firstf, safe, text) -import Data.Text.Internal.Reverse (reverseNonEmpty) -import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) -import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8) -import Data.Text.Internal.Lazy.Search (indices) -import qualified GHC.CString as GHC -import qualified GHC.Exts as Exts -import GHC.Prim (Addr#) -import GHC.Stack (HasCallStack) -import qualified Language.Haskell.TH.Lib as TH -import qualified Language.Haskell.TH.Syntax as TH -import Text.Printf (PrintfArg, formatArg, formatString) - --- $fusion --- --- Starting from @text-1.3@ fusion is no longer implicit, --- and pipelines of transformations usually allocate intermediate 'Text' values. --- Users, who observe significant changes to performances, --- are encouraged to use fusion framework explicitly, employing --- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". - --- $replacement --- --- A 'Text' value is a sequence of Unicode scalar values, as defined --- in --- . --- As such, a 'Text' cannot contain values in the range U+D800 to --- U+DFFF inclusive. Haskell implementations admit all Unicode code --- points --- () --- as 'Char' values, including code points from this invalid range. --- This means that there are some 'Char' values --- (corresponding to 'Data.Char.Surrogate' category) that are not valid --- Unicode scalar values, and the functions in this module must handle --- those cases. --- --- Within this module, many functions construct a 'Text' from one or --- more 'Char' values. Those functions will substitute 'Char' values --- that are not valid Unicode scalar values with the replacement --- character \"�\" (U+FFFD). Functions that perform this --- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". The functions replace --- invalid scalar values, instead of dropping them, as a security --- measure. For details, see --- .) - --- $setup --- >>> :set -package transformers --- >>> import Control.Monad.Trans.State --- >>> import Data.Text --- >>> import qualified Data.Text as T --- >>> :seti -XOverloadedStrings - -instance Eq Text where - (==) = equal - {-# INLINE (==) #-} - -instance Ord Text where - compare = compareText - -compareText :: Text -> Text -> Ordering -compareText Empty Empty = EQ -compareText Empty _ = LT -compareText _ Empty = GT -compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = - A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of - LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) - EQ -> compareText as bs - GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs --- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), --- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering --- of underlying bytearrays, no decoding is needed. - -instance Show Text where - showsPrec p ps r = showsPrec p (unpack ps) r - -instance Read Text where - readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] - --- | @since 1.2.2.0 -instance Semigroup Text where - (<>) = append - -instance Monoid Text where - mempty = empty - mappend = (<>) - mconcat = concat - --- | Performs replacement on invalid scalar values: --- --- >>> :set -XOverloadedStrings --- >>> "\55555" :: Data.Text.Lazy.Text --- "\65533" -instance IsString Text where - fromString = pack - --- | Performs replacement on invalid scalar values: --- --- >>> :set -XOverloadedLists --- >>> ['\55555'] :: Data.Text.Lazy.Text --- "\65533" --- --- @since 1.2.0.0 -instance Exts.IsList Text where - type Item Text = Char - fromList = pack - toList = unpack - -instance NFData Text where - rnf Empty = () - rnf (Chunk _ ts) = rnf ts - --- | @since 1.2.1.0 -instance Binary Text where - put t = put (encodeUtf8 t) - get = do - bs <- get - case decodeUtf8' bs of - P.Left exn -> P.fail (P.show exn) - P.Right a -> P.return a - --- | This instance preserves data abstraction at the cost of inefficiency. --- We omit reflection services for the sake of data abstraction. --- --- This instance was created by copying the updated behavior of --- @"Data.Text".@'Data.Text.Text' -instance Data Text where - gfoldl f z txt = z pack `f` (unpack txt) - toConstr _ = packConstr - gunfold k z c = case constrIndex c of - 1 -> k (z pack) - _ -> error "Data.Text.Lazy.Text.gunfold" - dataTypeOf _ = textDataType - --- | @since 1.2.4.0 -instance TH.Lift Text where - lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict -#if MIN_VERSION_template_haskell(2,17,0) - liftTyped = TH.unsafeCodeCoerce . TH.lift -#elif MIN_VERSION_template_haskell(2,16,0) - liftTyped = TH.unsafeTExpCoerce . TH.lift -#endif - --- | @since 1.2.2.0 -instance PrintfArg Text where - formatArg txt = formatString $ unpack txt - -packConstr :: Constr -packConstr = mkConstr textDataType "pack" [] Prefix - -textDataType :: DataType -textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] - --- | /O(n)/ Convert a 'String' into a 'Text'. --- --- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id': --- --- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555") --- "\65533" -pack :: -#if defined(ASSERTS) - HasCallStack => -#endif - String -> Text -pack = unstream . S.streamList . L.map safe -{-# INLINE [1] pack #-} - --- | /O(n)/ Convert a 'Text' into a 'String'. -unpack :: -#if defined(ASSERTS) - HasCallStack => -#endif - Text -> String -unpack t = S.unstreamList (stream t) -{-# INLINE [1] unpack #-} - --- | /O(n)/ Convert a literal string into a Text. -unpackCString# :: Addr# -> Text -unpackCString# addr# = unstream (S.streamCString# addr#) -{-# NOINLINE unpackCString# #-} - -{-# RULES "TEXT literal" forall a. - unstream (S.streamList (L.map safe (GHC.unpackCString# a))) - = unpackCString# a #-} - -{-# RULES "TEXT literal UTF8" forall a. - unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) - = unpackCString# a #-} - -{-# RULES "LAZY TEXT empty literal" - unstream (S.streamList (L.map safe [])) - = Empty #-} - -{-# RULES "LAZY TEXT empty literal" forall a. - unstream (S.streamList (L.map safe [a])) - = Chunk (T.singleton a) Empty #-} - --- | /O(1)/ Convert a character into a Text. --- Performs replacement on invalid scalar values. -singleton :: Char -> Text -singleton c = Chunk (T.singleton c) Empty -{-# INLINE [1] singleton #-} - --- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. -fromChunks :: [T.Text] -> Text -fromChunks cs = L.foldr chunk Empty cs - --- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. -toChunks :: Text -> [T.Text] -toChunks cs = foldrChunks (:) [] cs - --- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. -toStrict :: LazyText -> T.StrictText -toStrict t = T.concat (toChunks t) -{-# INLINE [1] toStrict #-} - --- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. -fromStrict :: T.StrictText -> LazyText -fromStrict t = chunk t Empty -{-# INLINE [1] fromStrict #-} - --- ----------------------------------------------------------------------------- --- * Basic functions - --- | /O(1)/ Adds a character to the front of a 'Text'. -cons :: Char -> Text -> Text -cons c t = Chunk (T.singleton c) t -{-# INLINE [1] cons #-} - -infixr 5 `cons` - --- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process. -snoc :: Text -> Char -> Text -snoc t c = foldrChunks Chunk (singleton c) t -{-# INLINE [1] snoc #-} - --- | /O(n\/c)/ Appends one 'Text' to another. -append :: Text -> Text -> Text -append xs ys = foldrChunks Chunk ys xs -{-# INLINE [1] append #-} - --- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. -uncons :: Text -> Maybe (Char, Text) -uncons Empty = Nothing -uncons (Chunk t ts) = Just (T.unsafeHead t, ts') - where ts' | T.compareLength t 1 == EQ = ts - | otherwise = Chunk (T.unsafeTail t) ts -{-# INLINE uncons #-} - --- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. This is a partial function, consider using 'uncons' instead. -head :: HasCallStack => Text -> Char -head t = S.head (stream t) -{-# INLINE head #-} - --- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. This is a partial function, consider using 'uncons' instead. -tail :: HasCallStack => Text -> Text -tail (Chunk t ts) = chunk (T.tail t) ts -tail Empty = emptyError "tail" -{-# INLINE [1] tail #-} - --- | /O(n\/c)/ Returns all but the last character of a 'Text', which must --- be non-empty. This is a partial function, consider using 'unsnoc' instead. -init :: HasCallStack => Text -> Text -init (Chunk t0 ts0) = go t0 ts0 - where go t (Chunk t' ts) = Chunk t (go t' ts) - go t Empty = chunk (T.init t) Empty -init Empty = emptyError "init" -{-# INLINE [1] init #-} - --- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if --- empty. --- --- * It is no faster than using 'init' and 'last'. --- --- @since 1.2.3.0 -unsnoc :: Text -> Maybe (Text, Char) -unsnoc Empty = Nothing -unsnoc ts@(Chunk _ _) = Just (init ts, last ts) -{-# INLINE unsnoc #-} - --- | /O(1)/ Tests whether a 'Text' is empty or not. -null :: Text -> Bool -null Empty = True -null _ = False -{-# INLINE [1] null #-} - --- | /O(1)/ Tests whether a 'Text' contains exactly one character. -isSingleton :: Text -> Bool -isSingleton = S.isSingleton . stream -{-# INLINE isSingleton #-} - --- | /O(n\/c)/ Returns the last character of a 'Text', which must be --- non-empty. This is a partial function, consider using 'unsnoc' instead. -last :: HasCallStack => Text -> Char -last Empty = emptyError "last" -last (Chunk t ts) = go t ts - where go _ (Chunk t' ts') = go t' ts' - go t' Empty = T.last t' -{-# INLINE [1] last #-} - --- | /O(n)/ Returns the number of characters in a 'Text'. -length :: Text -> Int64 -length = foldlChunks go 0 - where - go :: Int64 -> T.Text -> Int64 - go l t = l + intToInt64 (T.length t) -{-# INLINE [1] length #-} - -{-# RULES -"TEXT length/map -> length" forall f t. - length (map f t) = length t -"TEXT length/zipWith -> length" forall f t1 t2. - length (zipWith f t1 t2) = min (length t1) (length t2) -"TEXT length/replicate -> n" forall n t. - length (replicate n t) = max 0 n P.* length t -"TEXT length/cons -> length+1" forall c t. - length (cons c t) = 1 + length t -"TEXT length/intersperse -> 2*length-1" forall c t. - length (intersperse c t) = max 0 (2 P.* length t - 1) -"TEXT length/intercalate -> n*length" forall s ts. - length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) - #-} - --- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. --- --- @ --- 'compareLength' t c = 'P.compare' ('length' t) c --- @ --- --- This function gives the same answer as comparing against the result --- of 'length', but can short circuit if the count of characters is --- greater than the number, and hence be more efficient. -compareLength :: Text -> Int64 -> Ordering -compareLength t c = S.compareLengthI (stream t) c -{-# INLINE [1] compareLength #-} - --- We don't apply those otherwise appealing length-to-compareLength --- rewrite rules here, because they can change the strictness --- properties of code. - --- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Performs replacement on --- invalid scalar values. -map :: (Char -> Char) -> Text -> Text -map f = foldrChunks (Chunk . mapNonEmpty f) Empty -{-# INLINE [1] map #-} - -{-# RULES -"TEXT map/map -> map" forall f g t. - map f (map g t) = map (f . safe . g) t -#-} - --- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of --- 'Text's and concatenates the list after interspersing the first --- argument between each element of the list. -intercalate :: Text -> [Text] -> Text -intercalate t = concat . L.intersperse t -{-# INLINE [1] intercalate #-} - --- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Performs --- replacement on invalid scalar values. -intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse (safe c) (stream t)) -{-# INLINE [1] intersperse #-} - --- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Performs --- replacement on invalid scalar values. --- --- Examples: --- --- > justifyLeft 7 'x' "foo" == "fooxxxx" --- > justifyLeft 3 'x' "foobar" == "foobar" -justifyLeft :: Int64 -> Char -> Text -> Text -justifyLeft k c t - | len >= k = t - | otherwise = t `append` replicateChunk (k-len) (T.singleton c) - where len = length t -{-# INLINE [1] justifyLeft #-} - --- | /O(n)/ Right-justify a string to the given length, using the --- specified fill character on the left. Performs replacement on --- invalid scalar values. --- --- Examples: --- --- > justifyRight 7 'x' "bar" == "xxxxbar" --- > justifyRight 3 'x' "foobar" == "foobar" -justifyRight :: Int64 -> Char -> Text -> Text -justifyRight k c t - | len >= k = t - | otherwise = replicateChunk (k-len) (T.singleton c) `append` t - where len = length t -{-# INLINE justifyRight #-} - --- | /O(n)/ Center a string to the given length, using the specified --- fill character on either side. Performs replacement on invalid --- scalar values. --- --- Examples: --- --- > center 8 'x' "HS" = "xxxHSxxx" -center :: Int64 -> Char -> Text -> Text -center k c t - | len >= k = t - | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) - where len = length t - d = k - len - r = d `quot` 2 - l = d - r -{-# INLINE center #-} - --- | /O(n)/ The 'transpose' function transposes the rows and columns --- of its 'Text' argument. Note that this function uses 'pack', --- 'unpack', and the list version of transpose, and is thus not very --- efficient. -transpose :: [Text] -> [Text] -transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) - (L.transpose (L.map unpack ts)) --- TODO: make this fast - --- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. -reverse :: -#if defined(ASSERTS) - HasCallStack => -#endif - Text -> Text -reverse = rev Empty - where rev a Empty = a - rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts - --- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in --- @haystack@ with @replacement@. --- --- This function behaves as though it was defined as follows: --- --- @ --- replace needle replacement haystack = --- 'intercalate' replacement ('splitOn' needle haystack) --- @ --- --- As this suggests, each occurrence is replaced exactly once. So if --- @needle@ occurs in @replacement@, that occurrence will /not/ itself --- be replaced recursively: --- --- > replace "oo" "foo" "oo" == "foo" --- --- In cases where several instances of @needle@ overlap, only the --- first one will be replaced: --- --- > replace "ofo" "bar" "ofofo" == "barfo" --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -replace :: HasCallStack - => Text - -- ^ @needle@ to search for. If this string is empty, an - -- error will occur. - -> Text - -- ^ @replacement@ to replace @needle@ with. - -> Text - -- ^ @haystack@ in which to search. - -> Text -replace s d = intercalate d . splitOn s -{-# INLINE replace #-} - --- ---------------------------------------------------------------------------- --- ** Case conversions (folds) - --- $case --- --- With Unicode text, it is incorrect to use combinators like @map --- toUpper@ to case convert each character of a string individually. --- Instead, use the whole-string case conversion functions from this --- module. For correctness in different writing systems, these --- functions may map one input character to two or three output --- characters. - --- | /O(n)/ Convert a string to folded case. --- --- This function is mainly useful for performing caseless (or case --- insensitive) string comparisons. --- --- A string @x@ is a caseless match for a string @y@ if and only if: --- --- @toCaseFold x == toCaseFold y@ --- --- The result string may be longer than the input string, and may --- differ from applying 'toLower' to the input string. For instance, --- the Armenian small ligature men now (U+FB13) is case folded to the --- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is --- case folded to the Greek small letter letter mu (U+03BC) instead of --- itself. -toCaseFold :: Text -> Text -toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty -{-# INLINE toCaseFold #-} - --- | /O(n)/ Convert a string to lower case, using simple case --- conversion. --- --- The result string may be longer than the input string. For --- instance, the Latin capital letter I with dot above (U+0130) maps --- to the sequence Latin small letter i (U+0069) followed by combining --- dot above (U+0307). -toLower :: Text -> Text -toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty -{-# INLINE toLower #-} - --- | /O(n)/ Convert a string to upper case, using simple case --- conversion. --- --- The result string may be longer than the input string. For --- instance, the German eszett (U+00DF) maps to the two-letter --- sequence SS. -toUpper :: Text -> Text -toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty -{-# INLINE toUpper #-} - - --- | /O(n)/ Convert a string to title case, using simple case --- conversion. --- --- The first letter (as determined by 'Data.Char.isLetter') --- of the input is converted to title case, as is --- every subsequent letter that immediately follows a non-letter. --- Every letter that immediately follows another letter is converted --- to lower case. --- --- The result string may be longer than the input string. For example, --- the Latin small ligature fl (U+FB02) is converted to the --- sequence Latin capital letter F (U+0046) followed by Latin small --- letter l (U+006C). --- --- This function is not idempotent. --- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). --- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input --- is converted to title case, becoming two letters. --- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter --- and as such is recognised as a letter by 'Data.Char.isLetter', --- so 'T.toTitle' @"ʼN"@ = @"'n"@. --- --- /Note/: this function does not take language or culture specific --- rules into account. For instance, in English, different style --- guides disagree on whether the book name \"The Hill of the Red --- Fox\" is correctly title cased—but this function will --- capitalize /every/ word. --- --- @since 1.0.0.0 -toTitle :: Text -> Text -toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty -{-# INLINE toTitle #-} - --- | /O(n)/ 'foldl', applied to a binary operator, a starting value --- (typically the left-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from left to right. -foldl :: (a -> Char -> a) -> a -> Text -> a -foldl f z t = S.foldl f z (stream t) -{-# INLINE foldl #-} - --- | /O(n)/ A strict version of 'foldl'. --- -foldl' :: (a -> Char -> a) -> a -> Text -> a -foldl' f z t = S.foldl' f z (stream t) -{-# INLINE foldl' #-} - --- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. -foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldl1 f t = S.foldl1 f (stream t) -{-# INLINE foldl1 #-} - --- | /O(n)/ A strict version of 'foldl1'. -foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldl1' f t = S.foldl1' f (stream t) -{-# INLINE foldl1' #-} - --- | /O(n)/ 'foldr', applied to a binary operator, a starting value --- (typically the right-identity of the operator), and a 'Text', --- reduces the 'Text' using the binary operator, from right to left. --- --- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually --- traverses the 'Text' from left to right, only as far as it needs to. --- --- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': --- --- @ --- head :: Text -> Char --- head = foldr const (error "head empty") --- @ -foldr :: (Char -> a -> a) -> a -> Text -> a -foldr f z t = S.foldr f z (stream t) -{-# INLINE foldr #-} - --- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. -foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char -foldr1 f t = S.foldr1 f (stream t) -{-# INLINE foldr1 #-} - --- | /O(n)/ Concatenate a list of 'Text's. -concat :: [Text] -> Text -concat [] = Empty -concat (Empty : css) = concat css -concat (Chunk c Empty : css) = Chunk c (concat css) -concat (Chunk c cs : css) = Chunk c (concat (cs : css)) -{-# INLINE concat #-} - --- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and --- concatenate the results. -concatMap :: (Char -> Text) -> Text -> Text -concatMap f = concat . foldr ((:) . f) [] -{-# INLINE concatMap #-} - --- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. -any :: (Char -> Bool) -> Text -> Bool -any p t = S.any p (stream t) -{-# INLINE any #-} - --- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. -all :: (Char -> Bool) -> Text -> Bool -all p t = S.all p (stream t) -{-# INLINE all #-} - --- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. -maximum :: HasCallStack => Text -> Char -maximum t = S.maximum (stream t) -{-# INLINE maximum #-} - --- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. -minimum :: HasCallStack => Text -> Char -minimum t = S.minimum (stream t) -{-# INLINE minimum #-} - --- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only --- U+0000 through U+007F). --- --- This is a more efficient version of @'all' 'Data.Char.isAscii'@. --- --- >>> isAscii "" --- True --- --- >>> isAscii "abc\NUL" --- True --- --- >>> isAscii "abcd€" --- False --- --- prop> isAscii t == all (< '\x80') t --- --- @since 2.0.2 -isAscii :: Text -> Bool -isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True - --- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. --- Performs replacement on invalid scalar values. --- --- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] --- --- Note that --- --- > last (scanl f z xs) == foldl f z xs. -scanl :: (Char -> Char -> Char) -> Char -> Text -> Text -scanl f z t = unstream (S.scanl g z (stream t)) - where g a b = safe (f a b) -{-# INLINE scanl #-} - --- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting --- value argument. Performs replacement on invalid scalar values. --- --- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] -scanl1 :: (Char -> Char -> Char) -> Text -> Text -scanl1 f t0 = case uncons t0 of - Nothing -> empty - Just (t,ts) -> scanl f t ts -{-# INLINE scanl1 #-} - --- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs --- replacement on invalid scalar values. --- --- > scanr f v == reverse . scanl (flip f) v . reverse -scanr :: (Char -> Char -> Char) -> Char -> Text -> Text -scanr f v = reverse . scanl g v . reverse - where g a b = safe (f b a) - --- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting --- value argument. Performs replacement on invalid scalar values. -scanr1 :: (Char -> Char -> Char) -> Text -> Text -scanr1 f t | null t = empty - | otherwise = scanr f (last t) (init t) - --- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a --- function to each element of a 'Text', passing an accumulating --- parameter from left to right, and returns a final 'Text'. Performs --- replacement on invalid scalar values. -mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumL f = go - where - go z (Chunk c cs) = (z'', Chunk c' cs') - where (z', c') = T.mapAccumL f z c - (z'', cs') = go z' cs - go z Empty = (z, Empty) -{-# INLINE mapAccumL #-} - --- | The 'mapAccumR' function behaves like a combination of 'map' and --- a strict 'foldr'; it applies a function to each element of a --- 'Text', passing an accumulating parameter from right to left, and --- returning a final value of this accumulator together with the new --- 'Text'. Performs replacement on invalid scalar values. -mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumR f = go - where - go z (Chunk c cs) = (z'', Chunk c' cs') - where (z'', c') = T.mapAccumR f z' c - (z', cs') = go z cs - go z Empty = (z, Empty) -{-# INLINE mapAccumR #-} - --- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every --- element. --- --- @since 1.2.0.5 -repeat :: Char -> Text -repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t - in t - --- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input --- @t@ repeated @n@ times. -replicate :: Int64 -> Text -> Text -replicate n - | n <= 0 = P.const Empty - | otherwise = \case - Empty -> Empty - Chunk t Empty -> replicateChunk n t - t -> concat (rep n) - where - rep 0 = [] - rep i = t : rep (i - 1) -{-# INLINE [1] replicate #-} - -replicateChunk :: Int64 -> T.Text -> Text -replicateChunk !n !t@(T.Text _ _ len) - | n <= 0 = Empty - | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) - where - perChunk = defaultChunkSize `quot` len - normalChunk = T.replicate perChunk t - (q, r) = n `P.quotRem` intToInt64 perChunk - headChunk = T.replicate (int64ToInt r) t -{-# INLINE replicateChunk #-} - --- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or --- equivalently, the infinite repetition of the original 'Text'. --- --- @since 1.2.0.5 -cycle :: HasCallStack => Text -> Text -cycle Empty = emptyError "cycle" -cycle t = let t' = foldrChunks Chunk t' t - in t' - --- | @'iterate' f x@ returns an infinite 'Text' of repeated applications --- of @f@ to @x@: --- --- > iterate f x == [x, f x, f (f x), ...] --- --- @since 1.2.0.5 -iterate :: (Char -> Char) -> Char -> Text -iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) - in t c - --- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' --- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a --- 'Text' from a seed value. The function takes the element and --- returns 'Nothing' if it is done producing the 'Text', otherwise --- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. --- Performs replacement on invalid scalar values. -unfoldr :: (a -> Maybe (Char,a)) -> a -> Text -unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) -{-# INLINE unfoldr #-} - --- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed --- value. However, the length of the result should be limited by the --- first argument to 'unfoldrN'. This function is more efficient than --- 'unfoldr' when the maximum length of the result is known and --- correct, otherwise its performance is similar to 'unfoldr'. --- Performs replacement on invalid scalar values. -unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text -unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) -{-# INLINE unfoldrN #-} - --- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the --- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. -take :: Int64 -> Text -> Text -take i _ | i <= 0 = Empty -take i t0 = take' i t0 - where - take' :: Int64 -> Text -> Text - take' 0 _ = Empty - take' _ Empty = Empty - take' n (Chunk t@(T.Text arr off _) ts) - | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = - if m >= 0 - then fromStrict (T.Text arr off m) - else Chunk t (take' (n + intToInt64 m) ts) - - | n < l = Chunk (T.take (int64ToInt n) t) Empty - | otherwise = Chunk t (take' (n - l) ts) - where l = intToInt64 (T.length t) -{-# INLINE [1] take #-} - --- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after --- taking @n@ characters from the end of @t@. --- --- Examples: --- --- > takeEnd 3 "foobar" == "bar" --- --- @since 1.1.1.0 -takeEnd :: Int64 -> Text -> Text -takeEnd n t0 - | n <= 0 = empty - | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 - where - takeChunk :: Int64 -> Text -> [T.Text] -> Text - takeChunk _ acc [] = acc - takeChunk i acc (t:ts) - | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc - | otherwise = takeChunk (i-l) (Chunk t acc) ts - where l = intToInt64 (T.length t) - --- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the --- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. -drop :: Int64 -> Text -> Text -drop i t0 - | i <= 0 = t0 - | otherwise = drop' i t0 - where - drop' :: Int64 -> Text -> Text - drop' 0 ts = ts - drop' _ Empty = Empty - drop' n (Chunk t@(T.Text arr off len) ts) - | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = - if m >= 0 - then chunk (T.Text arr (off + m) (len - m)) ts - else drop' (n + intToInt64 m) ts - - | n < l = Chunk (T.drop (int64ToInt n) t) ts - | otherwise = drop' (n - l) ts - where l = intToInt64 (T.length t) -{-# INLINE [1] drop #-} - --- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after --- dropping @n@ characters from the end of @t@. --- --- Examples: --- --- > dropEnd 3 "foobar" == "foo" --- --- @since 1.1.1.0 -dropEnd :: Int64 -> Text -> Text -dropEnd n t0 - | n <= 0 = t0 - | otherwise = dropChunk n . L.reverse . toChunks $ t0 - where - dropChunk :: Int64 -> [T.Text] -> Text - dropChunk _ [] = empty - dropChunk m (t:ts) - | m >= l = dropChunk (m-l) ts - | otherwise = fromChunks . L.reverse $ - T.dropEnd (int64ToInt m) t : ts - where l = intToInt64 (T.length t) - --- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' --- values dropped, or the empty 'Text' if @n@ is greater than the --- number of 'Word8' values present. -dropWords :: Int64 -> Text -> Text -dropWords i t0 - | i <= 0 = t0 - | otherwise = drop' i t0 - where - drop' :: Int64 -> Text -> Text - drop' 0 ts = ts - drop' _ Empty = Empty - drop' n (Chunk (T.Text arr off len) ts) - | n < len' = chunk (text arr (off+n') (len-n')) ts - | otherwise = drop' (n - len') ts - where len' = intToInt64 len - n' = int64ToInt n - --- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', --- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. -takeWhile :: (Char -> Bool) -> Text -> Text -takeWhile p t0 = takeWhile' t0 - where takeWhile' Empty = Empty - takeWhile' (Chunk t ts) = - case T.findIndex (not . p) t of - Just n | n > 0 -> Chunk (T.take n t) Empty - | otherwise -> Empty - Nothing -> Chunk t (takeWhile' ts) -{-# INLINE [1] takeWhile #-} - --- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', --- returns the longest suffix (possibly empty) of elements that --- satisfy @p@. --- Examples: --- --- > takeWhileEnd (=='o') "foo" == "oo" --- --- @since 1.2.2.0 -takeWhileEnd :: (Char -> Bool) -> Text -> Text -takeWhileEnd p = takeChunk empty . L.reverse . toChunks - where takeChunk acc [] = acc - takeChunk acc (t:ts) - | T.lengthWord8 t' < T.lengthWord8 t - = chunk t' acc - | otherwise = takeChunk (Chunk t' acc) ts - where t' = T.takeWhileEnd p t -{-# INLINE takeWhileEnd #-} - --- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. -dropWhile :: (Char -> Bool) -> Text -> Text -dropWhile p t0 = dropWhile' t0 - where dropWhile' Empty = Empty - dropWhile' (Chunk t ts) = - case T.findIndex (not . p) t of - Just n -> Chunk (T.drop n t) ts - Nothing -> dropWhile' ts -{-# INLINE [1] dropWhile #-} - --- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after --- dropping characters that satisfy the predicate @p@ from the end of --- @t@. --- --- Examples: --- --- > dropWhileEnd (=='.') "foo..." == "foo" -dropWhileEnd :: (Char -> Bool) -> Text -> Text -dropWhileEnd p = go - where go Empty = Empty - go (Chunk t Empty) = if T.null t' - then Empty - else Chunk t' Empty - where t' = T.dropWhileEnd p t - go (Chunk t ts) = case go ts of - Empty -> go (Chunk t Empty) - ts' -> Chunk t ts' -{-# INLINE dropWhileEnd #-} - --- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after --- dropping characters that satisfy the predicate @p@ from both the --- beginning and end of @t@. -dropAround :: (Char -> Bool) -> Text -> Text -dropAround p = dropWhile p . dropWhileEnd p -{-# INLINE [1] dropAround #-} - --- | /O(n)/ Remove leading white space from a string. Equivalent to: --- --- > dropWhile isSpace -stripStart :: Text -> Text -stripStart = dropWhile isSpace -{-# INLINE stripStart #-} - --- | /O(n)/ Remove trailing white space from a string. Equivalent to: --- --- > dropWhileEnd isSpace -stripEnd :: Text -> Text -stripEnd = dropWhileEnd isSpace -{-# INLINE [1] stripEnd #-} - --- | /O(n)/ Remove leading and trailing white space from a string. --- Equivalent to: --- --- > dropAround isSpace -strip :: Text -> Text -strip = dropAround isSpace -{-# INLINE [1] strip #-} - --- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a --- prefix of @t@ of length @n@, and whose second is the remainder of --- the string. It is equivalent to @('take' n t, 'drop' n t)@. -splitAt :: Int64 -> Text -> (Text, Text) -splitAt = loop - where - loop :: Int64 -> Text -> (Text, Text) - loop _ Empty = (empty, empty) - loop n t | n <= 0 = (empty, t) - loop n (Chunk t ts) - | n < len = let (t',t'') = T.splitAt (int64ToInt n) t - in (Chunk t' Empty, Chunk t'' ts) - | otherwise = let (ts',ts'') = loop (n - len) ts - in (Chunk t ts', ts'') - where len = intToInt64 (T.length t) - --- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first --- element is a prefix of @t@ whose chunks contain @n@ 'Word8' --- values, and whose second is the remainder of the string. -splitAtWord :: Int64 -> Text -> PairS Text Text -splitAtWord _ Empty = empty :*: empty -splitAtWord x (Chunk c@(T.Text arr off len) cs) - | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs - in Chunk c h :*: t - | otherwise = chunk (text arr off y) empty :*: - chunk (text arr (off+y) (len-y)) cs - where y = int64ToInt x - --- | /O(n+m)/ Find the first instance of @needle@ (which must be --- non-'null') in @haystack@. The first element of the returned tuple --- is the prefix of @haystack@ before @needle@ is matched. The second --- is the remainder of @haystack@, starting with the match. --- --- Examples: --- --- > breakOn "::" "a::b::c" ==> ("a", "::b::c") --- > breakOn "/" "foobar" ==> ("foobar", "") --- --- Laws: --- --- > append prefix match == haystack --- > where (prefix, match) = breakOn needle haystack --- --- If you need to break a string by a substring repeatedly (e.g. you --- want to break on every instance of a substring), use 'breakOnAll' --- instead, as it has lower startup overhead. --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -breakOn :: HasCallStack => Text -> Text -> (Text, Text) -breakOn pat src - | null pat = emptyError "breakOn" - | otherwise = case indices pat src of - [] -> (src, empty) - (x:_) -> let h :*: t = splitAtWord x src - in (h, t) - --- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. --- --- The first element of the returned tuple is the prefix of @haystack@ --- up to and including the last match of @needle@. The second is the --- remainder of @haystack@, following the match. --- --- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") -breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) -breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) - in (reverse b, reverse a) -{-# INLINE breakOnEnd #-} - --- | /O(n+m)/ Find all non-overlapping instances of @needle@ in --- @haystack@. Each element of the returned list consists of a pair: --- --- * The entire string prior to the /k/th match (i.e. the prefix) --- --- * The /k/th match, followed by the remainder of the string --- --- Examples: --- --- > breakOnAll "::" "" --- > ==> [] --- > breakOnAll "/" "a/b/c/" --- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. --- --- The @needle@ parameter may not be empty. -breakOnAll :: HasCallStack - => Text -- ^ @needle@ to search for - -> Text -- ^ @haystack@ in which to search - -> [(Text, Text)] -breakOnAll pat src - | null pat = emptyError "breakOnAll" - | otherwise = go 0 empty src (indices pat src) - where - go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s - h' = append p h - in (h',t) : go x h' t xs - go _ _ _ _ = [] - --- | /O(n)/ 'break' is like 'span', but the prefix returned is over --- elements that fail the predicate @p@. --- --- >>> T.break (=='c') "180cm" --- ("180","cm") -break :: (Char -> Bool) -> Text -> (Text, Text) -break p t0 = break' t0 - where break' Empty = (empty, empty) - break' c@(Chunk t ts) = - case T.findIndex p t of - Nothing -> let (ts', ts'') = break' ts - in (Chunk t ts', ts'') - Just n | n == 0 -> (Empty, c) - | otherwise -> let (a,b) = T.splitAt n t - in (Chunk a Empty, Chunk b ts) - --- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns --- a pair whose first element is the longest prefix (possibly empty) --- of @t@ of elements that satisfy @p@, and whose second is the --- remainder of the text. --- --- >>> T.span (=='0') "000AB" --- ("000","AB") -span :: (Char -> Bool) -> Text -> (Text, Text) -span p = break (not . p) -{-# INLINE span #-} - --- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, --- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of --- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. --- --- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 --- (("abc","efg"),101) --- --- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': --- --- @ --- -- for all p :: Char -> Bool --- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) --- @ --- --- @since 2.0.1 -spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) -spanM p t0 = go t0 - where - go Empty = pure (empty, empty) - go (Chunk t ts) = do - (t1, t2) <- T.spanM p t - if T.null t2 then first (chunk t) <$> go ts - else pure (chunk t1 empty, Chunk t2 ts) -{-# INLINE spanM #-} - --- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, --- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of --- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. --- --- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 --- (("tuv","xyz"),118) --- --- @ --- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p --- @ --- --- @since 2.0.1 -spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) -spanEndM p t0 = go t0 - where - go Empty = pure (empty, empty) - go (Chunk t ts) = do - (t3, t4) <- go ts - if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t - else pure (Chunk t t3, t4) -{-# INLINE spanEndM #-} - --- | The 'group' function takes a 'Text' and returns a list of 'Text's --- such that the concatenation of the result is equal to the argument. --- Moreover, each sublist in the result contains only equal elements. --- For example, --- --- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] --- --- It is a special case of 'groupBy', which allows the programmer to --- supply their own equality test. -group :: Text -> [Text] -group = groupBy (==) -{-# INLINE group #-} - --- | The 'groupBy' function is the non-overloaded version of 'group'. -groupBy :: (Char -> Char -> Bool) -> Text -> [Text] -groupBy _ Empty = [] -groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs - where (ys,zs) = span (eq x) xs - x = T.unsafeHead t - xs = chunk (T.unsafeTail t) ts - --- | /O(n)/ Return all initial segments of the given 'Text', --- shortest first. -inits :: Text -> [Text] -inits = (Empty :) . inits' - where inits' Empty = [] - inits' (Chunk t ts) = L.map (\t' -> Chunk t' Empty) (L.drop 1 (T.inits t)) - ++ L.map (Chunk t) (inits' ts) - --- | /O(n)/ Return all final segments of the given 'Text', longest --- first. -tails :: Text -> [Text] -tails Empty = Empty : [] -tails ts@(Chunk t ts') - | T.length t == 1 = ts : tails ts' - | otherwise = ts : tails (Chunk (T.unsafeTail t) ts') - --- $split --- --- Splitting functions in this library do not perform character-wise --- copies to create substrings; they just construct new 'Text's that --- are slices of the original. - --- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' --- argument (which cannot be an empty string), consuming the --- delimiter. An empty delimiter is invalid, and will cause an error --- to be raised. --- --- Examples: --- --- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] --- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] --- > splitOn "x" "x" == ["",""] --- --- and --- --- > intercalate s . splitOn s == id --- > splitOn (singleton c) == split (==c) --- --- (Note: the string @s@ to split on above cannot be empty.) --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -splitOn :: HasCallStack - => Text - -- ^ String to split on. If this string is empty, an error - -- will occur. - -> Text - -- ^ Input text. - -> [Text] -splitOn pat src - | null pat = emptyError "splitOn" - | isSingleton pat = split (== head pat) src - | otherwise = go 0 (indices pat src) src - where - go _ [] cs = [cs] - go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs - in h : go (x+l) xs (dropWords l t) - l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat -{-# INLINE [1] splitOn #-} - -{-# RULES -"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. - splitOn (singleton c) t = split (==c) t - #-} - --- | /O(n)/ Splits a 'Text' into components delimited by separators, --- where the predicate returns True for a separator element. The --- resulting components do not contain the separators. Two adjacent --- separators result in an empty component in the output. eg. --- --- > split (=='a') "aabbaca" == ["","","bb","c",""] --- > split (=='a') [] == [""] -split :: (Char -> Bool) -> Text -> [Text] -split _ Empty = [Empty] -split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 - where comb acc (s:[]) Empty = revChunks (s:acc) : [] - comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts - comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts - comb _ [] _ = impossibleError "split" -{-# INLINE split #-} - --- | /O(n)/ Splits a 'Text' into components of length @k@. The last --- element may be shorter than the other chunks, depending on the --- length of the input. Examples: --- --- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] --- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] -chunksOf :: Int64 -> Text -> [Text] -chunksOf k = go - where - go t = case splitAt k t of - (a,b) | null a -> [] - | otherwise -> a : go b -{-# INLINE chunksOf #-} - --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters --- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. --- --- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. -lines :: Text -> [Text] -lines Empty = [] -lines t = NE.toList $ go t - where - go :: Text -> NonEmpty Text - go Empty = Empty :| [] - go (Chunk x xs) - -- x is non-empty, so T.lines x is non-empty as well - | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs - | otherwise = case unsnocList (T.lines x) of - Nothing -> impossibleError "lines" - Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls - -prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text -prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! - -unsnocList :: [a] -> Maybe ([a], a) -unsnocList [] = Nothing -unsnocList (x : xs) = Just $ go x xs - where - go y [] = ([], y) - go y (z : zs) = first (y :) (go z zs) - -hasNlEnd :: T.Text -> Bool -hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A - --- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's --- representing white space. -words :: Text -> [Text] -words = L.filter (not . null) . split isSpace -{-# INLINE words #-} - --- | /O(n)/ Joins lines, after appending a terminating newline to --- each. -unlines :: [Text] -> Text -unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] -{-# INLINE unlines #-} - --- | /O(n)/ Joins words using single space characters. -unwords :: [Text] -> Text -unwords = intercalate (singleton ' ') -{-# INLINE unwords #-} - --- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' if and only if the first is a prefix of the second. -isPrefixOf :: Text -> Text -> Bool -isPrefixOf Empty _ = True -isPrefixOf _ Empty = False -isPrefixOf (Chunk x xs) (Chunk y ys) - | lx == ly = x == y && isPrefixOf xs ys - | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) - | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys - where (xh,xt) = T.splitAt ly x - (yh,yt) = T.splitAt lx y - lx = T.length x - ly = T.length y - --- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns --- 'True' if and only if the first is a suffix of the second. -isSuffixOf :: Text -> Text -> Bool -isSuffixOf x y = reverse x `isPrefixOf` reverse y -{-# INLINE isSuffixOf #-} --- TODO: a better implementation - --- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns --- 'True' if and only if the first is contained, wholly and intact, anywhere --- within the second. --- --- This function is strict in its first argument, and lazy in its --- second. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -isInfixOf :: Text -> Text -> Bool -isInfixOf needle haystack - | null needle = True - | isSingleton needle = S.elem (head needle) . S.stream $ haystack - | otherwise = not . L.null . indices needle $ haystack -{-# INLINE [1] isInfixOf #-} - -------------------------------------------------------------------------------- --- * View patterns - --- | /O(n)/ Return the suffix of the second string if its prefix --- matches the entire first string. --- --- Examples: --- --- > stripPrefix "foo" "foobar" == Just "bar" --- > stripPrefix "" "baz" == Just "baz" --- > stripPrefix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text.Lazy as T --- > --- > fnordLength :: Text -> Int --- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf --- > fnordLength _ = -1 -stripPrefix :: Text -> Text -> Maybe Text -stripPrefix p t - | null p = Just t - | otherwise = case commonPrefixes p t of - Just (_,c,r) | null c -> Just r - _ -> Nothing - --- | /O(n)/ Find the longest non-empty common prefix of two strings --- and return it, along with the suffixes of each string at which they --- no longer match. --- --- If the strings do not have a common prefix or either one is empty, --- this function returns 'Nothing'. --- --- Examples: --- --- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") --- > commonPrefixes "veeble" "fetzer" == Nothing --- > commonPrefixes "" "baz" == Nothing -commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) -commonPrefixes Empty _ = Nothing -commonPrefixes _ Empty = Nothing -commonPrefixes a0 b0 = Just (go a0 b0 []) - where - go t0@(Chunk x xs) t1@(Chunk y ys) ps - = case T.commonPrefixes x y of - Just (p,a,b) - | T.null a -> go xs (chunk b ys) (p:ps) - | T.null b -> go (chunk a xs) ys (p:ps) - | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) - Nothing -> (fromChunks (L.reverse ps),t0,t1) - go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) - --- | /O(n)/ Return the prefix of the second string if its suffix --- matches the entire first string. --- --- Examples: --- --- > stripSuffix "bar" "foobar" == Just "foo" --- > stripSuffix "" "baz" == Just "baz" --- > stripSuffix "foo" "quux" == Nothing --- --- This is particularly useful with the @ViewPatterns@ extension to --- GHC, as follows: --- --- > {-# LANGUAGE ViewPatterns #-} --- > import Data.Text.Lazy as T --- > --- > quuxLength :: Text -> Int --- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre --- > quuxLength _ = -1 -stripSuffix :: Text -> Text -> Maybe Text -stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) - --- | /O(n)/ 'filter', applied to a predicate and a 'Text', --- returns a 'Text' containing those characters that satisfy the --- predicate. -filter :: (Char -> Bool) -> Text -> Text -filter p = foldrChunks (chunk . filter_ T.Text p) Empty -{-# INLINE [1] filter #-} - -{-# RULES -"TEXT filter/filter -> filter" forall p q t. - filter p (filter q t) = filter (\c -> q c && p c) t -#-} - --- | /O(n)/ The 'find' function takes a predicate and a 'Text', and --- returns the first element in matching the predicate, or 'Nothing' --- if there is no such element. -find :: (Char -> Bool) -> Text -> Maybe Char -find p t = S.findBy p (stream t) -{-# INLINE find #-} - --- | /O(n)/ The 'elem' function takes a character and a 'Text', and --- returns 'True' if the element is found in the given 'Text', or --- 'False' otherwise. -elem :: Char -> Text -> Bool -elem c t = S.any (== c) (stream t) -{-# INLINE elem #-} - --- | /O(n)/ The 'partition' function takes a predicate and a 'Text', --- and returns the pair of 'Text's with elements which do and do not --- satisfy the predicate, respectively; i.e. --- --- > partition p t == (filter p t, filter (not . p) t) -partition :: (Char -> Bool) -> Text -> (Text, Text) -partition p t = (filter p t, filter (not . p) t) -{-# INLINE partition #-} - --- | /O(n)/ 'Text' index (subscript) operator, starting from 0. -index :: HasCallStack => Text -> Int64 -> Char -index t n = S.index (stream t) n -{-# INLINE index #-} - --- | /O(n+m)/ The 'count' function returns the number of times the --- query string appears in the given 'Text'. An empty query string is --- invalid, and will cause an error to be raised. --- --- In (unlikely) bad cases, this function's time complexity degrades --- towards /O(n*m)/. -count :: HasCallStack => Text -> Text -> Int64 -count pat - | null pat = emptyError "count" - | otherwise = go 0 . indices pat - where go !n [] = n - go !n (_:xs) = go (n+1) xs -{-# INLINE [1] count #-} - -{-# RULES -"LAZY TEXT count/singleton -> countChar" [~1] forall c t. - count (singleton c) t = countChar c t - #-} - --- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. -countChar :: Char -> Text -> Int64 -countChar c t = S.countChar c (stream t) - --- | /O(n)/ 'zip' takes two 'Text's and returns a list of --- corresponding pairs of bytes. If one input 'Text' is short, --- excess elements of the longer 'Text' are discarded. This is --- equivalent to a pair of 'unpack' operations. -zip :: Text -> Text -> [(Char,Char)] -zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) -{-# INLINE [0] zip #-} - --- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function --- given as the first argument, instead of a tupling function. --- Performs replacement on invalid scalar values. -zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text -zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) - where g a b = safe (f a b) -{-# INLINE [0] zipWith #-} - -revChunks :: [T.Text] -> Text -revChunks = L.foldl' (flip chunk) Empty - -emptyError :: HasCallStack => String -> a -emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") - -impossibleError :: HasCallStack => String -> a -impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") - -intToInt64 :: Exts.Int -> Int64 -intToInt64 = fromIntegral - -int64ToInt :: Int64 -> Exts.Int -int64ToInt = fromIntegral +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy +-- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text using +-- lists of packed arrays. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- The representation used by this module is suitable for high +-- performance use and for streaming large quantities of data. It +-- provides a means to manipulate a large body of text without +-- requiring that the entire content be resident in memory. +-- +-- Some operations, such as 'concat', 'append', 'reverse' and 'cons', +-- have better time complexity than their "Data.Text" equivalents, due +-- to the underlying representation being a list of chunks. For other +-- operations, lazy 'Text's are usually within a few percent of strict +-- ones, but often with better heap usage if used in a streaming +-- fashion. For data larger than available memory, or if you have +-- tight memory constraints, this module will be the only option. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions. eg. +-- +-- > import qualified Data.Text.Lazy as L + +module Data.Text.Lazy + ( + -- * Fusion + -- $fusion + + -- * Acceptable data + -- $replacement + + -- * Types + Text + , LazyText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + , fromChunks + , toChunks + , toStrict + , fromStrict + , foldrChunks + , foldlChunks + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , repeat + , replicate + , cycle + , iterate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , span + , spanM + , spanEndM + , breakOn + , breakOnEnd + , break + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + -- , breakSubstring + + -- ** Breaking into lines and words + , lines + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , find + , elem + , breakOnAll + , partition + + -- , findSubstring + + -- * Indexing + , index + , count + + -- * Zipping and unzipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + ) where + +import Prelude (Char, Bool(..), Maybe(..), String, + Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec), + Monad(..), pure, (<$>), + (&&), (+), (-), (.), ($), (++), + error, flip, fmap, fromIntegral, not, otherwise, quot) +import qualified Prelude as P +import Control.Arrow (first) +import Control.DeepSeq (NFData(..)) +import Data.Bits (finiteBitSize, toIntegralSized) +import Data.Int (Int64) +import qualified Data.List as L hiding (head, tail) +import Data.Char (isSpace) +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Unsafe as T +import qualified Data.Text.Internal.Lazy.Fusion as S +import Data.Text.Internal.Fusion.Types (PairS(..)) +import Data.Text.Internal.Lazy.Fusion (stream, unstream) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, + foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText) +import Data.Text.Internal (firstf, safe, text) +import Data.Text.Internal.Reverse (reverseNonEmpty) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) +import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Lazy.Search (indices) +import qualified GHC.CString as GHC +import qualified GHC.Exts as Exts +import GHC.Prim (Addr#) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Syntax as TH +import qualified Language.Haskell.TH.Lib as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- . +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- () +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- .) + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +instance Eq Text where + (==) = equal + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +compareText :: Text -> Text -> Ordering +compareText Empty Empty = EQ +compareText Empty _ = LT +compareText _ Empty = GT +compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of + LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) + EQ -> compareText as bs + GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +instance Show Text where + showsPrec p ps r = showsPrec p (unpack ps) r + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +instance Semigroup Text where + (<>) = append + stimes n _ | n < 0 = P.error "Data.Text.Lazy.stimes: given number is negative!" + stimes n a = + let nInt64 = fromIntegral n :: Int64 + len = if n == fromIntegral nInt64 && nInt64 >= 0 then nInt64 else P.maxBound + -- We clamp the length to maxBound :: Int64. + -- To tell the difference, the caller would have to skip through 2^63 chunks. + in replicate len a + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Data.Text.Lazy.Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Data.Text.Lazy.Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Text".@'Data.Text.Text' +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> error "Data.Text.Lazy.Text.gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 900 + lift x = [| fromStrict $(TH.lift . toStrict $ x) |] +#else + lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- +-- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555") +-- "\65533" +pack :: +#if defined(ASSERTS) + HasCallStack => +#endif + String -> Text +pack = unstream . S.streamList . L.map safe +{-# INLINE [1] pack #-} + +-- | /O(n)/ Convert a 'Text' into a 'String'. +unpack :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> String +unpack t = S.unstreamList (stream t) +{-# NOINLINE unpack #-} + +foldrFB :: (Char -> b -> b) -> b -> Text -> b +foldrFB = foldr +{-# INLINE [0] foldrFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us `foldrFB (:) []` +-- and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrFB` inlines and `foldr` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.Lazy.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrFB lcons lnil t) +"Text.Lazy.unpackBack" [1] foldrFB (:) [] = unpack + #-} + +-- | /O(n)/ Convert a literal string into a Text. +unpackCString# :: Addr# -> Text +unpackCString# addr# = unstream (S.streamCString# addr#) +{-# NOINLINE unpackCString# #-} + +{-# RULES "TEXT literal" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCString# a))) + = unpackCString# a #-} + +{-# RULES "TEXT literal UTF8" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) + = unpackCString# a #-} + +{-# RULES "LAZY TEXT empty literal" + unstream (S.streamList (L.map safe [])) + = Empty #-} + +{-# RULES "LAZY TEXT empty literal" forall a. + unstream (S.streamList (L.map safe [a])) + = Chunk (T.singleton a) Empty #-} + +-- | /O(1)/ Convert a character into a Text. +-- Performs replacement on invalid scalar values. +singleton :: Char -> Text +singleton c = Chunk (T.singleton c) Empty +{-# INLINE [1] singleton #-} + +-- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. +fromChunks :: [T.Text] -> Text +fromChunks cs = L.foldr chunk Empty cs + +-- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. +toChunks :: Text -> [T.Text] +toChunks cs = foldrChunks (:) [] cs + +-- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. +toStrict :: LazyText -> T.StrictText +toStrict t = T.concat (toChunks t) +{-# INLINE [1] toStrict #-} + +-- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. +fromStrict :: T.StrictText -> LazyText +fromStrict t = chunk t Empty +{-# INLINE [1] fromStrict #-} + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(1)/ Adds a character to the front of a 'Text'. +cons :: Char -> Text -> Text +cons c t = Chunk (T.singleton c) t +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +snoc :: Text -> Char -> Text +snoc t c = foldrChunks Chunk (singleton c) t +{-# INLINE [1] snoc #-} + +-- | /O(n\/c)/ Appends one 'Text' to another. +append :: Text -> Text -> Text +append xs ys = foldrChunks Chunk ys xs +{-# INLINE [1] append #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons Empty = Nothing +uncons (Chunk t ts) = Just (T.unsafeHead t, ts') + where ts' | T.compareLength t 1 == EQ = ts + | otherwise = Chunk (T.unsafeTail t) ts +{-# INLINE uncons #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t = S.head (stream t) +{-# INLINE head #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail (Chunk t ts) = chunk (T.tail t) ts +tail Empty = emptyError "tail" +{-# INLINE [1] tail #-} + +-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init (Chunk t0 ts0) = go t0 ts0 + where go t (Chunk t' ts) = Chunk t (go t' ts) + go t Empty = chunk (T.init t) Empty +init Empty = emptyError "init" +{-# INLINE [1] init #-} + +-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if +-- empty. +-- +-- * It is no faster than using 'init' and 'last'. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc Empty = Nothing +unsnoc ts@(Chunk _ _) = Just (init ts, last ts) +{-# INLINE unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null Empty = True +null _ = False +{-# INLINE [1] null #-} + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + +-- | /O(n\/c)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last Empty = emptyError "last" +last (Chunk t ts) = go t ts + where go _ (Chunk t' ts') = go t' ts' + go t' Empty = T.last t' +{-# INLINE [1] last #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: Text -> Int64 +length = foldlChunks go 0 + where + go :: Int64 -> T.Text -> Int64 + go l t = l + intToInt64 (T.length t) +{-# INLINE [1] length #-} + +{-# RULES +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = max 0 n P.* length t +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (2 P.* length t - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int64 -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +-- We don't apply those otherwise appealing length-to-compareLength +-- rewrite rules here, because they can change the strictness +-- properties of code. + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. Performs replacement on +-- invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = foldrChunks (Chunk . mapNonEmpty f) Empty +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. Performs +-- replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t = unstream (S.intersperse (safe c) (stream t)) +{-# INLINE [1] intersperse #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. Performs +-- replacement on invalid scalar values. +-- +-- Examples: +-- +-- > justifyLeft 7 'x' "foo" == "fooxxxx" +-- > justifyLeft 3 'x' "foobar" == "foobar" +justifyLeft :: Int64 -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChunk (k-len) (T.singleton c) + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- > justifyRight 7 'x' "bar" == "xxxxbar" +-- > justifyRight 3 'x' "foobar" == "foobar" +justifyRight :: Int64 -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChunk (k-len) (T.singleton c) `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- > center 8 'x' "HS" = "xxxHSxxx" +center :: Int64 -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +transpose :: [Text] -> [Text] +transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) + (L.transpose (L.map unpack ts)) +-- TODO: make this fast + +-- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text +reverse = rev Empty + where rev a Empty = a + rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- > replace "oo" "foo" "oo" == "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- > replace "ofo" "bar" "ofofo" == "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace s d = intercalate d . splitOn s +{-# INLINE replace #-} + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- With Unicode text, it is incorrect to use combinators like @map +-- toUpper@ to case convert each character of a string individually. +-- Instead, use the whole-string case conversion functions from this +-- module. For correctness in different writing systems, these +-- functions may map one input character to two or three output +-- characters. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (or case +-- insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature men now (U+FB13) is case folded to the +-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is +-- case folded to the Greek small letter letter mu (U+03BC) instead of +-- itself. +toCaseFold :: Text -> Text +toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the Latin capital letter I with dot above (U+0130) maps +-- to the sequence Latin small letter i (U+0069) followed by combining +-- dot above (U+0307). +toLower :: Text -> Text +toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German eszett (U+00DF) maps to the two-letter +-- sequence SS. +toUpper :: Text -> Text +toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty +{-# INLINE toUpper #-} + + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty +{-# INLINE toTitle #-} + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z t = S.foldl f z (stream t) +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +-- +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z t = S.foldl' f z (stream t) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = S.foldl1 f (stream t) +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = S.foldl1' f (stream t) +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z t = S.foldlM' f z (stream t) +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z t = S.foldr f z (stream t) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = S.foldr1 f (stream t) +{-# INLINE foldr1 #-} + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat [] = Empty +concat (Empty : css) = concat css +concat (Chunk c Empty : css) = Chunk c (concat css) +concat (Chunk c cs : css) = Chunk c (concat (cs : css)) +{-# INLINE concat #-} + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p t = S.any p (stream t) +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p t = S.all p (stream t) +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum t = S.maximum (stream t) +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum t = S.minimum (stream t) +{-# INLINE minimum #-} + +-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only +-- U+0000 through U+007F). +-- +-- This is a more efficient version of @'all' 'Data.Char.isAscii'@. +-- +-- >>> isAscii "" +-- True +-- +-- >>> isAscii "abc\NUL" +-- True +-- +-- >>> isAscii "abcd€" +-- False +-- +-- prop> isAscii t == all (< '\x80') t +-- +-- @since 2.0.2 +isAscii :: Text -> Bool +isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True + +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- Note that +-- +-- > last (scanl f z xs) == foldl f z xs. +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f z t = cons z $ P.snd $ + mapAccumL (\acc c -> let c' = f acc c in (c', c')) (safe z) t +-- This is a bit suboptimal: we could have used +-- Data.Text.scanl for the first chunk and mapAccumL +-- for subsequent ones, but but I doubt anyone cares +-- about the performance of 'scanl' much. +{-# INLINE scanl #-} + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t0 = case uncons t0 of + Nothing -> empty + Just (t,ts) -> scanl f t ts +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f z t = (`snoc` z) $ P.snd $ + mapAccumR (\acc c -> let c' = f c acc in (c', c')) (safe z) t +-- See the comment for 'scanl' above. +{-# INLINE scanr #-} + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumL f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z', c') = T.mapAccumL f z c + (z'', cs') = go z' cs + go z Empty = (z, Empty) +{-# INLINE mapAccumL #-} + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. Performs replacement on invalid scalar values. +mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumR f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z'', c') = T.mapAccumR f z' c + (z', cs') = go z cs + go z Empty = (z, Empty) +{-# INLINE mapAccumR #-} + +-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every +-- element. +-- +-- @since 1.2.0.5 +repeat :: Char -> Text +repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t + in t + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int64 -> Text -> Text +replicate n + | n <= 0 = P.const Empty + | otherwise = \case + Empty -> Empty + Chunk t Empty -> replicateChunk n t + t -> concat (rep n) + where + rep 0 = [] + rep i = t : rep (i - 1) +{-# INLINE [1] replicate #-} + +replicateChunk :: Int64 -> T.Text -> Text +replicateChunk !n !t@(T.Text _ _ len) + | n <= 0 = Empty + | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) + where + perChunk = defaultChunkSize `quot` len + normalChunk = T.replicate perChunk t + (q, r) = n `P.quotRem` intToInt64 perChunk + headChunk = T.replicate (int64ToInt r) t +{-# INLINE replicateChunk #-} + +-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or +-- equivalently, the infinite repetition of the original 'Text'. +-- +-- @since 1.2.0.5 +cycle :: HasCallStack => Text -> Text +cycle Empty = emptyError "cycle" +cycle t = let t' = foldrChunks Chunk t' t + in t' + +-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications +-- of @f@ to @x@: +-- +-- > iterate f x == [x, f x, f (f x), ...] +-- +-- @since 1.2.0.5 +iterate :: (Char -> Char) -> Char -> Text +iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) + in t c + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int64 -> Text -> Text +take i _ | i <= 0 = Empty +take i t0 = take' i t0 + where + take' :: Int64 -> Text -> Text + take' 0 _ = Empty + take' _ Empty = Empty + take' n (Chunk t@(T.Text arr off _) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then fromStrict (T.Text arr off m) + else Chunk t (take' (n + intToInt64 m) ts) + + | n < l = Chunk (T.take (int64ToInt n) t) Empty + | otherwise = Chunk t (take' (n - l) ts) + where l = intToInt64 (T.length t) +{-# INLINE [1] take #-} + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > takeEnd 3 "foobar" == "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int64 -> Text -> Text +takeEnd n t0 + | n <= 0 = empty + | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 + where + takeChunk :: Int64 -> Text -> [T.Text] -> Text + takeChunk _ acc [] = acc + takeChunk i acc (t:ts) + | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc + | otherwise = takeChunk (i-l) (Chunk t acc) ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int64 -> Text -> Text +drop i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk t@(T.Text arr off len) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then chunk (T.Text arr (off + m) (len - m)) ts + else drop' (n + intToInt64 m) ts + + | n < l = Chunk (T.drop (int64ToInt n) t) ts + | otherwise = drop' (n - l) ts + where l = intToInt64 (T.length t) +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > dropEnd 3 "foobar" == "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int64 -> Text -> Text +dropEnd n t0 + | n <= 0 = t0 + | otherwise = dropChunk n . L.reverse . toChunks $ t0 + where + dropChunk :: Int64 -> [T.Text] -> Text + dropChunk _ [] = empty + dropChunk m (t:ts) + | m >= l = dropChunk (m-l) ts + | otherwise = fromChunks . L.reverse $ + T.dropEnd (int64ToInt m) t : ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' +-- values dropped, or the empty 'Text' if @n@ is greater than the +-- number of 'Word8' values present. +dropWords :: Int64 -> Text -> Text +dropWords i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk (T.Text arr off len) ts) + | n < len' = chunk (text arr (off+n') (len-n')) ts + | otherwise = drop' (n - len') ts + where len' = intToInt64 len + n' = int64ToInt n + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t0 = takeWhile' t0 + where takeWhile' Empty = Empty + takeWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n | n > 0 -> Chunk (T.take n t) Empty + | otherwise -> Empty + Nothing -> Chunk t (takeWhile' ts) +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- > takeWhileEnd (=='o') "foo" == "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p = takeChunk empty . L.reverse . toChunks + where takeChunk acc [] = acc + takeChunk acc (t:ts) + | T.lengthWord8 t' < T.lengthWord8 t + = chunk t' acc + | otherwise = takeChunk (Chunk t' acc) ts + where t' = T.takeWhileEnd p t +{-# INLINE takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t0 = dropWhile' t0 + where dropWhile' Empty = Empty + dropWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n -> Chunk (T.drop n t) ts + Nothing -> dropWhile' ts +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- > dropWhileEnd (=='.') "foo..." == "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p = go + where go Empty = Empty + go (Chunk t Empty) = if T.null t' + then Empty + else Chunk t' Empty + where t' = T.dropWhileEnd p t + go (Chunk t ts) = case go ts of + Empty -> go (Chunk t Empty) + ts' -> Chunk t ts' +{-# INLINE dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int64 -> Text -> (Text, Text) +splitAt = loop + where + loop :: Int64 -> Text -> (Text, Text) + loop !_ Empty = (empty, empty) + loop n t | n <= 0 = (empty, t) + loop n (Chunk t@(T.Text arr off len) ts) + | n > mx = let (ts', ts'') = loop (n - intToInt64 (T.length t)) ts + in (Chunk t ts', ts'') + | m > 0, m >= len = (Chunk t Empty, ts) + | m > 0 = let t' = T.Text arr off m + t'' = T.Text arr (off+m) (len-m) + in (Chunk t' Empty, Chunk t'' ts) + | otherwise = let (ts', ts'') = loop (n + intToInt64 m) ts + in (Chunk t ts', ts'') + where + mx = intToInt64 P.maxBound + m = T.measureOff (int64ToInt n) t + + +-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first +-- element is a prefix of @t@ whose chunks contain @n@ 'Word8' +-- values, and whose second is the remainder of the string. +splitAtWord :: Int64 -> Text -> PairS Text Text +splitAtWord !_ Empty = empty :*: empty +splitAtWord x (Chunk c@(T.Text arr off len) cs) + | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs + in Chunk c h :*: t + | otherwise = chunk (text arr off y) empty :*: + chunk (text arr (off+y) (len-y)) cs + where y = int64ToInt x + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> let h :*: t = splitAtWord x src + in (h, t) + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) + in (reverse b, reverse a) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- > breakOnAll "::" "" +-- > ==> [] +-- > breakOnAll "/" "a/b/c/" +-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src + | null pat = emptyError "breakOnAll" + | otherwise = go 0 empty src (indices pat src) + where + go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s + h' = append p h + in (h',t) : go x h' t xs + go _ _ _ _ = [] + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is over +-- elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p t0 = break' t0 + where break' Empty = (empty, empty) + break' c@(Chunk t ts) = + case T.findIndex p t of + Nothing -> let (ts', ts'') = break' ts + in (Chunk t ts', ts'') + Just n | n == 0 -> (Empty, c) + | otherwise -> let (a,b) = T.splitAt n t + in (Chunk a Empty, Chunk b ts) + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p = break (not . p) +{-# INLINE span #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t1, t2) <- T.spanM p t + if T.null t2 then first (chunk t) <$> go ts + else pure (chunk t1 empty, Chunk t2 ts) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t3, t4) <- go ts + if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t + else pure (Chunk t t3, t4) +{-# INLINE spanEndM #-} + +-- | The 'group' function takes a 'Text' and returns a list of 'Text's +-- such that the concatenation of the result is equal to the argument. +-- Moreover, each sublist in the result contains only equal elements. +-- For example, +-- +-- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] +-- +-- It is a special case of 'groupBy', which allows the programmer to +-- supply their own equality test. +group :: Text -> [Text] +group = groupBy (==) +{-# INLINE group #-} + +-- | The 'groupBy' function is the non-overloaded version of 'group'. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy _ Empty = [] +groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs + where (ys,zs) = span (eq x) xs + x = T.unsafeHead t + xs = chunk (T.unsafeTail t) ts + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +inits :: Text -> [Text] +inits = (NE.toList P.$!) . initsNE + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmpty Text +initsNE ts0 = Empty NE.:| inits' 0 ts0 + where + inits' :: Int64 -- Number of previous chunks i + -> Text -- The remainder after dropping i chunks from ts0 + -> [Text] -- Prefixes longer than the first i chunks of ts0. + inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t)) + ++ inits' (i + 1) ts + inits' _ Empty = [] + +takeChunks :: Int64 -> Text -> T.Text -> Text +takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk) +takeChunks _ _ lastChunk = Chunk lastChunk Empty + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NE.toList P.$!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmpty Text +tailsNE Empty = Empty :| [] +tailsNE ts@(Chunk t ts') + | T.length t == 1 = ts :| tails ts' + | otherwise = ts :| tails (Chunk (T.unsafeTail t) ts') + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be an empty string), consuming the +-- delimiter. An empty delimiter is invalid, and will cause an error +-- to be raised. +-- +-- Examples: +-- +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat src + | null pat = emptyError "splitOn" + | isSingleton pat = split (== head pat) src + | otherwise = go 0 (indices pat src) src + where + go _ [] cs = [cs] + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in h : go (x+l) xs (dropWords l t) + l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat +{-# INLINE [1] splitOn #-} + +{-# RULES +"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') [] == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ Empty = [Empty] +split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 + where comb acc (s:[]) Empty = revChunks (s:acc) : [] + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts + comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts + comb _ [] _ = impossibleError "split" +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] +-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] +chunksOf :: Int64 -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines Empty = [] +lines t = NE.toList $ go t + where + go :: Text -> NonEmpty Text + go Empty = Empty :| [] + go (Chunk x xs) + -- x is non-empty, so T.lines x is non-empty as well + | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs + | otherwise = case unsnocList (T.lines x) of + Nothing -> impossibleError "lines" + Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls + +prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text +prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! + +unsnocList :: [a] -> Maybe ([a], a) +unsnocList [] = Nothing +unsnocList (x : xs) = Just $ go x xs + where + go y [] = ([], y) + go y (z : zs) = first (y :) (go z zs) + +hasNlEnd :: T.Text -> Bool +hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words = L.filter (not . null) . split isSpace +{-# INLINE words #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf Empty _ = True +isPrefixOf _ Empty = False +isPrefixOf (Chunk x xs) (Chunk y ys) + | lx == ly = x == y && isPrefixOf xs ys + | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) + | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys + where (xh,xt) = T.splitAt ly x + (yh,yt) = T.splitAt lx y + lx = T.length x + ly = T.length y + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf x y = reverse x `isPrefixOf` reverse y +{-# INLINE isSuffixOf #-} +-- TODO: a better implementation + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (head needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripPrefix "foo" "foobar" == Just "bar" +-- > stripPrefix "" "baz" == Just "baz" +-- > stripPrefix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p t + | null p = Just t + | otherwise = case commonPrefixes p t of + Just (_,c,r) | null c -> Just r + _ -> Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") +-- > commonPrefixes "veeble" "fetzer" == Nothing +-- > commonPrefixes "" "baz" == Nothing +commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) +commonPrefixes Empty _ = Nothing +commonPrefixes _ Empty = Nothing +commonPrefixes a0 b0 = Just (go a0 b0 []) + where + go t0@(Chunk x xs) t1@(Chunk y ys) ps + = case T.commonPrefixes x y of + Just (p,a,b) + | T.null a -> go xs (chunk b ys) (p:ps) + | T.null b -> go (chunk a xs) ys (p:ps) + | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) + Nothing -> (fromChunks (L.reverse ps),t0,t1) + go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripSuffix "bar" "foobar" == Just "foo" +-- > stripSuffix "" "baz" == Just "baz" +-- > stripSuffix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = foldrChunks (chunk . filter_ T.Text p) Empty +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' +-- if there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem c t = S.any (== c) (stream t) +{-# INLINE elem #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int64 -> Char +index lazyText ix + | ix < 0 = P.error $ "Data.Text.Lazy.index: negative index " ++ P.show ix + | otherwise = go lazyText ix + where + go :: Text -> Int64 -> Char + go Empty _ = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + go (Chunk t@(T.Text _ _ lenInBytes) ts) n = case toIntegralSized n of + Nothing -> + go ts (n - fromIntegral (T.length t)) + Just n' + | off < 0 -> go ts (n + fromIntegral off) + | off == lenInBytes -> go ts 0 + | otherwise -> ch + where + off = T.measureOff n' t + T.Iter ch _ = T.iter t off +{-# INLINE index #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int64 +count pat + | null pat = emptyError "count" + | otherwise = go 0 . indices pat + where go !n [] = n + go !n (_:xs) = go (n+1) xs +{-# INLINE [1] count #-} + +{-# RULES +"LAZY TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int64 +countChar c t = S.countChar c (stream t) + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE [0] zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [0] zipWith #-} + +-- | Convert a value to lazy 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +revChunks :: [T.Text] -> Text +revChunks = L.foldl' (flip chunk) Empty + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") + +impossibleError :: HasCallStack => String -> a +impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") + +intToInt64 :: Exts.Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Exts.Int +int64ToInt = fromIntegral diff --git a/src/Data/Text/Lazy/Builder.hs b/src/Data/Text/Lazy/Builder.hs --- a/src/Data/Text/Lazy/Builder.hs +++ b/src/Data/Text/Lazy/Builder.hs @@ -39,6 +39,7 @@ module Data.Text.Lazy.Builder ( -- * The Builder type Builder + , LazyTextBuilder , toLazyText , toLazyTextWith diff --git a/src/Data/Text/Lazy/Builder/RealFloat.hs b/src/Data/Text/Lazy/Builder/RealFloat.hs --- a/src/Data/Text/Lazy/Builder/RealFloat.hs +++ b/src/Data/Text/Lazy/Builder/RealFloat.hs @@ -1,6 +1,9 @@ {-# LANGUAGE CPP, OverloadedStrings #-} {-# LANGUAGE Trustworthy #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + -- | -- Module: Data.Text.Lazy.Builder.RealFloat -- Copyright: (c) The University of Glasgow 1994-2002 diff --git a/src/Data/Text/Lazy/Encoding.hs b/src/Data/Text/Lazy/Encoding.hs --- a/src/Data/Text/Lazy/Encoding.hs +++ b/src/Data/Text/Lazy/Encoding.hs @@ -1,244 +1,254 @@ -{-# LANGUAGE BangPatterns,CPP #-} -{-# LANGUAGE Trustworthy #-} -{-# LANGUAGE ViewPatterns #-} - --- | --- Module : Data.Text.Lazy.Encoding --- Copyright : (c) 2009, 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Portability : portable --- --- Functions for converting lazy 'Text' values to and from lazy --- 'ByteString', using several standard encodings. --- --- To gain access to a much larger family of encodings, use the --- . - -module Data.Text.Lazy.Encoding - ( - -- * Decoding ByteStrings to Text - -- $strict - - -- ** Total Functions #total# - -- $total - decodeLatin1 - - -- *** Catchable failure - , decodeUtf8' - - -- *** Controllable error handling - , decodeUtf8With - , decodeUtf16LEWith - , decodeUtf16BEWith - , decodeUtf32LEWith - , decodeUtf32BEWith - - -- ** Partial Functions - -- $partial - , decodeASCII - , decodeUtf8 - , decodeUtf16LE - , decodeUtf16BE - , decodeUtf32LE - , decodeUtf32BE - - -- * Encoding Text to ByteStrings - , encodeUtf8 - , encodeUtf16LE - , encodeUtf16BE - , encodeUtf32LE - , encodeUtf32BE - - -- * Encoding Text using ByteString Builders - , encodeUtf8Builder - , encodeUtf8BuilderEscaped - ) where - -import Control.Exception (evaluate, try) -import Data.Monoid (Monoid(..)) -import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) -import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) -import Data.Word (Word8) -import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Builder.Prim as BP -import qualified Data.ByteString.Lazy as B -import qualified Data.ByteString.Lazy.Internal as B -import qualified Data.Text.Encoding as TE -import qualified Data.Text.Internal.Encoding as TE -import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E -import qualified Data.Text.Internal.Lazy.Fusion as F -import qualified Data.Text.Internal.StrictBuilder as SB -import Data.Text.Unsafe (unsafeDupablePerformIO) - --- $strict --- --- All of the single-parameter functions for decoding bytestrings --- encoded in one of the Unicode Transformation Formats (UTF) operate --- in a /strict/ mode: each will throw an exception if given invalid --- input. --- --- Each function has a variant, whose name is suffixed with -'With', --- that gives greater control over the handling of decoding errors. --- For instance, 'decodeUtf8' will throw an exception, but --- 'decodeUtf8With' allows the programmer to determine what to do on a --- decoding error. - --- $total --- --- These functions facilitate total decoding and should be preferred --- over their partial counterparts. - --- $partial --- --- These functions are partial and should only be used with great caution --- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better --- solutions. - --- | Decode a 'ByteString' containing 7-bit ASCII --- encoded text. -decodeASCII :: B.ByteString -> Text -decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks - --- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. -decodeLatin1 :: B.ByteString -> Text -decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks - --- | Decode a 'ByteString' containing UTF-8 encoded text. -decodeUtf8With :: OnDecodeError -> B.ByteString -> Text -decodeUtf8With onErr = loop TE.startUtf8State - where - chunkb builder t | SB.sbLength builder == 0 = t - | otherwise = Chunk (TE.strictBuilderToText builder) t - loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of - (builder, _, s') -> chunkb builder (loop s' bs) - loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty - msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" - --- | Decode a 'ByteString' containing UTF-8 encoded text that is known --- to be valid. --- --- If the input contains any invalid UTF-8 data, an exception will be --- thrown that cannot be caught in pure code. For more control over --- the handling of invalid data, use 'decodeUtf8'' or --- 'decodeUtf8With'. -decodeUtf8 :: B.ByteString -> Text -decodeUtf8 = decodeUtf8With strictDecode -{-# INLINE[0] decodeUtf8 #-} - --- | Decode a 'ByteString' containing UTF-8 encoded text.. --- --- If the input contains any invalid UTF-8 data, the relevant --- exception will be returned, otherwise the decoded text. --- --- /Note/: this function is /not/ lazy, as it must decode its entire --- input before it can return a result. If you need lazy (streaming) --- decoding, use 'decodeUtf8With' in lenient mode. -decodeUtf8' :: B.ByteString -> Either UnicodeException Text -decodeUtf8' bs = unsafeDupablePerformIO $ do - let t = decodeUtf8 bs - try (evaluate (rnf t `seq` t)) - where - rnf Empty = () - rnf (Chunk _ ts) = rnf ts -{-# INLINE decodeUtf8' #-} - --- | Encode text using UTF-8 encoding. -encodeUtf8 :: Text -> B.ByteString -encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty - --- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. --- --- @since 1.1.0.0 -encodeUtf8Builder :: Text -> B.Builder -encodeUtf8Builder = - foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty - --- | Encode text using UTF-8 encoding and escape the ASCII characters using --- a 'BP.BoundedPrim'. --- --- Use this function is to implement efficient encoders for text-based formats --- like JSON or HTML. --- --- @since 1.1.0.0 -{-# INLINE encodeUtf8BuilderEscaped #-} -encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder -encodeUtf8BuilderEscaped prim = - foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty - --- | Decode text from little endian UTF-16 encoding. -decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) -{-# INLINE decodeUtf16LEWith #-} - --- | Decode text from little endian UTF-16 encoding. --- --- If the input contains any invalid little endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16LEWith'. -decodeUtf16LE :: B.ByteString -> Text -decodeUtf16LE = decodeUtf16LEWith strictDecode -{-# INLINE decodeUtf16LE #-} - --- | Decode text from big endian UTF-16 encoding. -decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) -{-# INLINE decodeUtf16BEWith #-} - --- | Decode text from big endian UTF-16 encoding. --- --- If the input contains any invalid big endian UTF-16 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf16BEWith'. -decodeUtf16BE :: B.ByteString -> Text -decodeUtf16BE = decodeUtf16BEWith strictDecode -{-# INLINE decodeUtf16BE #-} - --- | Encode text using little endian UTF-16 encoding. -encodeUtf16LE :: Text -> B.ByteString -encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) -{-# INLINE encodeUtf16LE #-} - --- | Encode text using big endian UTF-16 encoding. -encodeUtf16BE :: Text -> B.ByteString -encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) -{-# INLINE encodeUtf16BE #-} - --- | Decode text from little endian UTF-32 encoding. -decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) -{-# INLINE decodeUtf32LEWith #-} - --- | Decode text from little endian UTF-32 encoding. --- --- If the input contains any invalid little endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32LEWith'. -decodeUtf32LE :: B.ByteString -> Text -decodeUtf32LE = decodeUtf32LEWith strictDecode -{-# INLINE decodeUtf32LE #-} - --- | Decode text from big endian UTF-32 encoding. -decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text -decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) -{-# INLINE decodeUtf32BEWith #-} - --- | Decode text from big endian UTF-32 encoding. --- --- If the input contains any invalid big endian UTF-32 data, an --- exception will be thrown. For more control over the handling of --- invalid data, use 'decodeUtf32BEWith'. -decodeUtf32BE :: B.ByteString -> Text -decodeUtf32BE = decodeUtf32BEWith strictDecode -{-# INLINE decodeUtf32BE #-} - --- | Encode text using little endian UTF-32 encoding. -encodeUtf32LE :: Text -> B.ByteString -encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) -{-# INLINE encodeUtf32LE #-} - --- | Encode text using big endian UTF-32 encoding. -encodeUtf32BE :: Text -> B.ByteString -encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) -{-# INLINE encodeUtf32BE #-} +{-# LANGUAGE BangPatterns,CPP #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy.Encoding +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting lazy 'Text' values to and from lazy +-- 'ByteString', using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- . + +module Data.Text.Lazy.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeUtf8Lenient + + -- *** Catchable failure + , decodeUtf8' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + ) where + +import Control.Exception (evaluate, try) +import Data.Monoid (Monoid(..)) +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) +import Data.Word (Word8) +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Lazy as B +import qualified Data.ByteString.Lazy.Internal as B +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Internal.Encoding as TE +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as F +import qualified Data.Text.Internal.StrictBuilder as SB +import Data.Text.Unsafe (unsafeDupablePerformIO) + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing 7-bit ASCII +-- encoded text. +decodeASCII :: B.ByteString -> Text +decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +decodeLatin1 :: B.ByteString -> Text +decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +decodeUtf8With :: OnDecodeError -> B.ByteString -> Text +decodeUtf8With onErr = loop TE.startUtf8State + where + chunkb builder t | SB.sbLength builder == 0 = t + | otherwise = Chunk (TE.strictBuilderToText builder) t + loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of + (builder, _, s') -> chunkb builder (loop s' bs) + loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty + msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +decodeUtf8 :: B.ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text.. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +-- +-- /Note/: this function is /not/ lazy, as it must decode its entire +-- input before it can return a result. If you need lazy (streaming) +-- decoding, use 'decodeUtf8With' in lenient mode. +decodeUtf8' :: B.ByteString -> Either UnicodeException Text +decodeUtf8' bs = unsafeDupablePerformIO $ do + let t = decodeUtf8 bs + try (evaluate (rnf t `seq` t)) + where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts +{-# INLINE decodeUtf8' #-} + +-- | Decode a lazy 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.1.4 +decodeUtf8Lenient :: B.ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> B.ByteString +encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped prim = + foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: B.ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: B.ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> B.ByteString +encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> B.ByteString +encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: B.ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: B.ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> B.ByteString +encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> B.ByteString +encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) +{-# INLINE encodeUtf32BE #-} diff --git a/src/Data/Text/Lazy/IO.hs b/src/Data/Text/Lazy/IO.hs --- a/src/Data/Text/Lazy/IO.hs +++ b/src/Data/Text/Lazy/IO.hs @@ -41,15 +41,15 @@ import Data.Text.Lazy (Text) import Prelude hiding (appendFile, getContents, getLine, interact, putStr, putStrLn, readFile, writeFile) -import System.IO (Handle, IOMode(..), hPutChar, openFile, stdin, stdout, +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, withFile) -import qualified Data.Text.IO as T import qualified Data.Text.Lazy as L import qualified Control.Exception as E import Control.Monad (when) import Data.IORef (readIORef) -import Data.Text.Internal.IO (hGetLineWith, readChunk) +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStream) import Data.Text.Internal.Lazy (chunk, empty) +import Data.Text.Internal.Lazy.Fusion (stream, streamLn) import GHC.IO.Buffer (isEmptyBuffer) import GHC.IO.Exception (IOException(..), IOErrorType(..), ioException) import GHC.IO.Handle.Internals (augmentIOError, hClose_help, @@ -129,11 +129,11 @@ -- | Write a string to a handle. hPutStr :: Handle -> Text -> IO () -hPutStr h = mapM_ (T.hPutStr h) . L.toChunks +hPutStr h = hPutStream h . stream -- | Write a string to a handle, followed by a newline. hPutStrLn :: Handle -> Text -> IO () -hPutStrLn h t = hPutStr h t >> hPutChar h '\n' +hPutStrLn h = hPutStream h . streamLn -- | The 'interact' function takes a function of type @Text -> Text@ -- as its argument. The entire input from the standard input device is diff --git a/src/Data/Text/Lazy/Internal.hs b/src/Data/Text/Lazy/Internal.hs --- a/src/Data/Text/Lazy/Internal.hs +++ b/src/Data/Text/Lazy/Internal.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} -- | -- Module : Data.Text.Lazy.Internal -- Copyright : (c) 2013 Bryan O'Sullivan diff --git a/src/Data/Text/Lazy/Read.hs b/src/Data/Text/Lazy/Read.hs --- a/src/Data/Text/Lazy/Read.hs +++ b/src/Data/Text/Lazy/Read.hs @@ -134,8 +134,16 @@ -- >rational "3e" == Right (3.0, "e") rational :: Fractional a => Reader a {-# SPECIALIZE rational :: Reader Double #-} -rational = floaty $ \real frac fracDenom -> fromRational $ - real % 1 + frac % fracDenom +rational = floaty $ \real frac fracDenom power -> + -- We must be careful to prevent DDoS attacks: if the return type is 'Double', + -- a client rightfully expects 'rational' to operate within bounded memory. + -- Thus if power is small, we can compute fraction with full precision and divide. + -- Otherwise divide first, apply fromRational and scale last: + -- the small loss of precision for Double does not matter much because the result is + -- likely infinity or zero anyway. + if abs power < 1000 + then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power)) + else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) -- | Read a rational number. -- @@ -150,9 +158,9 @@ -- around the 15th decimal place. For 0.001% of numbers, this -- function will lose precision at the 13th or 14th decimal place. double :: Reader Double -double = floaty $ \real frac fracDenom -> - fromInteger real + - fromInteger frac / fromInteger fracDenom +double = floaty $ \real frac fracDenom power -> + (fromInteger real + + fromInteger frac / fromInteger fracDenom) * (10 ^^ power) signa :: Num a => Parser a -> Parser a {-# SPECIALIZE signa :: Parser Int -> Parser Int #-} @@ -174,7 +182,7 @@ then Right (c, if len <= 1 then ts else Chunk (T.Text arr (off + 1) (len - 1)) ts) else Left "character does not match" -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a +floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') @@ -190,9 +198,7 @@ then if power == 0 then fromInteger real else fromInteger real * (10 ^^ power) - else if power == 0 - then f real fraction (10 ^ fracDigits) - else f real fraction (10 ^ fracDigits) * (10 ^^ power) + else f real fraction (10 ^ fracDigits) power return $! if sign == ord8 '+' then n else -n diff --git a/src/Data/Text/Read.hs b/src/Data/Text/Read.hs --- a/src/Data/Text/Read.hs +++ b/src/Data/Text/Read.hs @@ -140,8 +140,16 @@ -- >rational "3e" == Right (3.0, "e") rational :: Fractional a => Reader a {-# SPECIALIZE rational :: Reader Double #-} -rational = floaty $ \real frac fracDenom -> fromRational $ - real % 1 + frac % fracDenom +rational = floaty $ \real frac fracDenom power -> + -- We must be careful to prevent DDoS attacks: if the return type is 'Double', + -- a client rightfully expects 'rational' to operate within bounded memory. + -- Thus if power is small, we can compute fraction with full precision and divide. + -- Otherwise divide first, apply fromRational and scale last: + -- the small loss of precision for Double does not matter much because the result is + -- likely infinity or zero anyway. + if abs power < 1000 + then fromRational ((real % 1 + frac % fracDenom) * (10 ^^ power)) + else fromRational (real % 1 + frac % fracDenom) * (10 ^^ power) -- | Read a rational number. -- @@ -156,9 +164,9 @@ -- around the 15th decimal place. For 0.001% of numbers, this -- function will lose precision at the 13th or 14th decimal place. double :: Reader Double -double = floaty $ \real frac fracDenom -> - fromInteger real + - fromInteger frac / fromInteger fracDenom +double = floaty $ \real frac fracDenom power -> + (fromInteger real + + fromInteger frac / fromInteger fracDenom) * (10 ^^ power) signa :: Num a => Parser a -> Parser a {-# SPECIALIZE signa :: Parser Int -> Parser Int #-} @@ -177,7 +185,7 @@ then Right (c, Text arr (off + 1) (len - 1)) else Left "character does not match" -floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a +floaty :: Fractional a => (Integer -> Integer -> Integer -> Int -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') @@ -193,9 +201,7 @@ then if power == 0 then fromInteger real else fromInteger real * (10 ^^ power) - else if power == 0 - then f real fraction (10 ^ fracDigits) - else f real fraction (10 ^ fracDigits) * (10 ^^ power) + else f real fraction (10 ^ fracDigits) power return $! if sign == ord8 '+' then n else -n diff --git a/src/Data/Text/Show.hs b/src/Data/Text/Show.hs --- a/src/Data/Text/Show.hs +++ b/src/Data/Text/Show.hs @@ -1,6 +1,8 @@ -{-# LANGUAGE CPP, MagicHash #-} -{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE BangPatterns #-} {-# LANGUAGE CApiFFI #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE Trustworthy #-} {-# LANGUAGE ViewPatterns #-} {-# OPTIONS_GHC -fno-warn-orphans #-} @@ -26,12 +28,12 @@ import Control.Monad.ST (ST, runST) import Data.Text.Internal (Text(..), empty, safe, pack) import Data.Text.Internal.Encoding.Utf8 (utf8Length) -import Data.Text.Internal.Fusion (stream) import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Unsafe (Iter(..), iterArray) import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#) +import qualified GHC.Exts as Exts import GHC.Word (Word8(..)) import qualified Data.Text.Array as A -import qualified Data.Text.Internal.Fusion.Common as S #if !MIN_VERSION_ghc_prim(0,7,0) import Foreign.C.String (CString) import Foreign.C.Types (CSize(..)) @@ -52,8 +54,32 @@ HasCallStack => #endif Text -> String -unpack = S.unstreamList . stream -{-# INLINE [1] unpack #-} +unpack t = foldrText (:) [] t +{-# NOINLINE unpack #-} + +foldrText :: (Char -> b -> b) -> b -> Text -> b +foldrText f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldrText #-} + +foldrTextFB :: (Char -> b -> b) -> b -> Text -> b +foldrTextFB = foldrText +{-# INLINE [0] foldrTextFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us +-- `foldrTextFB (:) []` and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrTextFB` inlines and `foldrText` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrTextFB lcons lnil t) +"Text.unpackBack" [1] foldrTextFB (:) [] = unpack + #-} -- | /O(n)/ Convert a null-terminated -- diff --git a/src/Data/Text/Unsafe.hs b/src/Data/Text/Unsafe.hs --- a/src/Data/Text/Unsafe.hs +++ b/src/Data/Text/Unsafe.hs @@ -80,7 +80,11 @@ {-# INLINE iter #-} -- | @since 2.0 -iterArray :: A.Array -> Int -> Iter +iterArray :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -> Int -> Iter iterArray arr j = Iter chr l where m0 = A.unsafeIndex arr j m1 = A.unsafeIndex arr (j+1) diff --git a/tests/Tests/Lift.hs b/tests/Tests/Lift.hs --- a/tests/Tests/Lift.hs +++ b/tests/Tests/Lift.hs @@ -1,34 +1,39 @@ -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TemplateHaskell #-} -module Tests.Lift - ( tests - ) - where - -import qualified Data.Text as S -import qualified Data.Text.Lazy as L -import Language.Haskell.TH.Syntax (lift) -import Test.Tasty.HUnit (testCase, assertEqual) -import Test.Tasty (TestTree, testGroup) - -tests :: TestTree -tests = testGroup "TH lifting Text" - [ testCase "strict" $ assertEqual "strict" - $(lift ("foo" :: S.Text)) - ("foo" :: S.Text) - , testCase "strict0" $ assertEqual "strict0" - $(lift ("f\0o\1o\2" :: S.Text)) - ("f\0o\1o\2" :: S.Text) - , testCase "strict-nihao" $ assertEqual "strict-nihao" - $(lift ("\20320\22909" :: S.Text)) - ("\20320\22909" :: S.Text) - , testCase "lazy" $ assertEqual "lazy" - $(lift ("foo" :: L.Text)) - ("foo" :: L.Text) - , testCase "lazy0" $ assertEqual "lazy0" - $(lift ("f\0o\1o\2" :: L.Text)) - ("f\0o\1o\2" :: L.Text) - , testCase "lazy-nihao" $ assertEqual "lazy-nihao" - $(lift ("\20320\22909" :: L.Text)) - ("\20320\22909" :: L.Text) - ] +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +module Tests.Lift + ( tests + ) + where + +import qualified Data.Text as S +import qualified Data.Text.Lazy as L +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) + +tests :: TestTree +tests = testGroup "TH lifting Text" + [ testCase "strict" $ assertEqual "strict" + $(lift ("foo" :: S.Text)) + ("foo" :: S.Text) + , testCase "strict0" $ assertEqual "strict0" + $(lift ("f\0o\1o\2" :: S.Text)) + ("f\0o\1o\2" :: S.Text) + , testCase "strict-nihao" $ assertEqual "strict-nihao" + $(lift ("\20320\22909" :: S.Text)) + ("\20320\22909" :: S.Text) + , testCase "lazy" $ assertEqual "lazy" + $(lift ("foo" :: L.Text)) + ("foo" :: L.Text) + , testCase "lazy0" $ assertEqual "lazy0" + $(lift ("f\0o\1o\2" :: L.Text)) + ("f\0o\1o\2" :: L.Text) + , testCase "lazy-nihao" $ assertEqual "lazy-nihao" + $(lift ("\20320\22909" :: L.Text)) + ("\20320\22909" :: L.Text) + ] diff --git a/tests/Tests/Properties.hs b/tests/Tests/Properties.hs --- a/tests/Tests/Properties.hs +++ b/tests/Tests/Properties.hs @@ -17,6 +17,7 @@ import Tests.Properties.Text (testText) import Tests.Properties.Transcoding (testTranscoding) import Tests.Properties.Validate (testValidate) +import Tests.Properties.CornerCases (testCornerCases) tests :: TestTree tests = @@ -30,5 +31,6 @@ testBuilder, testLowLevel, testRead, + testCornerCases, testValidate ] diff --git a/tests/Tests/Properties/Basics.hs b/tests/Tests/Properties/Basics.hs --- a/tests/Tests/Properties/Basics.hs +++ b/tests/Tests/Properties/Basics.hs @@ -4,6 +4,8 @@ {-# OPTIONS_GHC -Wno-missing-signatures #-} {-# OPTIONS_GHC -Wno-warnings-deprecations #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} module Tests.Properties.Basics ( testBasics diff --git a/tests/Tests/Properties/CornerCases.hs b/tests/Tests/Properties/CornerCases.hs new file mode 100644 --- /dev/null +++ b/tests/Tests/Properties/CornerCases.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +-- | Check that the definitions that are partial crash in the expected ways or +-- return sensible defaults. +module Tests.Properties.CornerCases (testCornerCases) where + +import Control.Exception +import Data.Either +import Data.Semigroup +import Data.Text +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils () + +testCornerCases :: TestTree +testCornerCases = + testGroup + "corner cases" + [ testGroup + "stimes" + $ let specimen = stimes :: Integer -> Text -> Text + in [ testProperty + "given a negative number, evaluate to error call" + $ \(Negative number) text -> + (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $ + specimen + (fromIntegral (number :: Int)) + text + , testProperty + "given a number that does not fit into Int, evaluate to error call" + $ \(NonNegative number) text -> + (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $ + specimen + (fromIntegral (number :: Int) + fromIntegral (maxBound :: Int) + 1) + text + ] + ] diff --git a/tests/Tests/Properties/Folds.hs b/tests/Tests/Properties/Folds.hs --- a/tests/Tests/Properties/Folds.hs +++ b/tests/Tests/Properties/Folds.hs @@ -1,297 +1,357 @@ --- | Test folds, scans, and unfolds - -{-# LANGUAGE ViewPatterns #-} - -{-# OPTIONS_GHC -fno-warn-missing-signatures #-} -module Tests.Properties.Folds - ( testFolds - ) where - -import Control.Arrow (second) -import Control.Exception (ErrorCall, evaluate, try) -import Data.Word (Word8, Word16) -import Test.Tasty (TestTree, testGroup) -import Test.Tasty.HUnit (testCase, assertFailure, assertBool) -import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) -import Tests.QuickCheckUtils -import qualified Data.List as L -import qualified Data.Text as T -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import qualified Data.Text.Lazy as TL -import qualified Data.Char as Char - --- Folds - -sf_foldl (applyFun -> p) (applyFun2 -> f) z = - (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) - where _types = f :: Char -> Char -> Char -tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) - where _types = f :: Char -> Char -> Char -sf_foldl' (applyFun -> p) (applyFun2 -> f) z = - (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z - where _types = f :: Char -> Char -> Char -tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z - where _types = f :: Char -> Char -> Char -sf_foldl1 (applyFun -> p) (applyFun2 -> f) = - (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) -t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f -tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f -sf_foldl1' (applyFun -> p) (applyFun2 -> f) = - (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) -t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f -tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f -sf_foldr (applyFun -> p) (applyFun2 -> f) z = - (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) - where _types = f :: Char -> Char -> Char -t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z - where _types = f :: Char -> Char -> Char -t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z - where _types = f :: Char -> Char -> Char -tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z - where _types = f :: Char -> Char -> Char -sf_foldr1 (applyFun -> p) (applyFun2 -> f) = - (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) -t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f -tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f - --- Distinguish foldl/foldr from foldl'/foldr' - -fold_apart :: IO () -fold_apart = do - ok (T.foldr f () (T.pack "az")) - ko (T.foldr' f () (T.pack "az")) - ok (T.foldl (flip f) () (T.pack "za")) - ko (T.foldl' (flip f) () (T.pack "za")) - where - f c _ = if c == 'z' then error "catchme" else () - ok = evaluate - ko t = do - x <- try (evaluate t) - case x :: Either ErrorCall () of - Left _ -> pure () - Right _ -> assertFailure "test should have failed but didn't" - --- Special folds - -s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) -sf_concat (applyFun -> p) - = (L.concat . map (L.filter p) . unSqrt) `eq` - (unpackS . S.concat . map (S.filter p . packS) . unSqrt) -t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) -tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) -sf_concatMap (applyFun -> p) (applyFun -> f) = - (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) -t_concatMap (applyFun -> f) - = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) -tl_concatMap (applyFun -> f) - = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) -sf_any (applyFun -> q) (applyFun -> p) - = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) -t_any (applyFun -> p) - = L.any p `eqP` T.any p -tl_any (applyFun -> p) - = L.any p `eqP` TL.any p -sf_all (applyFun -> q) (applyFun -> p) - = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) -t_all (applyFun -> p) - = L.all p `eqP` T.all p -tl_all (applyFun -> p) - = L.all p `eqP` TL.all p -sf_maximum (applyFun -> p) - = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) -t_maximum = L.maximum `eqP` T.maximum -tl_maximum = L.maximum `eqP` TL.maximum -sf_minimum (applyFun -> p) - = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) -t_minimum = L.minimum `eqP` T.minimum -tl_minimum = L.minimum `eqP` TL.minimum -t_isAscii = L.all Char.isAscii `eqP` T.isAscii -tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii - --- Scans - -sf_scanl (applyFun -> p) (applyFun2 -> f) z = - (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) -t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) -tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) -t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) -tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) -t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) -tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) -t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) -tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) - -t_mapAccumL_char c t = - snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) -t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) - where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumL_char c t = - snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) -tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) - where _types = f :: Int -> Char -> (Int,Char) -t_mapAccumR_char c t = - snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) -t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) - where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumR_char c t = - snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) -tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) - where _types = f :: Int -> Char -> (Int,Char) - --- Unfolds - -tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) - -s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) - -t_replicate_char (Small n) c = - L.replicate n c === T.unpack (T.replicate n (T.singleton c)) -tl_replicate_char (Small n) c = - L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) -t_length_replicate_char (Small n) c = - L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) -tl_length_replicate_char (Small n) c = - L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) - -t_replicate (Small n) = - (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) -tl_replicate (Small n) = - (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) -t_length_replicate (Small n) = - (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) -tl_length_replicate (Small n) = - (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) - -tl_cycle n = (L.take m . L.cycle) `eq` - (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) - where m = fromIntegral (n :: Word8) - -tl_iterate (applyFun -> f) n - = (L.take m . L.iterate f) `eq` - (unpackS . TL.take (fromIntegral m) . TL.iterate f) - where m = fromIntegral (n :: Word8) - -unf :: Int -> Char -> Maybe (Char, Char) -unf n c | fromEnum c * 100 > n = Nothing - | otherwise = Just (c, succ c) - -t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) - where m = fromIntegral (n :: Word16) -tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) - where m = fromIntegral (n :: Word16) -t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` - (unpackS . T.unfoldrN i (unf j)) - where i = fromIntegral (n :: Word16) - j = fromIntegral (m :: Word16) -tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` - (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) - where i = fromIntegral (n :: Word16) - j = fromIntegral (m :: Word16) - -isAscii_border :: IO () -isAscii_border = do - let text = T.drop 2 $ T.pack "XX1234五" - assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text - -testFolds :: TestTree -testFolds = - testGroup "folds-unfolds" [ - testGroup "folds" [ - testProperty "sf_foldl" sf_foldl, - testProperty "t_foldl" t_foldl, - testProperty "tl_foldl" tl_foldl, - testProperty "sf_foldl'" sf_foldl', - testProperty "t_foldl'" t_foldl', - testProperty "tl_foldl'" tl_foldl', - testProperty "sf_foldl1" sf_foldl1, - testProperty "t_foldl1" t_foldl1, - testProperty "tl_foldl1" tl_foldl1, - testProperty "t_foldl1'" t_foldl1', - testProperty "sf_foldl1'" sf_foldl1', - testProperty "tl_foldl1'" tl_foldl1', - testProperty "sf_foldr" sf_foldr, - testProperty "t_foldr" t_foldr, - testProperty "t_foldr'" t_foldr', - testProperty "tl_foldr" tl_foldr, - testProperty "sf_foldr1" sf_foldr1, - testProperty "t_foldr1" t_foldr1, - testProperty "tl_foldr1" tl_foldr1, - testCase "fold_apart" fold_apart, - - testGroup "special" [ - testProperty "s_concat_s" s_concat_s, - testProperty "sf_concat" sf_concat, - testProperty "t_concat" t_concat, - testProperty "tl_concat" tl_concat, - testProperty "sf_concatMap" sf_concatMap, - testProperty "t_concatMap" t_concatMap, - testProperty "tl_concatMap" tl_concatMap, - testProperty "sf_any" sf_any, - testProperty "t_any" t_any, - testProperty "tl_any" tl_any, - testProperty "sf_all" sf_all, - testProperty "t_all" t_all, - testProperty "tl_all" tl_all, - testProperty "sf_maximum" sf_maximum, - testProperty "t_maximum" t_maximum, - testProperty "tl_maximum" tl_maximum, - testProperty "sf_minimum" sf_minimum, - testProperty "t_minimum" t_minimum, - testProperty "tl_minimum" tl_minimum, - testProperty "t_isAscii " t_isAscii, - testProperty "tl_isAscii " tl_isAscii, - testCase "isAscii_border" isAscii_border - ] - ], - - testGroup "scans" [ - testProperty "sf_scanl" sf_scanl, - testProperty "t_scanl" t_scanl, - testProperty "tl_scanl" tl_scanl, - testProperty "t_scanl1" t_scanl1, - testProperty "tl_scanl1" tl_scanl1, - testProperty "t_scanr" t_scanr, - testProperty "tl_scanr" tl_scanr, - testProperty "t_scanr1" t_scanr1, - testProperty "tl_scanr1" tl_scanr1 - ], - - testGroup "mapAccum" [ - testProperty "t_mapAccumL_char" t_mapAccumL_char, - testProperty "t_mapAccumL" t_mapAccumL, - testProperty "tl_mapAccumL_char" tl_mapAccumL_char, - testProperty "tl_mapAccumL" tl_mapAccumL, - testProperty "t_mapAccumR_char" t_mapAccumR_char, - testProperty "t_mapAccumR" t_mapAccumR, - testProperty "tl_mapAccumR_char" tl_mapAccumR_char, - testProperty "tl_mapAccumR" tl_mapAccumR - ], - - testGroup "unfolds" [ - testProperty "tl_cycle" tl_cycle, - testProperty "tl_iterate" tl_iterate, - testProperty "t_unfoldr" t_unfoldr, - testProperty "tl_unfoldr" tl_unfoldr, - testProperty "t_unfoldrN" t_unfoldrN, - testProperty "tl_unfoldrN" tl_unfoldrN - ], - - testGroup "replicate" [ - testProperty "tl_repeat" tl_repeat, - testProperty "s_replicate" s_replicate, - testProperty "t_replicate_char" t_replicate_char, - testProperty "tl_replicate_char" tl_replicate_char, - testProperty "t_length_replicate_char" t_length_replicate_char, - testProperty "tl_length_replicate_char" tl_length_replicate_char, - testProperty "t_replicate" t_replicate, - testProperty "tl_replicate" tl_replicate, - testProperty "t_length_replicate" t_length_replicate, - testProperty "tl_length_replicate" tl_length_replicate - ] - - ] +-- | Test folds, scans, and unfolds + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.Folds + ( testFolds + ) where + +import Control.Arrow (second) +import Control.Exception (ErrorCall, evaluate, try) +import Data.Functor.Identity (Identity(..)) +import Control.Monad.Trans.State (runState, state) +import Data.Word (Word8, Word16) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertFailure, assertBool) +import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Lazy as TL +import qualified Data.Char as Char + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectTest, (==~)) +import GHC.Exts (inline) +#endif + +-- Folds + +sf_foldl (applyFun -> p) (applyFun2 -> f) z = + (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) + where _types = f :: Char -> Char -> Char +tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) + where _types = f :: Char -> Char -> Char +sf_foldl' (applyFun -> p) (applyFun2 -> f) z = + (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z + where _types = f :: Char -> Char -> Char +tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z + where _types = f :: Char -> Char -> Char +sf_foldl1 (applyFun -> p) (applyFun2 -> f) = + (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' (applyFun -> p) (applyFun2 -> f) = + (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr (applyFun -> p) (applyFun2 -> f) z = + (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z + where _types = f :: Char -> Char -> Char +t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z + where _types = f :: Char -> Char -> Char +tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z + where _types = f :: Char -> Char -> Char +sf_foldr1 (applyFun -> p) (applyFun2 -> f) = + (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) +t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f + +-- Distinguish foldl/foldr from foldl'/foldr' + +fold_apart :: IO () +fold_apart = do + ok (T.foldr f () (T.pack "az")) + ko (T.foldr' f () (T.pack "az")) + ok (T.foldl (flip f) () (T.pack "za")) + ko (T.foldl' (flip f) () (T.pack "za")) + where + f c _ = if c == 'z' then error "catchme" else () + ok = evaluate + ko t = do + x <- try (evaluate t) + case x :: Either ErrorCall () of + Left _ -> pure () + Right _ -> assertFailure "test should have failed but didn't" + +-- Special folds + +s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) +sf_concat (applyFun -> p) + = (L.concat . map (L.filter p) . unSqrt) `eq` + (unpackS . S.concat . map (S.filter p . packS) . unSqrt) +t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) +tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) +sf_concatMap (applyFun -> p) (applyFun -> f) = + (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) +t_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) +tl_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) +sf_any (applyFun -> q) (applyFun -> p) + = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any (applyFun -> p) + = L.any p `eqP` T.any p +tl_any (applyFun -> p) + = L.any p `eqP` TL.any p +sf_all (applyFun -> q) (applyFun -> p) + = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all (applyFun -> p) + = L.all p `eqP` T.all p +tl_all (applyFun -> p) + = L.all p `eqP` TL.all p +sf_maximum (applyFun -> p) + = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +t_maximum = L.maximum `eqP` T.maximum +tl_maximum = L.maximum `eqP` TL.maximum +sf_minimum (applyFun -> p) + = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +t_minimum = L.minimum `eqP` T.minimum +tl_minimum = L.minimum `eqP` TL.minimum +t_isAscii = L.all Char.isAscii `eqP` T.isAscii +tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii + +-- Scans + +sf_scanl (applyFun -> p) (applyFun2 -> f) z = + (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) + +t_scanl_is_safe = let c = '\55296' in + T.scanl undefined c mempty === T.singleton c +tl_scanl_is_safe = let c = '\55296' in + TL.scanl undefined c mempty === TL.singleton c +t_scanr_is_safe = let c = '\55296' in + T.scanr undefined c mempty === T.singleton c +tl_scanr_is_safe = let c = '\55296' in + TL.scanr undefined c mempty === TL.singleton c + +t_mapAccumL_char c t = + snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumL_char c t = + snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +t_mapAccumR_char c t = + snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumR_char c t = + snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) + +-- Unfolds + +tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) + +s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) + +t_replicate_char (Small n) c = + L.replicate n c === T.unpack (T.replicate n (T.singleton c)) +tl_replicate_char (Small n) c = + L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) +t_length_replicate_char (Small n) c = + L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) +tl_length_replicate_char (Small n) c = + L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) + +t_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) +tl_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) +t_length_replicate (Small n) = + (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) +tl_length_replicate (Small n) = + (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) + +tl_cycle n = (L.take m . L.cycle) `eq` + (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) + where m = fromIntegral (n :: Word8) + +tl_iterate (applyFun -> f) n + = (L.take m . L.iterate f) `eq` + (unpackS . TL.take (fromIntegral m) . TL.iterate f) + where m = fromIntegral (n :: Word8) + +unf :: Int -> Char -> Maybe (Char, Char) +unf n c | fromEnum c * 100 > n = Nothing + | otherwise = Just (c, succ c) + +t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . T.unfoldrN i (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) +tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) + +-- Monadic folds + +-- Parametric polymorphism allows us to only test foldlM' specialized to +-- one function in the state monad (called @logger@ in the following tests) +-- that just logs the arguments it was applied to and produces a fresh +-- accumulator. That alone determines the general behavior of foldlM' with an +-- arbitrary function in any monad. +-- Reference: "Testing Polymorphic Properties" by Bernardy et al. +-- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf + +t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order +tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order + +#ifdef MIN_VERSION_tasty_inspection_testing +-- As a sanity check for performance, the simplified Core +-- foldlM' specialized to Identity is the same as foldl'. + +_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x + +_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl' = inline S.foldl' +#endif + +isAscii_border :: IO () +isAscii_border = do + let text = T.drop 2 $ T.pack "XX1234五" + assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text + +testFolds :: TestTree +testFolds = + testGroup "folds-unfolds" [ + testGroup "folds" [ + testProperty "sf_foldl" sf_foldl, + testProperty "t_foldl" t_foldl, + testProperty "tl_foldl" tl_foldl, + testProperty "sf_foldl'" sf_foldl', + testProperty "t_foldl'" t_foldl', + testProperty "tl_foldl'" tl_foldl', + testProperty "sf_foldl1" sf_foldl1, + testProperty "t_foldl1" t_foldl1, + testProperty "tl_foldl1" tl_foldl1, + testProperty "t_foldl1'" t_foldl1', + testProperty "sf_foldl1'" sf_foldl1', + testProperty "tl_foldl1'" tl_foldl1', + testProperty "sf_foldr" sf_foldr, + testProperty "t_foldr" t_foldr, + testProperty "t_foldr'" t_foldr', + testProperty "tl_foldr" tl_foldr, + testProperty "sf_foldr1" sf_foldr1, + testProperty "t_foldr1" t_foldr1, + testProperty "tl_foldr1" tl_foldr1, + testProperty "t_foldlM'" t_foldlM', + testProperty "tl_foldlM'" tl_foldlM', +#ifdef MIN_VERSION_tasty_inspection_testing + let _unused = ['_S_foldl'_from_foldlM', '_S_foldl'] in + $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')), +#endif + testCase "fold_apart" fold_apart, + + testGroup "special" [ + testProperty "s_concat_s" s_concat_s, + testProperty "sf_concat" sf_concat, + testProperty "t_concat" t_concat, + testProperty "tl_concat" tl_concat, + testProperty "sf_concatMap" sf_concatMap, + testProperty "t_concatMap" t_concatMap, + testProperty "tl_concatMap" tl_concatMap, + testProperty "sf_any" sf_any, + testProperty "t_any" t_any, + testProperty "tl_any" tl_any, + testProperty "sf_all" sf_all, + testProperty "t_all" t_all, + testProperty "tl_all" tl_all, + testProperty "sf_maximum" sf_maximum, + testProperty "t_maximum" t_maximum, + testProperty "tl_maximum" tl_maximum, + testProperty "sf_minimum" sf_minimum, + testProperty "t_minimum" t_minimum, + testProperty "tl_minimum" tl_minimum, + testProperty "t_isAscii " t_isAscii, + testProperty "tl_isAscii " tl_isAscii, + testCase "isAscii_border" isAscii_border + ] + ], + + testGroup "scans" [ + testProperty "sf_scanl" sf_scanl, + testProperty "t_scanl" t_scanl, + testProperty "tl_scanl" tl_scanl, + testProperty "t_scanl1" t_scanl1, + testProperty "tl_scanl1" tl_scanl1, + testProperty "t_scanr" t_scanr, + testProperty "tl_scanr" tl_scanr, + testProperty "t_scanr1" t_scanr1, + testProperty "tl_scanr1" tl_scanr1, + + testProperty "t_scanl_is_safe" t_scanl_is_safe, + testProperty "tl_scanl_is_safe" tl_scanl_is_safe, + testProperty "t_scanr_is_safe" t_scanr_is_safe, + testProperty "tl_scanr_is_safe" tl_scanr_is_safe + ], + + testGroup "mapAccum" [ + testProperty "t_mapAccumL_char" t_mapAccumL_char, + testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL_char" tl_mapAccumL_char, + testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR_char" t_mapAccumR_char, + testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR_char" tl_mapAccumR_char, + testProperty "tl_mapAccumR" tl_mapAccumR + ], + + testGroup "unfolds" [ + testProperty "tl_cycle" tl_cycle, + testProperty "tl_iterate" tl_iterate, + testProperty "t_unfoldr" t_unfoldr, + testProperty "tl_unfoldr" tl_unfoldr, + testProperty "t_unfoldrN" t_unfoldrN, + testProperty "tl_unfoldrN" tl_unfoldrN + ], + + testGroup "replicate" [ + testProperty "tl_repeat" tl_repeat, + testProperty "s_replicate" s_replicate, + testProperty "t_replicate_char" t_replicate_char, + testProperty "tl_replicate_char" tl_replicate_char, + testProperty "t_length_replicate_char" t_length_replicate_char, + testProperty "tl_length_replicate_char" tl_length_replicate_char, + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_length_replicate" t_length_replicate, + testProperty "tl_length_replicate" tl_length_replicate + ] + + ] diff --git a/tests/Tests/Properties/Instances.hs b/tests/Tests/Properties/Instances.hs --- a/tests/Tests/Properties/Instances.hs +++ b/tests/Tests/Properties/Instances.hs @@ -6,6 +6,8 @@ ( testInstances ) where +import Data.Binary (encode, decodeOrFail) +import Data.Semigroup import Data.String (IsString(fromString)) import Test.QuickCheck import Test.Tasty (TestTree, testGroup) @@ -36,6 +38,12 @@ tl_Show = show `eq` (show . TL.pack) t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s)) +t_stimes = \ number -> eq + ((stimes :: Int -> String -> String) number . unSqrt) + (unpackS . (stimes :: Int -> T.Text -> T.Text) number . T.pack . unSqrt) +tl_stimes = \ number -> eq + ((stimes :: Int -> String -> String) number . unSqrt) + (unpackS . (stimes :: Int -> TL.Text -> TL.Text) number . TL.pack . unSqrt) t_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt) tl_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt) t_mempty = mempty === (unpackS (mempty :: T.Text)) @@ -43,6 +51,16 @@ t_IsString = fromString `eqP` (T.unpack . fromString) tl_IsString = fromString `eqP` (TL.unpack . fromString) +t_Binary s = + case decodeOrFail . encode $ (s :: T.Text) of + Left _ -> counterexample (show (T.unpack s)) (property False) + Right (_, _, s') -> s === s' + +tl_Binary s = + case decodeOrFail . encode $ (s :: TL.Text) of + Left _ -> counterexample (show (TL.unpack s)) (property False) + Right (_, _, s') -> s === s' + testInstances :: TestTree testInstances = testGroup "instances" [ @@ -60,10 +78,14 @@ testProperty "tl_Show" tl_Show, testProperty "t_mappend" t_mappend, testProperty "tl_mappend" tl_mappend, + testProperty "t_stimes" t_stimes, + testProperty "tl_stimes" tl_stimes, testProperty "t_mconcat" t_mconcat, testProperty "tl_mconcat" tl_mconcat, testProperty "t_mempty" t_mempty, testProperty "tl_mempty" tl_mempty, testProperty "t_IsString" t_IsString, - testProperty "tl_IsString" tl_IsString + testProperty "tl_IsString" tl_IsString, + testProperty "t_Binary" t_Binary, + testProperty "tl_Binary" tl_Binary ] diff --git a/tests/Tests/Properties/LowLevel.hs b/tests/Tests/Properties/LowLevel.hs --- a/tests/Tests/Properties/LowLevel.hs +++ b/tests/Tests/Properties/LowLevel.hs @@ -1,157 +1,176 @@ --- | Test low-level operations - -{-# LANGUAGE CPP #-} -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE ScopedTypeVariables #-} - -{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} - -#ifdef MIN_VERSION_tasty_inspection_testing -{-# LANGUAGE TemplateHaskell #-} -{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} -#endif - -module Tests.Properties.LowLevel (testLowLevel) where - -import Prelude hiding (head, tail) -import Control.Applicative ((<$>), pure) -import Control.Exception as E (SomeException, catch, evaluate) -import Data.Int (Int32, Int64) -import Data.Text.Foreign -import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) -import Data.Word (Word8, Word16, Word32) -import System.IO.Unsafe (unsafePerformIO) -import Test.Tasty (TestTree, testGroup) -import Test.Tasty.HUnit (testCase, assertEqual) -import Test.Tasty.QuickCheck (testProperty) -import Test.QuickCheck hiding ((.&.)) -import Tests.QuickCheckUtils -import Tests.Utils -import qualified Data.Text as T -import qualified Data.Text.IO as T -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.IO as TL -import qualified Data.Text.IO.Utf8 as TU -import qualified System.IO as IO - -#ifdef MIN_VERSION_tasty_inspection_testing -import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import qualified GHC.CString as GHC -#endif - -mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a -mulRef a b - | ab < bot || ab > top = Nothing - | otherwise = Just (fromIntegral ab) - where ab = fromIntegral a * fromIntegral b - top = fromIntegral (maxBound `asTypeOf` a) :: Integer - bot = fromIntegral (minBound `asTypeOf` a) :: Integer - -eval :: (a -> b -> c) -> a -> b -> Maybe c -eval f a b = unsafePerformIO $ - (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) - -t_mul32 :: Int32 -> Int32 -> Property -t_mul32 a b = mulRef a b === eval mul32 a b - -t_mul64 :: Int64 -> Int64 -> Property -t_mul64 a b = mulRef a b === eval mul64 a b - -t_mul :: Int -> Int -> Property -t_mul a b = mulRef a b === eval mul a b - --- Misc. - -t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t -t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t -t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t -t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr -t_use_from0 t = ioProperty $ do - let t' = t `T.snoc` '\0' - (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0) - -t_copy t = T.copy t === t - -t_literal_length1 = assertEqual xs (length xs) byteLen - where - xs = "\0\1\0\1\0" - Text _ _ byteLen = T.pack xs -t_literal_length2 = assertEqual xs (length xs) byteLen - where - xs = "\1\2\3\4\5" - Text _ _ byteLen = T.pack xs -t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) - where - ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" - xs = map safe ys - -#ifdef MIN_VERSION_tasty_inspection_testing -t_literal_foo :: Text -t_literal_foo = T.pack "foo" -#endif - --- Input and output. - --- t_put_get = write_read T.unlines T.filter put get --- where put h = withRedirect h IO.stdout . T.putStr --- get h = withRedirect h IO.stdin T.getContents --- tl_put_get = write_read TL.unlines TL.filter put get --- where put h = withRedirect h IO.stdout . TL.putStr --- get h = withRedirect h IO.stdin TL.getContents -t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents -tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents - -t_write_read_line m b t = write_read (T.concat . take 1) T.filter T.hPutStrLn - T.hGetLine m b [t] -tl_write_read_line m b t = write_read (TL.concat . take 1) TL.filter TL.hPutStrLn - TL.hGetLine m b [t] - -utf8_write_read = write_read T.unlines T.filter TU.hPutStr TU.hGetContents -utf8_write_read_line m b t = write_read (T.concat . take 1) T.filter TU.hPutStrLn - TU.hGetLine m b [t] - -testLowLevel :: TestTree -testLowLevel = - testGroup "lowlevel" [ - testGroup "mul" [ - testProperty "t_mul" t_mul, - testProperty "t_mul32" t_mul32, - testProperty "t_mul64" t_mul64 - ], - - testGroup "misc" [ - testProperty "t_dropWord8" t_dropWord8, - testProperty "t_takeWord8" t_takeWord8, - testProperty "t_take_drop_8" t_take_drop_8, - testProperty "t_use_from" t_use_from, - testProperty "t_use_from0" t_use_from0, - testProperty "t_copy" t_copy, - testCase "t_literal_length1" t_literal_length1, - testCase "t_literal_length2" t_literal_length2, - testCase "t_literal_surrogates" t_literal_surrogates -#ifdef MIN_VERSION_tasty_inspection_testing - , $(inspectObligations - [ (`hasNoTypes` [''Char, ''[]]) - , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) - , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) - , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) - ] - 't_literal_foo) -#endif - ], - - testGroup "input-output" [ - testProperty "t_write_read" t_write_read, - testProperty "tl_write_read" tl_write_read, - testProperty "t_write_read_line" t_write_read_line, - testProperty "tl_write_read_line" tl_write_read_line, - testProperty "utf8_write_read" utf8_write_read, - testProperty "utf8_write_read_line" utf8_write_read_line - -- These tests are subject to I/O race conditions - -- testProperty "t_put_get" t_put_get, - -- testProperty "tl_put_get" tl_put_get - ] - ] - +-- | Test low-level operations + +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.LowLevel (testLowLevel) where + +import Prelude hiding (head, tail) +import Control.Applicative ((<$>), pure) +import Control.Exception as E (SomeException, catch, evaluate) +import Data.Functor.Identity (Identity(..)) +import Data.Int (Int32, Int64) +import Data.Text.Foreign +import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) +import Data.Word (Word8, Word16, Word32) +import System.IO.Unsafe (unsafePerformIO) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty.QuickCheck (testProperty) +import Test.QuickCheck hiding ((.&.)) +import Tests.QuickCheckUtils +import Tests.Utils +import qualified Data.Text as T +import qualified Data.Text.Foreign as T +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.IO.Utf8 as TU +import qualified System.IO as IO + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified GHC.CString as GHC +#endif + +mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a +mulRef a b + | ab < bot || ab > top = Nothing + | otherwise = Just (fromIntegral ab) + where ab = fromIntegral a * fromIntegral b + top = fromIntegral (maxBound `asTypeOf` a) :: Integer + bot = fromIntegral (minBound `asTypeOf` a) :: Integer + +eval :: (a -> b -> c) -> a -> b -> Maybe c +eval f a b = unsafePerformIO $ + (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) + +t_mul32 :: Int32 -> Int32 -> Property +t_mul32 a b = mulRef a b === eval mul32 a b + +t_mul64 :: Int64 -> Int64 -> Property +t_mul64 a b = mulRef a b === eval mul64 a b + +t_mul :: Int -> Int -> Property +t_mul a b = mulRef a b === eval mul a b + +-- Misc. + +t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t +t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t +t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t +t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr +t_use_from0 t = ioProperty $ do + let t' = t `T.snoc` '\0' + (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0) + +t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do + roundTrip <- T.withCString t T.peekCString + assertEqual "cstring" t roundTrip + +t_peek_cstring_len t = ioProperty $ do + roundTrip <- T.withCStringLen t T.peekCStringLen + assertEqual "cstring_len" t roundTrip + +t_copy t = T.copy t === t + +t_literal_length1 = assertEqual xs (length xs) byteLen + where + xs = "\0\1\0\1\0" + Text _ _ byteLen = T.pack xs +t_literal_length2 = assertEqual xs (length xs) byteLen + where + xs = "\1\2\3\4\5" + Text _ _ byteLen = T.pack xs +t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) + where + ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" + xs = map safe ys + +#ifdef MIN_VERSION_tasty_inspection_testing +t_literal_foo :: Text +t_literal_foo = T.pack "foo" +#endif + +-- Input and output. + +-- t_put_get = write_read T.unlines T.filter put get +-- where put h = withRedirect h IO.stdout . T.putStr +-- get h = withRedirect h IO.stdin T.getContents +-- tl_put_get = write_read TL.unlines TL.filter put get +-- where put h = withRedirect h IO.stdout . TL.putStr +-- get h = withRedirect h IO.stdin TL.getContents + +inputOutput :: TestTree +inputOutput = testGroup "input-output" [ + testProperty "t_write_read" $ write_read arbitrary shrink (T.replace "\n" "\r\n") T.hPutStr T.hGetContents, + testProperty "tl_write_read" $ write_read arbitrary shrink (TL.replace "\n" "\r\n") TL.hPutStr TL.hGetContents, + testProperty "t_write_read_line" $ write_read genTLine shrinkTLine (`T.append` "\r") T.hPutStrLn T.hGetLine, + testProperty "tl_write_read_line" $ write_read genTLLine shrinkTLLine (`TL.append` "\r") TL.hPutStrLn TL.hGetLine, + -- Note: Data.Text.IO.Utf8 does NO newline translation + testProperty "utf8_write_read" $ write_read arbitrary shrink id TU.hPutStr TU.hGetContents, + testProperty "utf8_write_read_line" $ write_read genTLine shrinkTLine id TU.hPutStrLn TU.hGetLine + -- These tests are subject to I/O race conditions + -- testProperty "t_put_get" t_put_get, + -- testProperty "tl_put_get" tl_put_get + ] + +genTLine :: Gen T.Text +genTLine = T.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +genTLLine :: Gen TL.Text +genTLLine = TL.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +shrinkTLine :: T.Text -> [T.Text] +shrinkTLine = filter (T.all (/= '\n')) . shrink + +shrinkTLLine :: TL.Text -> [TL.Text] +shrinkTLLine = filter (TL.all (/= '\n')) . shrink + +testLowLevel :: TestTree +testLowLevel = + testGroup "lowlevel" [ + testGroup "mul" [ + testProperty "t_mul" t_mul, + testProperty "t_mul32" t_mul32, + testProperty "t_mul64" t_mul64 + ], + + testGroup "misc" [ + testProperty "t_dropWord8" t_dropWord8, + testProperty "t_takeWord8" t_takeWord8, + testProperty "t_take_drop_8" t_take_drop_8, + testProperty "t_use_from" t_use_from, + testProperty "t_use_from0" t_use_from0, + testProperty "t_copy" t_copy, + testProperty "t_peek_cstring" t_peek_cstring, + testProperty "t_peek_cstring_len" t_peek_cstring_len, + testCase "t_literal_length1" t_literal_length1, + testCase "t_literal_length2" t_literal_length2, + testCase "t_literal_surrogates" t_literal_surrogates +#ifdef MIN_VERSION_tasty_inspection_testing + -- Hack to force GHC to keep the binding around until this is fixed: https://gitlab.haskell.org/ghc/ghc/-/issues/26436 + , let _unused = 't_literal_foo in + $(inspectObligations + [ (`hasNoTypes` [''Char, ''[]]) + , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) + , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) + , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) + ] + 't_literal_foo) +#endif + ], + + inputOutput + ] diff --git a/tests/Tests/Properties/Read.hs b/tests/Tests/Properties/Read.hs --- a/tests/Tests/Properties/Read.hs +++ b/tests/Tests/Properties/Read.hs @@ -8,7 +8,7 @@ import Data.Char (isDigit, isHexDigit) import Numeric (showHex) -import Test.Tasty (TestTree, testGroup) +import Test.Tasty (TestTree, testGroup, localOption, mkTimeout) import Test.Tasty.QuickCheck (testProperty) import Test.QuickCheck import Tests.QuickCheckUtils () @@ -38,23 +38,39 @@ isFloaty c = c `elem` ("+-.0123456789eE" :: String) -t_read_rational p tol (n::Double) s = - case p (T.pack (show n) `T.append` t) of - Left err -> counterexample err $ property False - Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol) - where t = T.dropWhile isFloaty s +t_read_rational :: Double -> T.Text -> Property +t_read_rational n s = + case T.rational (T.pack (show n) `T.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. n' === n'' + where + t = T.dropWhile isFloaty s + n'' = read (show n) :: Double -tl_read_rational p tol (n::Double) s = - case p (TL.pack (show n) `TL.append` t) of - Left err -> counterexample err $ property False - Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol) - where t = TL.dropWhile isFloaty s +t_read_double :: Double -> Double -> T.Text -> Property +t_read_double tol n s = + case T.double (T.pack (show n) `T.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol) + where + t = T.dropWhile isFloaty s -t_double = t_read_rational T.double 1e-13 -tl_double = tl_read_rational TL.double 1e-13 -t_rational = t_read_rational T.rational 1e-16 -tl_rational = tl_read_rational TL.rational 1e-16 +tl_read_rational :: Double -> TL.Text -> Property +tl_read_rational n s = + case TL.rational (TL.pack (show n) `TL.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. n' === n'' + where + t = TL.dropWhile isFloaty s + n'' = read (show n) :: Double +tl_read_double :: Double -> Double -> TL.Text -> Property +tl_read_double tol n s = + case TL.rational (TL.pack (show n) `TL.append` t) of + Left err -> counterexample err $ property False + Right (n', t') -> t === t' .&&. property (abs (n - n') <= tol) + where + t = TL.dropWhile isFloaty s testRead :: TestTree testRead = @@ -63,8 +79,33 @@ testProperty "tl_decimal" tl_decimal, testProperty "t_hexadecimal" t_hexadecimal, testProperty "tl_hexadecimal" tl_hexadecimal, - testProperty "t_double" t_double, - testProperty "tl_double" tl_double, - testProperty "t_rational" t_rational, - testProperty "tl_rational" tl_rational - ] + + testProperty "t_double" $ t_read_double 1e-13, + testProperty "tl_double" $ tl_read_double 1e-13, + + testProperty "t_rational" t_read_rational, + testProperty "t_rational 1.3e-2" (t_read_rational 1.3e-2), + testProperty "tl_rational" tl_read_rational, + testProperty "tl_rational 9e-3" (tl_read_rational 9e-3), + + localOption (mkTimeout 100000) $ testGroup "DDoS attacks" [ + testProperty "t_double large positive exponent" $ + T.double (T.pack "1.1e1000000000") === Right (1 / 0, mempty), + testProperty "t_double large negative exponent" $ + T.double (T.pack "1.1e-1000000000") === Right (0.0, mempty), + testProperty "tl_double large positive exponent" $ + TL.double (TL.pack "1.1e1000000000") === Right (1 / 0, mempty), + testProperty "tl_double large negative exponent" $ + TL.double (TL.pack "1.1e-1000000000") === Right (0.0, mempty), + + testProperty "t_rational large positive exponent" $ + T.rational (T.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty), + testProperty "t_rational large negative exponent" $ + T.rational (T.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty), + testProperty "tl_rational large positive exponent" $ + TL.rational (TL.pack "1.1e1000000000") === Right (1 / 0 :: Double, mempty), + testProperty "tl_rational large negative exponent" $ + TL.rational (TL.pack "1.1e-1000000000") === Right (0.0 :: Double, mempty) + ] + + ] diff --git a/tests/Tests/Properties/Substrings.hs b/tests/Tests/Properties/Substrings.hs --- a/tests/Tests/Properties/Substrings.hs +++ b/tests/Tests/Properties/Substrings.hs @@ -15,6 +15,7 @@ import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList import qualified Data.Text as T import qualified Data.Text.Internal.Fusion as S import qualified Data.Text.Internal.Fusion.Common as S @@ -156,9 +157,13 @@ tl_groupBy (applyFun2 -> p) = L.groupBy p `eqP` (map unpackS . TL.groupBy p) t_inits = L.inits `eqP` (map unpackS . T.inits) +t_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . T.initsNE) tl_inits = L.inits `eqP` (map unpackS . TL.inits) +tl_initsNE = NonEmptyList.inits `eqP` (fmap unpackS . TL.initsNE) t_tails = L.tails `eqP` (map unpackS . T.tails) +t_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . T.tailsNE) tl_tails = L.tails `eqPSqrt` (map unpackS . TL.tails) +tl_tailsNE = NonEmptyList.tails `eqP` (fmap unpackS . TL.tailsNE) spanML :: Monad m => (b -> m Bool) -> [b] -> m ([b], [b]) spanML p s = go [] s @@ -224,9 +229,8 @@ (l, []) -> [l] (l, _ : s') -> l : loop s' -t_chunksOf_same_lengths k = conjoin . map ((===k) . T.length) . ini . T.chunksOf k - where ini [] = [] - ini xs = init xs +t_chunksOf_same_lengths k = + conjoin . map ((===k) . T.length) . drop 1 . reverse . T.chunksOf k t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0) where len = L.sum . L.map T.length $ T.chunksOf k t @@ -361,9 +365,13 @@ testProperty "t_groupBy" t_groupBy, testProperty "tl_groupBy" tl_groupBy, testProperty "t_inits" t_inits, + testProperty "t_initsNE" t_initsNE, testProperty "tl_inits" tl_inits, + testProperty "tl_initsNE" tl_initsNE, testProperty "t_tails" t_tails, + testProperty "t_tailsNE" t_tailsNE, testProperty "tl_tails" tl_tails, + testProperty "tl_tailsNE" tl_tailsNE, testProperty "t_spanM" t_spanM, testProperty "t_spanEndM" t_spanEndM, testProperty "tl_spanM" tl_spanM, diff --git a/tests/Tests/Properties/Text.hs b/tests/Tests/Properties/Text.hs --- a/tests/Tests/Properties/Text.hs +++ b/tests/Tests/Properties/Text.hs @@ -1,483 +1,491 @@ --- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. - -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE CPP #-} -{-# LANGUAGE ViewPatterns #-} - -{-# OPTIONS_GHC -fno-warn-missing-signatures #-} - -module Tests.Properties.Text - ( testText - ) where - -import Data.Char (isLower, isLetter, isUpper) -import Data.Maybe (mapMaybe) -import Data.Text.Internal.Fusion.Size -import Data.Word (Word8) -import Test.QuickCheck -import Test.Tasty (TestTree, testGroup) -import Test.Tasty.QuickCheck (testProperty) -import Tests.QuickCheckUtils -import qualified Data.Char as C -import qualified Data.List as L -import qualified Data.Text as T -import qualified Data.Text.Internal.Fusion as S -import qualified Data.Text.Internal.Fusion.Common as S -import qualified Data.Text.Internal.Lazy.Fusion as SL -import qualified Data.Text.Internal.Lazy.Search as S (indices) -import qualified Data.Text.Internal.Search as T (indices) -import qualified Data.Text.Lazy as TL -import qualified Tests.SlowFunctions as Slow - -t_pack_unpack = (T.unpack . T.pack) `eq` id -tl_pack_unpack = (TL.unpack . TL.pack) `eq` id -t_stream_unstream = (S.unstream . S.stream) `eq` id -tl_stream_unstream = (SL.unstream . SL.stream) `eq` id -t_reverse_stream t = (S.reverse . S.reverseStream) t === t -t_singleton c = [c] === (T.unpack . T.singleton) c -tl_singleton c = [c] === (TL.unpack . TL.singleton) c -tl_unstreamChunks x = f 11 x === f 1000 x - where f n = SL.unstreamChunks n . S.streamList -tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id -tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id - -s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) -s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) -sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) - -t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) -tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) -t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) -tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) -t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) -tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) - -s_intercalate c = (L.intercalate c . unSqrt) `eq` - (unpackS . S.intercalate (packS c) . map packS . unSqrt) -t_intercalate c = (L.intercalate c . unSqrt) `eq` - (unpackS . T.intercalate (packS c) . map packS . unSqrt) -tl_intercalate c = (L.intercalate c . unSqrt) `eq` - (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) -t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` - (T.length . T.intercalate (packS c) . map packS . unSqrt) -tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` - (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) -s_intersperse c = L.intersperse c `eqP` - (unpackS . S.intersperse c) -s_intersperse_s c = L.intersperse c `eqP` - (unpackS . S.unstream . S.intersperse c) -sf_intersperse (applyFun -> p) c - = (L.intersperse c . L.filter p) `eqP` - (unpackS . S.intersperse c . S.filter p) -t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) -tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) -t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) -tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) -t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) -tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) -t_reverse = L.reverse `eqP` (unpackS . T.reverse) -tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) -t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) - -t_replace s d = (L.intercalate d . splitOn s) `eqP` - (unpackS . T.replace (T.pack s) (T.pack d)) -tl_replace s d = (L.intercalate d . splitOn s) `eqP` - (unpackS . TL.replace (TL.pack s) (TL.pack d)) - -splitOn :: (Eq a) => [a] -> [a] -> [[a]] -splitOn pat src0 - | l == 0 = error "splitOn: empty" - | otherwise = go src0 - where - l = length pat - go src = search 0 src - where - search _ [] = [src] - search !n s@(_:s') - | pat `L.isPrefixOf` s = take n src : go (drop l s) - | otherwise = search (n+1) s' - -s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs - where s = S.streamList xs -sf_toCaseFold_length (applyFun -> p) xs = - (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) - where s = S.streamList xs -t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t - -tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t -#if MIN_VERSION_base(4,16,0) -t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==> - T.toCaseFold (T.singleton c) === T.singleton (C.toLower c) -#endif - --- | Baseline generated with GHC 9.2 + text-1.2.5.0, -t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389" -t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower)) -t_toCaseFold_cherokeeUpper = conjoin $ - map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper - --- | Generated with GHC 9.2 + text-1.2.5.0, --- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $ --- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c)) --- [minBound .. maxBound] -toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" -cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase -cherokeeLower = ['\xAB70'..'\xABBF'] - -t_toLower_length t = T.length (T.toLower t) >= T.length t -t_toLower_lower t = p (T.toLower t) >= p t - where p = T.length . T.filter isLower -tl_toLower_lower t = p (TL.toLower t) >= p t - where p = TL.length . TL.filter isLower -#if MIN_VERSION_base(4,13,0) -t_toLower_char c = c /= '\304' ==> - T.toLower (T.singleton c) === T.singleton (C.toLower c) -#endif -t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775" - -t_toUpper_length t = T.length (T.toUpper t) >= T.length t -t_toUpper_upper t = p (T.toUpper t) >= p t - where p = T.length . T.filter isUpper -tl_toUpper_upper t = p (TL.toUpper t) >= p t - where p = TL.length . TL.filter isUpper -#if MIN_VERSION_base(4,13,0) -t_toUpper_char c = c `notElem` toUpperExceptions ==> - T.toUpper (T.singleton c) === T.singleton (C.toUpper c) -#endif - --- | Baseline generated with GHC 9.2 + text-1.2.5.0, -t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341" - --- | Generated with GHC 9.2 + text-1.2.5.0, --- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c)) --- [minBound .. maxBound] -toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" - -t_toTitle_title t = all (<= 1) (caps w) - where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle - -- TIL: there exist uppercase-only letters - w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t -t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian) - where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words - -- Surprise! The Spanish/Portuguese ordinal indicators changed - -- from category Ll (letter, lowercase) to Lo (letter, other) - -- in Unicode 7.0 - -- Oh, and there exist lowercase-only letters (see previous test) - stable c = if isLower c - then C.toUpper c /= c - else c /= '\170' && c /= '\186' - -- Georgian text does not have a concept of title case - -- https://en.wikipedia.org/wiki/Georgian_Extended - isGeorgian c = c >= '\4256' && c < '\4352' -#if MIN_VERSION_base(4,13,0) -t_toTitle_char c = c `notElem` toTitleExceptions ==> - T.toTitle (T.singleton c) === T.singleton (C.toUpper c) -#endif - --- | Baseline generated with GHC 9.2 + text-1.2.5.0, -t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389" - --- | Generated with GHC 9.2 + text-1.2.5.0, --- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c)) --- [minBound .. maxBound] -toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" - -t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t -t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t -t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t - -ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) -ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) -ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) - -ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) - where - referenceToTitle _ [] = [] - referenceToTitle False (y : ys) - | C.isLetter y = C.toUpper y : referenceToTitle True ys - | otherwise = y : referenceToTitle False ys - referenceToTitle True (y : ys) - | C.isLetter y = C.toLower y : referenceToTitle True ys - | otherwise = y : referenceToTitle (not (C.isSpace y)) ys - -justifyLeft k c xs = xs ++ L.replicate (k - length xs) c -justifyRight m n xs = L.replicate (m - length xs) n ++ xs -center k c xs - | len >= k = xs - | otherwise = L.replicate l c ++ xs ++ L.replicate r c - where len = length xs - d = k - len - r = d `div` 2 - l = d - r - -s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) - where j = fromIntegral (k :: Word8) -s_justifyLeft_s k c = justifyLeft j c `eqP` - (unpackS . S.unstream . S.justifyLeftI j c) - where j = fromIntegral (k :: Word8) -sf_justifyLeft (applyFun -> p) k c - = (justifyLeft j c . L.filter p) `eqP` - (unpackS . S.justifyLeftI j c . S.filter p) - where j = fromIntegral (k :: Word8) -t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) - where j = fromIntegral (k :: Word8) -tl_justifyLeft k c = justifyLeft j c `eqP` - (unpackS . TL.justifyLeft (fromIntegral j) c) - where j = fromIntegral (k :: Word8) -t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) - where j = fromIntegral (k :: Word8) -tl_justifyRight k c = justifyRight j c `eqP` - (unpackS . TL.justifyRight (fromIntegral j) c) - where j = fromIntegral (k :: Word8) -t_center k c = center j c `eqP` (unpackS . T.center j c) - where j = fromIntegral (k :: Word8) -tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) - where j = fromIntegral (k :: Word8) - -t_elem c = L.elem c `eqP` T.elem c -tl_elem c = L.elem c `eqP` TL.elem c -sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) -sf_filter (applyFun -> q) (applyFun -> p) - = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) - -t_filter (applyFun -> p) - = L.filter p `eqP` (unpackS . T.filter p) -tl_filter (applyFun -> p) - = L.filter p `eqP` (unpackS . TL.filter p) -t_filter_filter (applyFun -> p) (applyFun -> q) - = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) -tl_filter_filter (applyFun -> p) (applyFun -> q) - = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) -t_length_filter (applyFun -> p) - = (L.length . L.filter p) `eqP` (T.length . T.filter p) -tl_length_filter (applyFun -> p) - = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) - -sf_findBy (applyFun -> q) (applyFun -> p) - = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) -t_find (applyFun -> p) = L.find p `eqP` T.find p -tl_find (applyFun -> p) = L.find p `eqP` TL.find p -t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) -tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) - -sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j - where l = L.length s - j = if l == 0 then 0 else i `mod` (3 * l) - l -t_index s i = ((s L.!!) `eq` T.index (packS s)) j - where l = L.length s - j = if l == 0 then 0 else i `mod` (3 * l) - l - -tl_index s i = ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) j - where l = L.length s - j = if l == 0 then 0 else i `mod` (3 * l) - l - -t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p -t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t -tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` - TL.count t -t_zip s = L.zip s `eqP` T.zip (packS s) -tl_zip s = L.zip s `eqP` TL.zip (packS s) -sf_zipWith (applyFun -> p) (applyFun2 -> c) s - = (L.zipWith c (L.filter p s) . L.filter p) `eqP` - (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) -t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) -tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) -t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) -tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) - -t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s -tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s - where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) - conc = T.concat . TL.toChunks -t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> - let s = T.intercalate t ts - in Slow.indices t s === T.indices t s - -t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] -tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] - -t_indices_drop n s pref suff = T.indices s t === Slow.indices s t - where - t = T.drop n $ pref `T.append` s `T.append` suff -tl_indices_drop n s pref suff = - map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) - where - t = TL.drop n $ pref `TL.append` s `TL.append` suff - -tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] -tl_indices_drop_chunked n s pref suff = - map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) - where - -- constructing a pathologically chunked haystack - t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff - -t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t - where - s = T.singleton c - t = T.drop n $ pref `T.append` s `T.append` suff -tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) - where - s = TL.singleton c - t = TL.drop n $ pref `TL.append` s `TL.append` suff - --- Make a stream appear shorter than it really is, to ensure that --- functions that consume inaccurately sized streams behave --- themselves. -shorten :: Int -> S.Stream a -> S.Stream a -shorten n t@(S.Stream arr off len) - | n > 0 = S.Stream arr off (smaller (exactSize n) len) - | otherwise = t - -testText :: TestTree -testText = - testGroup "Text" [ - testGroup "creation/elimination" [ - testProperty "t_pack_unpack" t_pack_unpack, - testProperty "tl_pack_unpack" tl_pack_unpack, - testProperty "t_stream_unstream" t_stream_unstream, - testProperty "tl_stream_unstream" tl_stream_unstream, - testProperty "t_reverse_stream" t_reverse_stream, - testProperty "t_singleton" t_singleton, - testProperty "tl_singleton" tl_singleton, - testProperty "tl_unstreamChunks" tl_unstreamChunks, - testProperty "tl_chunk_unchunk" tl_chunk_unchunk, - testProperty "tl_from_to_strict" tl_from_to_strict - ], - - testGroup "transformations" [ - testProperty "s_map" s_map, - testProperty "s_map_s" s_map_s, - testProperty "sf_map" sf_map, - - testProperty "t_map" t_map, - testProperty "tl_map" tl_map, - testProperty "t_map_map" t_map_map, - testProperty "tl_map_map" tl_map_map, - testProperty "t_length_map" t_length_map, - testProperty "tl_length_map" tl_length_map, - - testProperty "s_intercalate" s_intercalate, - testProperty "t_intercalate" t_intercalate, - testProperty "tl_intercalate" tl_intercalate, - testProperty "t_length_intercalate" t_length_intercalate, - testProperty "tl_length_intercalate" tl_length_intercalate, - testProperty "s_intersperse" s_intersperse, - testProperty "s_intersperse_s" s_intersperse_s, - testProperty "sf_intersperse" sf_intersperse, - testProperty "t_intersperse" t_intersperse, - testProperty "tl_intersperse" tl_intersperse, - testProperty "t_length_intersperse" t_length_intersperse, - testProperty "tl_length_intersperse" tl_length_intersperse, - testProperty "t_transpose" t_transpose, - testProperty "tl_transpose" tl_transpose, - testProperty "t_reverse" t_reverse, - testProperty "tl_reverse" tl_reverse, - testProperty "t_reverse_short" t_reverse_short, - testProperty "t_replace" t_replace, - testProperty "tl_replace" tl_replace, - - testGroup "case conversion" [ - testProperty "s_toCaseFold_length" s_toCaseFold_length, - testProperty "sf_toCaseFold_length" sf_toCaseFold_length, - testProperty "t_toCaseFold_length" t_toCaseFold_length, - testProperty "tl_toCaseFold_length" tl_toCaseFold_length, -#if MIN_VERSION_base(4,16,0) - testProperty "t_toCaseFold_char" t_toCaseFold_char, -#endif - testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions, - testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower, - testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper, - - testProperty "t_toLower_length" t_toLower_length, - testProperty "t_toLower_lower" t_toLower_lower, - testProperty "tl_toLower_lower" tl_toLower_lower, - testProperty "t_toLower_dotted_i" t_toLower_dotted_i, - - testProperty "t_toUpper_length" t_toUpper_length, - testProperty "t_toUpper_upper" t_toUpper_upper, - testProperty "tl_toUpper_upper" tl_toUpper_upper, - testProperty "t_toUpper_exceptions" t_toUpper_exceptions, - - testProperty "t_toTitle_title" t_toTitle_title, - testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, - testProperty "t_toTitle_exceptions" t_toTitle_exceptions, - -#if MIN_VERSION_base(4,13,0) - -- Requires base compliant with Unicode 12.0 - testProperty "t_toLower_char" t_toLower_char, - testProperty "t_toUpper_char" t_toUpper_char, - testProperty "t_toTitle_char" t_toTitle_char, -#endif - - testProperty "t_toUpper_idempotent" t_toUpper_idempotent, - testProperty "t_toLower_idempotent" t_toLower_idempotent, - testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent, - - testProperty "ascii_toLower" ascii_toLower, - testProperty "ascii_toUpper" ascii_toUpper, - testProperty "ascii_toTitle" ascii_toTitle, - testProperty "ascii_toCaseFold" ascii_toCaseFold - ], - - testGroup "justification" [ - testProperty "s_justifyLeft" s_justifyLeft, - testProperty "s_justifyLeft_s" s_justifyLeft_s, - testProperty "sf_justifyLeft" sf_justifyLeft, - testProperty "t_justifyLeft" t_justifyLeft, - testProperty "tl_justifyLeft" tl_justifyLeft, - testProperty "t_justifyRight" t_justifyRight, - testProperty "tl_justifyRight" tl_justifyRight, - testProperty "t_center" t_center, - testProperty "tl_center" tl_center - ] - ], - - testGroup "searching" [ - testProperty "t_elem" t_elem, - testProperty "tl_elem" tl_elem, - testProperty "sf_elem" sf_elem, - testProperty "sf_filter" sf_filter, - testProperty "t_filter" t_filter, - testProperty "tl_filter" tl_filter, - testProperty "t_filter_filter" t_filter_filter, - testProperty "tl_filter_filter" tl_filter_filter, - testProperty "t_length_filter" t_length_filter, - testProperty "tl_length_filter" tl_length_filter, - testProperty "sf_findBy" sf_findBy, - testProperty "t_find" t_find, - testProperty "tl_find" tl_find, - testProperty "t_partition" t_partition, - testProperty "tl_partition" tl_partition - ], - - testGroup "indexing" [ - testProperty "sf_index" sf_index, - testProperty "t_index" t_index, - testProperty "tl_index" tl_index, - testProperty "t_findIndex" t_findIndex, - testProperty "t_count" t_count, - testProperty "tl_count" tl_count, - testProperty "t_indices" t_indices, - testProperty "tl_indices" tl_indices, - testProperty "t_indices_occurs" t_indices_occurs, - - testProperty "t_indices_drop5" t_indices_drop5, - testProperty "tl_indices_drop5" tl_indices_drop5, - testProperty "t_indices_drop" t_indices_drop, - testProperty "tl_indices_drop" tl_indices_drop, - testProperty "tl_indices_chunked" tl_indices_chunked, - testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, - testProperty "t_indices_char_drop" t_indices_char_drop, - testProperty "tl_indices_char_drop" tl_indices_char_drop - ], - - testGroup "zips" [ - testProperty "t_zip" t_zip, - testProperty "tl_zip" tl_zip, - testProperty "sf_zipWith" sf_zipWith, - testProperty "t_zipWith" t_zipWith, - testProperty "tl_zipWith" tl_zipWith, - testProperty "t_length_zipWith" t_length_zipWith, - testProperty "tl_length_zipWith" tl_length_zipWith - ] - ] +-- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. + +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Avoid restricted function" #-} + +module Tests.Properties.Text + ( testText + ) where + +import Control.Exception (SomeException, evaluate, try) +import Data.Char (isLower, isLetter, isUpper) +import Data.Maybe (mapMaybe) +import Data.Text.Internal.Fusion.Size +import Data.Word (Word8) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.Char as C +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Internal.Lazy.Search as S (indices) +import qualified Data.Text.Internal.Search as T (indices) +import qualified Data.Text.Lazy as TL +import qualified Tests.SlowFunctions as Slow +#if MIN_VERSION_base(4, 15, 0) +import qualified GHC.Unicode as G (unicodeVersion) +import qualified Data.Text.Internal.Fusion.CaseMapping as T (unicodeVersion) +#endif + +t_pack_unpack = (T.unpack . T.pack) `eq` id +tl_pack_unpack = (TL.unpack . TL.pack) `eq` id +t_stream_unstream = (S.unstream . S.stream) `eq` id +tl_stream_unstream = (SL.unstream . SL.stream) `eq` id +t_reverse_stream t = (S.reverse . S.reverseStream) t === t +t_singleton c = [c] === (T.unpack . T.singleton) c +tl_singleton c = [c] === (TL.unpack . TL.singleton) c +tl_unstreamChunks x = f 11 x === f 1000 x + where f n = SL.unstreamChunks n . S.streamList +tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id +tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id + +s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) +s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) + +t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) +tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) +t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) +tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) +t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) +tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) + +s_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . S.intercalate (packS c) . map packS . unSqrt) +t_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . T.intercalate (packS c) . map packS . unSqrt) +tl_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` + (T.length . T.intercalate (packS c) . map packS . unSqrt) +tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` + (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +s_intersperse c = L.intersperse c `eqP` + (unpackS . S.intersperse c) +s_intersperse_s c = L.intersperse c `eqP` + (unpackS . S.unstream . S.intersperse c) +sf_intersperse (applyFun -> p) c + = (L.intersperse c . L.filter p) `eqP` + (unpackS . S.intersperse c . S.filter p) +t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) +t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) +tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) +t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) +tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) +t_reverse = L.reverse `eqP` (unpackS . T.reverse) +tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) +t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) + +t_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . T.replace (T.pack s) (T.pack d)) +tl_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . TL.replace (TL.pack s) (TL.pack d)) + +splitOn :: (Eq a) => [a] -> [a] -> [[a]] +splitOn pat src0 + | l == 0 = error "splitOn: empty" + | otherwise = go src0 + where + l = length pat + go src = search 0 src + where + search _ [] = [src] + search !n s@(_:s') + | pat `L.isPrefixOf` s = take n src : go (drop l s) + | otherwise = search (n+1) s' + +s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs + where s = S.streamList xs +sf_toCaseFold_length (applyFun -> p) xs = + (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) + where s = S.streamList xs +t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t + +tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t +t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==> + T.toCaseFold (T.singleton c) === T.singleton (C.toLower c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389" +t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower)) +t_toCaseFold_cherokeeUpper = conjoin $ + map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $ +-- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c)) +-- [minBound .. maxBound] +toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" +cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase +cherokeeLower = ['\xAB70'..'\xABBF'] + +t_toLower_length t = T.length (T.toLower t) >= T.length t +t_toLower_lower t = p (T.toLower t) >= p t + where p = T.length . T.filter isLower +tl_toLower_lower t = p (TL.toLower t) >= p t + where p = TL.length . TL.filter isLower +t_toLower_char c = c /= '\304' ==> + T.toLower (T.singleton c) === T.singleton (C.toLower c) +t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775" + +t_toUpper_length t = T.length (T.toUpper t) >= T.length t +t_toUpper_upper t = p (T.toUpper t) >= p t + where p = T.length . T.filter isUpper +tl_toUpper_upper t = p (TL.toUpper t) >= p t + where p = TL.length . TL.filter isUpper +t_toUpper_char c = c `notElem` toUpperExceptions ==> + T.toUpper (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toTitle_title t = all (<= 1) (caps w) + where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle + -- TIL: there exist uppercase-only letters + w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t +t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian) + where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words + -- Surprise! The Spanish/Portuguese ordinal indicators changed + -- from category Ll (letter, lowercase) to Lo (letter, other) + -- in Unicode 7.0 + -- Oh, and there exist lowercase-only letters (see previous test) + stable c = if isLower c + then C.toUpper c /= c + else c /= '\170' && c /= '\186' + -- Georgian text does not have a concept of title case + -- https://en.wikipedia.org/wiki/Georgian_Extended + isGeorgian c = c >= '\4256' && c < '\4352' +t_toTitle_char c = c `notElem` toTitleExceptions ==> + T.toTitle (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t +t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t +t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t + +ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) +ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) +ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) + +ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) + where + referenceToTitle _ [] = [] + referenceToTitle False (y : ys) + | C.isLetter y = C.toUpper y : referenceToTitle True ys + | otherwise = y : referenceToTitle False ys + referenceToTitle True (y : ys) + | C.isLetter y = C.toLower y : referenceToTitle True ys + | otherwise = y : referenceToTitle (not (C.isSpace y)) ys + +justifyLeft k c xs = xs ++ L.replicate (k - length xs) c +justifyRight m n xs = L.replicate (m - length xs) n ++ xs +center k c xs + | len >= k = xs + | otherwise = L.replicate l c ++ xs ++ L.replicate r c + where len = length xs + d = k - len + r = d `div` 2 + l = d - r + +s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +s_justifyLeft_s k c = justifyLeft j c `eqP` + (unpackS . S.unstream . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +sf_justifyLeft (applyFun -> p) k c + = (justifyLeft j c . L.filter p) `eqP` + (unpackS . S.justifyLeftI j c . S.filter p) + where j = fromIntegral (k :: Word8) +t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) + where j = fromIntegral (k :: Word8) +tl_justifyLeft k c = justifyLeft j c `eqP` + (unpackS . TL.justifyLeft (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) + where j = fromIntegral (k :: Word8) +tl_justifyRight k c = justifyRight j c `eqP` + (unpackS . TL.justifyRight (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_center k c = center j c `eqP` (unpackS . T.center j c) + where j = fromIntegral (k :: Word8) +tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) + where j = fromIntegral (k :: Word8) + +t_elem c = L.elem c `eqP` T.elem c +tl_elem c = L.elem c `eqP` TL.elem c +sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter (applyFun -> q) (applyFun -> p) + = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) + +t_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . T.filter p) +tl_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . TL.filter p) +t_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) +tl_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) +t_length_filter (applyFun -> p) + = (L.length . L.filter p) `eqP` (T.length . T.filter p) +tl_length_filter (applyFun -> p) + = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) + +sf_findBy (applyFun -> q) (applyFun -> p) + = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_find (applyFun -> p) = L.find p `eqP` T.find p +tl_find (applyFun -> p) = L.find p `eqP` TL.find p +t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) + +sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j + where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l + +t_index :: T.Text -> Int -> Property +t_index xs i = ioProperty $ do + ch <- try (evaluate (T.index xs i)) + pure $ case ch of + Left (_ :: SomeException) -> i < 0 .||. i >= T.length xs + Right c -> i >= 0 .&&. i < T.length xs .&&. c === T.unpack xs L.!! i + +tl_index :: TL.Text -> Int -> Property +tl_index xs i = ioProperty $ do + let i' = fromIntegral i + ch <- try (evaluate (TL.index xs i')) + pure $ case ch of + Left (_ :: SomeException) -> i' < 0 .||. i' >= TL.length xs + Right c -> i >= 0 .&&. i' < TL.length xs .&&. c === TL.unpack xs L.!! i + +t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p +t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t +tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` + TL.count t +t_zip s = L.zip s `eqP` T.zip (packS s) +tl_zip s = L.zip s `eqP` TL.zip (packS s) +sf_zipWith (applyFun -> p) (applyFun2 -> c) s + = (L.zipWith c (L.filter p s) . L.filter p) `eqP` + (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) +t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) +tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) + +t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s +tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s + where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) + conc = T.concat . TL.toChunks +t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> + let s = T.intercalate t ts + in Slow.indices t s === T.indices t s + +t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] +tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] + +t_indices_drop n s pref suff = T.indices s t === Slow.indices s t + where + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_drop n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] +tl_indices_drop_chunked n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + -- constructing a pathologically chunked haystack + t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff + +t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t + where + s = T.singleton c + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + s = TL.singleton c + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +-- Make a stream appear shorter than it really is, to ensure that +-- functions that consume inaccurately sized streams behave +-- themselves. +shorten :: Int -> S.Stream a -> S.Stream a +shorten n t@(S.Stream arr off len) + | n > 0 = S.Stream arr off (smaller (exactSize n) len) + | otherwise = t + +testText :: TestTree +testText = + testGroup "Text" [ + testGroup "creation/elimination" [ + testProperty "t_pack_unpack" t_pack_unpack, + testProperty "tl_pack_unpack" tl_pack_unpack, + testProperty "t_stream_unstream" t_stream_unstream, + testProperty "tl_stream_unstream" tl_stream_unstream, + testProperty "t_reverse_stream" t_reverse_stream, + testProperty "t_singleton" t_singleton, + testProperty "tl_singleton" tl_singleton, + testProperty "tl_unstreamChunks" tl_unstreamChunks, + testProperty "tl_chunk_unchunk" tl_chunk_unchunk, + testProperty "tl_from_to_strict" tl_from_to_strict + ], + + testGroup "transformations" [ + testProperty "s_map" s_map, + testProperty "s_map_s" s_map_s, + testProperty "sf_map" sf_map, + + testProperty "t_map" t_map, + testProperty "tl_map" tl_map, + testProperty "t_map_map" t_map_map, + testProperty "tl_map_map" tl_map_map, + testProperty "t_length_map" t_length_map, + testProperty "tl_length_map" tl_length_map, + + testProperty "s_intercalate" s_intercalate, + testProperty "t_intercalate" t_intercalate, + testProperty "tl_intercalate" tl_intercalate, + testProperty "t_length_intercalate" t_length_intercalate, + testProperty "tl_length_intercalate" tl_length_intercalate, + testProperty "s_intersperse" s_intersperse, + testProperty "s_intersperse_s" s_intersperse_s, + testProperty "sf_intersperse" sf_intersperse, + testProperty "t_intersperse" t_intersperse, + testProperty "tl_intersperse" tl_intersperse, + testProperty "t_length_intersperse" t_length_intersperse, + testProperty "tl_length_intersperse" tl_length_intersperse, + testProperty "t_transpose" t_transpose, + testProperty "tl_transpose" tl_transpose, + testProperty "t_reverse" t_reverse, + testProperty "tl_reverse" tl_reverse, + testProperty "t_reverse_short" t_reverse_short, + testProperty "t_replace" t_replace, + testProperty "tl_replace" tl_replace, + + testGroup "case conversion" [ + testProperty "s_toCaseFold_length" s_toCaseFold_length, + testProperty "sf_toCaseFold_length" sf_toCaseFold_length, + testProperty "t_toCaseFold_length" t_toCaseFold_length, + testProperty "tl_toCaseFold_length" tl_toCaseFold_length, + testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions, + testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower, + testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper, + + testProperty "t_toLower_length" t_toLower_length, + testProperty "t_toLower_lower" t_toLower_lower, + testProperty "tl_toLower_lower" tl_toLower_lower, + testProperty "t_toLower_dotted_i" t_toLower_dotted_i, + + testProperty "t_toUpper_length" t_toUpper_length, + testProperty "t_toUpper_upper" t_toUpper_upper, + testProperty "tl_toUpper_upper" tl_toUpper_upper, + testProperty "t_toUpper_exceptions" t_toUpper_exceptions, + + testProperty "t_toTitle_title" t_toTitle_title, + testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, + testProperty "t_toTitle_exceptions" t_toTitle_exceptions, + + testProperty "t_toUpper_idempotent" t_toUpper_idempotent, + testProperty "t_toLower_idempotent" t_toLower_idempotent, + testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent, + + testProperty "ascii_toLower" ascii_toLower, + testProperty "ascii_toUpper" ascii_toUpper, + testProperty "ascii_toTitle" ascii_toTitle, + testProperty "ascii_toCaseFold" ascii_toCaseFold + ], + +#if MIN_VERSION_base(4, 15, 0) + -- Requires matching version of Unicode in base and text + testGroup "char case conversion" $ if T.unicodeVersion == G.unicodeVersion then [ + testProperty "t_toCaseFold_char" t_toCaseFold_char, + testProperty "t_toLower_char" t_toLower_char, + testProperty "t_toUpper_char" t_toUpper_char, + testProperty "t_toTitle_char" t_toTitle_char + ] else [], +#endif + + testGroup "justification" [ + testProperty "s_justifyLeft" s_justifyLeft, + testProperty "s_justifyLeft_s" s_justifyLeft_s, + testProperty "sf_justifyLeft" sf_justifyLeft, + testProperty "t_justifyLeft" t_justifyLeft, + testProperty "tl_justifyLeft" tl_justifyLeft, + testProperty "t_justifyRight" t_justifyRight, + testProperty "tl_justifyRight" tl_justifyRight, + testProperty "t_center" t_center, + testProperty "tl_center" tl_center + ] + ], + + testGroup "searching" [ + testProperty "t_elem" t_elem, + testProperty "tl_elem" tl_elem, + testProperty "sf_elem" sf_elem, + testProperty "sf_filter" sf_filter, + testProperty "t_filter" t_filter, + testProperty "tl_filter" tl_filter, + testProperty "t_filter_filter" t_filter_filter, + testProperty "tl_filter_filter" tl_filter_filter, + testProperty "t_length_filter" t_length_filter, + testProperty "tl_length_filter" tl_length_filter, + testProperty "sf_findBy" sf_findBy, + testProperty "t_find" t_find, + testProperty "tl_find" tl_find, + testProperty "t_partition" t_partition, + testProperty "tl_partition" tl_partition + ], + + testGroup "indexing" [ + testProperty "sf_index" sf_index, + testProperty "t_index" t_index, + testProperty "tl_index" tl_index, + testProperty "t_findIndex" t_findIndex, + testProperty "t_count" t_count, + testProperty "tl_count" tl_count, + testProperty "t_indices" t_indices, + testProperty "tl_indices" tl_indices, + testProperty "t_indices_occurs" t_indices_occurs, + + testProperty "t_indices_drop5" t_indices_drop5, + testProperty "tl_indices_drop5" tl_indices_drop5, + testProperty "t_indices_drop" t_indices_drop, + testProperty "tl_indices_drop" tl_indices_drop, + testProperty "tl_indices_chunked" tl_indices_chunked, + testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, + testProperty "t_indices_char_drop" t_indices_char_drop, + testProperty "tl_indices_char_drop" tl_indices_char_drop + ], + + testGroup "zips" [ + testProperty "t_zip" t_zip, + testProperty "tl_zip" tl_zip, + testProperty "sf_zipWith" sf_zipWith, + testProperty "t_zipWith" t_zipWith, + testProperty "tl_zipWith" tl_zipWith, + testProperty "t_length_zipWith" t_length_zipWith, + testProperty "tl_length_zipWith" tl_length_zipWith + ] + ] diff --git a/tests/Tests/Properties/Transcoding.hs b/tests/Tests/Properties/Transcoding.hs --- a/tests/Tests/Properties/Transcoding.hs +++ b/tests/Tests/Properties/Transcoding.hs @@ -1,7 +1,10 @@ -- | Tests for encoding and decoding {-# LANGUAGE CPP, OverloadedStrings, ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -Wno-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + module Tests.Properties.Transcoding ( testTranscoding ) where diff --git a/tests/Tests/QuickCheckUtils.hs b/tests/Tests/QuickCheckUtils.hs --- a/tests/Tests/QuickCheckUtils.hs +++ b/tests/Tests/QuickCheckUtils.hs @@ -1,289 +1,326 @@ --- | This module provides quickcheck utilities, e.g. arbitrary and show --- instances, and comparison functions, so we can focus on the actual properties --- in the 'Tests.Properties' module. --- -{-# LANGUAGE CPP #-} -{-# LANGUAGE DeriveFunctor #-} -{-# LANGUAGE FlexibleInstances #-} - -{-# OPTIONS_GHC -fno-warn-orphans #-} - -module Tests.QuickCheckUtils - ( BigInt(..) - , NotEmpty(..) - , Sqrt(..) - , SpacyString(..) - , SkewedBool(..) - - , Precision(..) - , precision - - , DecodeErr(..) - , genDecodeErr - - , Stringy(..) - , unpack2 - , eq - , eqP - , eqPSqrt - - , write_read - ) where - -import Control.Arrow ((***)) -import Control.DeepSeq (NFData (..), deepseq) -import Control.Exception (bracket) -import Data.Char (isSpace) -import Data.Text.Foreign (I8) -import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) -import Data.Word (Word8, Word16) -import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, discard, counterexample, scale, (===), (.&&.), NonEmptyList(..)) -import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) -import Tests.Utils -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.Text as T -import qualified Data.Text.Encoding.Error as T -import qualified Data.Text.Internal.Fusion as TF -import qualified Data.Text.Internal.Fusion.Common as TF -import qualified Data.Text.Internal.Lazy as TL -import qualified Data.Text.Internal.Lazy.Fusion as TLF -import qualified Data.Text.Lazy as TL -import qualified System.IO as IO - -genWord8 :: Gen Word8 -genWord8 = chooseAny - -instance Arbitrary I8 where - arbitrary = arbitrarySizedIntegral - shrink = shrinkIntegral - -instance Arbitrary B.ByteString where - arbitrary = B.pack `fmap` listOf genWord8 - shrink = map B.pack . shrink . B.unpack - -instance Arbitrary BL.ByteString where - arbitrary = oneof - [ BL.fromChunks <$> arbitrary - -- so that a single utf8 code point could appear split over up to 4 chunks - , BL.fromChunks . map B.singleton <$> listOf genWord8 - -- so that a code point with 4 byte long utf8 representation - -- could appear split over 3 non-singleton chunks - , (\a b c -> BL.fromChunks [a, b, c]) - <$> arbitrary - <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8) - <*> arbitrary - ] - shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) - --- | For tests that have O(n^2) running times or input sizes, resize --- their inputs to the square root of the originals. -newtype Sqrt a = Sqrt { unSqrt :: a } - deriving (Eq, Show) - -instance Arbitrary a => Arbitrary (Sqrt a) where - arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary - where - smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs - shrink = map Sqrt . shrink . unSqrt - -instance Arbitrary T.Text where - arbitrary = do - t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary - -- Generate chunks that start in the middle of their buffers. - (\i -> T.drop i t) <$> choose (0, T.length t) - shrink = map T.pack . shrink . T.unpack - -instance Arbitrary TL.Text where - arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary - shrink = map TL.pack . shrink . TL.unpack - -newtype BigInt = Big Integer - deriving (Eq, Show) - -instance Arbitrary BigInt where - arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) - shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] - where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer - -newtype NotEmpty a = NotEmpty { notEmpty :: a } - deriving (Eq, Ord, Show) - -instance Arbitrary (NotEmpty T.Text) where - arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary - shrink = fmap (NotEmpty . T.pack . getNonEmpty) - . shrink . NonEmpty . T.unpack . notEmpty - -instance Arbitrary (NotEmpty TL.Text) where - arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary - shrink = fmap (NotEmpty . TL.pack . getNonEmpty) - . shrink . NonEmpty . TL.unpack . notEmpty - -data DecodeErr = Lenient | Ignore | Strict | Replace - deriving (Show, Eq, Bounded, Enum) - -genDecodeErr :: DecodeErr -> Gen T.OnDecodeError -genDecodeErr Lenient = return T.lenientDecode -genDecodeErr Ignore = return T.ignore -genDecodeErr Strict = return T.strictDecode -genDecodeErr Replace = (\c _ _ -> c) <$> frequency - [ (1, return Nothing) - , (50, Just <$> arbitraryUnicodeChar) - ] - -instance Arbitrary DecodeErr where - arbitrary = arbitraryBoundedEnum - -class Stringy s where - packS :: String -> s - unpackS :: s -> String - splitAtS :: Int -> s -> (s,s) - packSChunkSize :: Int -> String -> s - packSChunkSize _ = packS - -instance Stringy String where - packS = id - unpackS = id - splitAtS = splitAt - -instance Stringy (TF.Stream Char) where - packS = TF.streamList - unpackS = TF.unstreamList - splitAtS n s = (TF.take n s, TF.drop n s) - -instance Stringy T.Text where - packS = T.pack - unpackS = T.unpack - splitAtS = T.splitAt - -instance Stringy TL.Text where - packSChunkSize k = TLF.unstreamChunks k . TF.streamList - packS = TL.pack - unpackS = TL.unpack - splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . - TL.splitAt . fromIntegral - -unpack2 :: (Stringy s) => (s,s) -> (String,String) -unpack2 = unpackS *** unpackS - --- Do two functions give the same answer? -eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property -eq a b s = a s =^= b s - --- What about with the RHS packed? -eqP :: (Eq a, Show a, Stringy s) => - (String -> a) -> (s -> a) -> String -> Word8 -> Property -eqP f g s w = counterexample "orig" (f s =^= g t) .&&. - counterexample "mini" (f s =^= g mini) .&&. - counterexample "head" (f sa =^= g ta) .&&. - counterexample "tail" (f sb =^= g tb) - where t = packS s - mini = packSChunkSize 10 s - (sa,sb) = splitAt m s - (ta,tb) = splitAtS m t - l = length s - m | l == 0 = n - | otherwise = n `mod` l - n = fromIntegral w - -eqPSqrt :: (Eq a, Show a, Stringy s) => - (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property -eqPSqrt f g s = eqP f g (unSqrt s) - -instance Arbitrary FPFormat where - arbitrary = arbitraryBoundedEnum - -newtype Precision a = Precision (Maybe Int) - deriving (Eq, Show) - -precision :: a -> Precision a -> Maybe Int -precision _ (Precision prec) = prec - -arbitraryPrecision :: Int -> Gen (Precision a) -arbitraryPrecision maxDigits = Precision <$> do - n <- choose (-1,maxDigits) - return $ if n == -1 - then Nothing - else Just n - -instance Arbitrary (Precision Float) where - arbitrary = arbitraryPrecision 11 - shrink = map Precision . shrink . precision undefined - -instance Arbitrary (Precision Double) where - arbitrary = arbitraryPrecision 22 - shrink = map Precision . shrink . precision undefined - -#if !MIN_VERSION_QuickCheck(2,14,3) -instance Arbitrary IO.Newline where - arbitrary = oneof [return IO.LF, return IO.CRLF] - -instance Arbitrary IO.NewlineMode where - arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary -#endif - -instance Arbitrary IO.BufferMode where - arbitrary = oneof [ return IO.NoBuffering, - return IO.LineBuffering, - return (IO.BlockBuffering Nothing), - (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` - (arbitrary :: Gen Word16) ] - --- This test harness is complex! What property are we checking? --- --- Reading after writing a multi-line file should give the same --- results as were written. --- --- What do we vary while checking this property? --- * The lines themselves, scrubbed to contain neither CR nor LF. (By --- working with a list of lines, we ensure that the data will --- sometimes contain line endings.) --- * Newline translation mode. --- * Buffering. -write_read :: (NFData a, Eq a, Show a) - => ([b] -> a) - -> ((Char -> Bool) -> a -> b) - -> (IO.Handle -> a -> IO ()) - -> (IO.Handle -> IO a) - -> IO.NewlineMode - -> IO.BufferMode - -> [a] - -> Property -write_read _ _ _ _ (IO.NewlineMode IO.LF IO.CRLF) _ _ = discard -write_read unline filt writer reader nl buf ts = ioProperty $ - (===t) <$> act - where - t = unline . map (filt (not . (`elem` "\r\n"))) $ ts - - act = withTempFile $ \path h -> do - IO.hSetEncoding h IO.utf8 - IO.hSetNewlineMode h nl - IO.hSetBuffering h buf - () <- writer h t - IO.hClose h - bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do - IO.hSetEncoding h' IO.utf8 - IO.hSetNewlineMode h' nl - IO.hSetBuffering h' buf - r <- reader h' - r `deepseq` return r - --- Generate various Unicode space characters with high probability -arbitrarySpacyChar :: Gen Char -arbitrarySpacyChar = oneof - [ arbitraryUnicodeChar - , elements $ filter isSpace [minBound..maxBound] - ] - -newtype SpacyString = SpacyString { getSpacyString :: String } - deriving (Eq, Ord, Show, Read) - -instance Arbitrary SpacyString where - arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar - shrink (SpacyString xs) = SpacyString `fmap` shrink xs - -newtype SkewedBool = Skewed { getSkewed :: Bool } - deriving Show - -instance Arbitrary SkewedBool where - arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)] +-- | This module provides quickcheck utilities, e.g. arbitrary and show +-- instances, and comparison functions, so we can focus on the actual properties +-- in the 'Tests.Properties' module. +-- +{-# LANGUAGE CPP #-} +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Tests.QuickCheckUtils + ( BigInt(..) + , NotEmpty(..) + , Sqrt(..) + , SpacyString(..) + , SkewedBool(..) + + , Precision(..) + , precision + + , DecodeErr(..) + , genDecodeErr + + , Stringy(..) + , unpack2 + , eq + , eqP + , eqPSqrt + + , write_read + ) where + +import Control.Arrow ((***)) +import Control.Monad (when) +import Data.Char (isSpace) +import Data.IORef (writeIORef) +import Data.Text.Foreign (I8) +import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) +import Data.Word (Word8) +#if !MIN_VERSION_QuickCheck(2,17,0) +import Data.Word (Word16) +#endif +import qualified GHC.IO.Buffer as GIO +import qualified GHC.IO.Handle.Internals as GIO +import qualified GHC.IO.Handle.Types as GIO +import GHC.IO.Encoding.Types (TextEncoding(textEncodingName)) +import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, counterexample, scale, (.&&.), NonEmptyList(..), forAllShrink) +import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) +import Tests.Utils +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import qualified Data.Text as T +import qualified Data.Text.Encoding.Error as T +import qualified Data.Text.Internal.Fusion as TF +import qualified Data.Text.Internal.Fusion.Common as TF +import qualified Data.Text.Internal.Lazy as TL +import qualified Data.Text.Internal.Lazy.Fusion as TLF +import qualified Data.Text.Lazy as TL +import qualified System.IO as IO + +genWord8 :: Gen Word8 +genWord8 = chooseAny + +instance Arbitrary I8 where + arbitrary = arbitrarySizedIntegral + shrink = shrinkIntegral + +instance Arbitrary B.ByteString where + arbitrary = B.pack `fmap` listOf genWord8 + shrink = map B.pack . shrink . B.unpack + +instance Arbitrary BL.ByteString where + arbitrary = oneof + [ BL.fromChunks <$> arbitrary + -- so that a single utf8 code point could appear split over up to 4 chunks + , BL.fromChunks . map B.singleton <$> listOf genWord8 + -- so that a code point with 4 byte long utf8 representation + -- could appear split over 3 non-singleton chunks + , (\a b c -> BL.fromChunks [a, b, c]) + <$> arbitrary + <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8) + <*> arbitrary + ] + shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) + +-- | For tests that have O(n^2) running times or input sizes, resize +-- their inputs to the square root of the originals. +newtype Sqrt a = Sqrt { unSqrt :: a } + deriving (Eq, Show) + +instance Arbitrary a => Arbitrary (Sqrt a) where + arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary + where + smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs + shrink = map Sqrt . shrink . unSqrt + +instance Arbitrary T.Text where + arbitrary = do + t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary + -- Generate chunks that start in the middle of their buffers. + (\i -> T.drop i t) <$> choose (0, T.length t) + shrink = map T.pack . shrink . T.unpack + +instance Arbitrary TL.Text where + arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary + shrink = map TL.pack . shrink . TL.unpack + +newtype BigInt = Big Integer + deriving (Eq, Show) + +instance Arbitrary BigInt where + arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) + shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] + where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer + +newtype NotEmpty a = NotEmpty { notEmpty :: a } + deriving (Eq, Ord, Show) + +instance Arbitrary (NotEmpty T.Text) where + arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . T.pack . getNonEmpty) + . shrink . NonEmpty . T.unpack . notEmpty + +instance Arbitrary (NotEmpty TL.Text) where + arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . TL.pack . getNonEmpty) + . shrink . NonEmpty . TL.unpack . notEmpty + +data DecodeErr = Lenient | Ignore | Strict | Replace + deriving (Show, Eq, Bounded, Enum) + +genDecodeErr :: DecodeErr -> Gen T.OnDecodeError +genDecodeErr Lenient = return T.lenientDecode +genDecodeErr Ignore = return T.ignore +genDecodeErr Strict = return T.strictDecode +genDecodeErr Replace = (\c _ _ -> c) <$> frequency + [ (1, return Nothing) + , (50, Just <$> arbitraryUnicodeChar) + ] + +instance Arbitrary DecodeErr where + arbitrary = arbitraryBoundedEnum + +class Stringy s where + packS :: String -> s + unpackS :: s -> String + splitAtS :: Int -> s -> (s,s) + packSChunkSize :: Int -> String -> s + packSChunkSize _ = packS + +instance Stringy String where + packS = id + unpackS = id + splitAtS = splitAt + +instance Stringy (TF.Stream Char) where + packS = TF.streamList + unpackS = TF.unstreamList + splitAtS n s = (TF.take n s, TF.drop n s) + +instance Stringy T.Text where + packS = T.pack + unpackS = T.unpack + splitAtS = T.splitAt + +instance Stringy TL.Text where + packSChunkSize k = TLF.unstreamChunks k . TF.streamList + packS = TL.pack + unpackS = TL.unpack + splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . + TL.splitAt . fromIntegral + +unpack2 :: (Stringy s) => (s,s) -> (String,String) +unpack2 = unpackS *** unpackS + +-- Do two functions give the same answer? +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property +eq a b s = a s =^= b s + +-- What about with the RHS packed? +eqP :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> String -> Word8 -> Property +eqP f g s w = counterexample "orig" (f s =^= g t) .&&. + counterexample "mini" (f s =^= g mini) .&&. + counterexample "head" (f sa =^= g ta) .&&. + counterexample "tail" (f sb =^= g tb) + where t = packS s + mini = packSChunkSize 10 s + (sa,sb) = splitAt m s + (ta,tb) = splitAtS m t + l = length s + m | l == 0 = n + | otherwise = n `mod` l + n = fromIntegral w + +eqPSqrt :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property +eqPSqrt f g s = eqP f g (unSqrt s) + +instance Arbitrary FPFormat where + arbitrary = arbitraryBoundedEnum + +newtype Precision a = Precision (Maybe Int) + deriving (Eq, Show) + +precision :: a -> Precision a -> Maybe Int +precision _ (Precision prec) = prec + +arbitraryPrecision :: Int -> Gen (Precision a) +arbitraryPrecision maxDigits = Precision <$> do + n <- choose (-1,maxDigits) + return $ if n == -1 + then Nothing + else Just n + +instance Arbitrary (Precision Float) where + arbitrary = arbitraryPrecision 11 + shrink = map Precision . shrink . precision undefined + +instance Arbitrary (Precision Double) where + arbitrary = arbitraryPrecision 22 + shrink = map Precision . shrink . precision undefined + +#if !MIN_VERSION_QuickCheck(2,14,3) +instance Arbitrary IO.Newline where + arbitrary = oneof [return IO.LF, return IO.CRLF] + +instance Arbitrary IO.NewlineMode where + arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary +#endif + +#if !MIN_VERSION_QuickCheck(2,17,0) +instance Arbitrary IO.BufferMode where + arbitrary = oneof [ return IO.NoBuffering, + return IO.LineBuffering, + return (IO.BlockBuffering Nothing), + (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` + genWord16 ] + +genWord16 :: Gen Word16 +genWord16 = chooseAny +#endif + +-- This test harness is complex! What property are we checking? +-- +-- Reading after writing a multi-line file should give the same +-- results as were written. +-- +-- What do we vary while checking this property? +-- * The lines themselves, scrubbed to contain neither CR nor LF. (By +-- working with a list of lines, we ensure that the data will +-- sometimes contain line endings.) +-- * Newline translation mode. +-- * Buffering. +write_read :: forall a. + (Eq a, Show a) + => Gen a + -> (a -> [a]) + -> (a -> a) -- ^ replace '\n' with '\r\n' (for multiline tests) or append '\r' (for single-line tests) + -> (IO.Handle -> a -> IO ()) + -> (IO.Handle -> IO a) + -> Property +write_read genTxt shrinkTxt expandNl writer reader + = forAllShrink genEncoding shrinkEncoding propTest + where + propTest :: TextEncoding -> IO.BufferMode -> Property + propTest enc mode = forAllShrink genTxt shrinkTxt $ \txt -> ioProperty $ do + file <- emptyTempFile + let with nl k = IO.withFile file IO.ReadWriteMode $ \h -> do + IO.hSetEncoding h enc + IO.hSetBuffering h mode + IO.hSetNewlineMode h nl + setSmallBuffer h + k h + -- Put a very small buffer in Handle to easily test boundary conditions in `writeBlocks` + setSmallBuffer h = GIO.withHandle_ "setSmallBuffer" h $ \h_ -> do + buf <- GIO.newCharBuffer 9 GIO.WriteBuffer + writeIORef (GIO.haCharBuffer h_) buf + readExpecting h txt' msg = do + out <- reader h + when (txt' /= out) $ error (show txt' ++ " /= " ++ show out ++ msg) + -- 'reader' may be 'hGetContents', which closes the handle + -- So we reopen a new file every time. + + -- Test with CRLF encoding + with (IO.NewlineMode IO.CRLF IO.CRLF) $ \h -> do + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 1)" + + -- Re-read without CRLF decoding to check that we did encode CRLF correctly + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + readExpecting h (expandNl txt) " (at location 2)" + + -- Test without CRLF encoding + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + IO.hSetFileSize h 0 + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 3)" + + genEncoding = elements [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be] + shrinkEncoding enc = if textEncodingName enc == textEncodingName IO.utf8 then [] else [IO.utf8] + +-- Generate various Unicode space characters with high probability +arbitrarySpacyChar :: Gen Char +arbitrarySpacyChar = oneof + [ arbitraryUnicodeChar + , elements $ filter isSpace [minBound..maxBound] + ] + +newtype SpacyString = SpacyString { getSpacyString :: String } + deriving (Eq, Ord, Show, Read) + +instance Arbitrary SpacyString where + arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar + shrink (SpacyString xs) = SpacyString `fmap` shrink xs + +newtype SkewedBool = Skewed { getSkewed :: Bool } + deriving Show + +instance Arbitrary SkewedBool where + arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)] diff --git a/tests/Tests/RebindableSyntaxTest.hs b/tests/Tests/RebindableSyntaxTest.hs --- a/tests/Tests/RebindableSyntaxTest.hs +++ b/tests/Tests/RebindableSyntaxTest.hs @@ -1,14 +1,18 @@ -{-# LANGUAGE RebindableSyntax, TemplateHaskell #-} - -module Tests.RebindableSyntaxTest where - -import qualified Data.Text as Text -import Language.Haskell.TH.Syntax (lift) -import Test.Tasty.HUnit (testCase, assertEqual) -import Test.Tasty (TestTree, testGroup) -import Prelude (($)) - -tests :: TestTree -tests = testGroup "RebindableSyntax" - [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a") - ] +{-# LANGUAGE CPP, RebindableSyntax, TemplateHaskell #-} + +module Tests.RebindableSyntaxTest where + +import qualified Data.Text as Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) +import Prelude (($)) + +tests :: TestTree +tests = testGroup "RebindableSyntax" + [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a") + ] diff --git a/tests/Tests/Regressions.hs b/tests/Tests/Regressions.hs --- a/tests/Tests/Regressions.hs +++ b/tests/Tests/Regressions.hs @@ -1,204 +1,236 @@ --- | Regression tests for specific bugs. --- -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} - -module Tests.Regressions - ( - tests - ) where - -import Control.Exception (SomeException, handle) -import Data.Char (isLetter, chr) -import GHC.Exts (Int(..), sizeofByteArray#) -import System.IO -import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure) -import qualified Data.ByteString as B -import Data.ByteString.Char8 () -import qualified Data.ByteString.Lazy as LB -import qualified Data.Text as T -import qualified Data.Text.Array as TA -import qualified Data.Text.Encoding as TE -import qualified Data.Text.Encoding.Error as E -import qualified Data.Text.Internal as T -import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E -import qualified Data.Text.Internal.Lazy.Fusion as LF -import qualified Data.Text.IO as T -import qualified Data.Text.Lazy as LT -import qualified Data.Text.Lazy.Builder as TB -import qualified Data.Text.Lazy.Encoding as LE -import qualified Data.Text.Unsafe as T -import qualified Test.Tasty as F -import qualified Test.Tasty.HUnit as F -import Test.Tasty.HUnit ((@?=)) -import System.Directory (removeFile) - -import Tests.Utils (withTempFile) - --- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring --- caused either a segfault or attempt to allocate a negative number --- of bytes. -lazy_encode_crash :: IO () -lazy_encode_crash = withTempFile $ \ _ h -> - LB.hPut h . LE.encodeUtf8 . LT.pack . replicate 100000 $ 'a' - --- Reported by Pieter Laeremans: attempting to read an incorrectly --- encoded file can result in a crash in the RTS (i.e. not merely an --- exception). -hGetContents_crash :: IO () -hGetContents_crash = do - (path, h) <- openTempFile "." "crashy.txt" - B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) >> hClose h - h' <- openFile path ReadMode - hSetEncoding h' utf8 - handle (\(_::SomeException) -> return ()) $ - T.hGetContents h' >> assertFailure "T.hGetContents should crash" - hClose h' - removeFile path - --- Reported by Ian Lynagh: attempting to allocate a sufficiently large --- string (via either Array.new or Text.replicate) could result in an --- integer overflow. -replicate_crash :: IO () -replicate_crash = handle (\(_::SomeException) -> return ()) $ - T.replicate (2^power) "0123456789abcdef" `seq` - assertFailure "T.replicate should crash" - where - power | maxBound == (2147483647::Int) = 28 - | otherwise = 60 :: Int - --- Reported by John Millikin: a UTF-8 decode error handler could --- return a bogus substitution character, which we would write without --- checking. -utf8_decode_unsafe :: IO () -utf8_decode_unsafe = do - let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" - assertBool "broken error recovery shouldn't break us" (t == "\xfffd") - --- Reported by Eric Seidel: we mishandled mapping Chars that fit in a --- single Word16 to Chars that require two. -mapAccumL_resize :: IO () -mapAccumL_resize = do - let f a _ = (a, '\65536') - count = 5 - val = T.mapAccumL f (0::Int) (T.replicate count "a") - assertEqual "mapAccumL should correctly fill buffers for four-byte results" - (0, T.replicate count "\65536") val - assertEqual "mapAccumL should correctly size buffers for four-byte results" - (count * 4) (T.lengthWord8 (snd val)) - --- See GitHub #197 -t197 :: IO () -t197 = - assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00") - where - currencyParser x = cond == 1 - where - cond = length fltr - fltr = filter (== ',') x - -t221 :: IO () -t221 = - assertEqual "toLower of large input shouldn't crash" - (T.toLower (T.replicate 200000 "0") `seq` ()) - () - -t227 :: IO () -t227 = - assertEqual "take (-3) shouldn't crash with overflow" - (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?") - 0 - -t280_fromString :: IO () -t280_fromString = - assertEqual "TB.fromString performs replacement on invalid scalar values" - (TB.toLazyText (TB.fromString "\xD800")) - (LT.pack "\xFFFD") - -t280_singleton :: IO () -t280_singleton = - assertEqual "TB.singleton performs replacement on invalid scalar values" - (TB.toLazyText (TB.singleton '\xD800')) - (LT.pack "\xFFFD") - --- See GitHub issue #301 --- This tests whether the "TEXT take . drop -> unfused" rule is applied to the --- slice function. When the slice function is fused, a new array will be --- constructed that is shorter than the original array. Without fusion the --- array remains unmodified. -t301 :: IO () -t301 = do - assertEqual "The length of the array remains the same despite slicing" - (I# (sizeofByteArray# originalArr)) - (I# (sizeofByteArray# newArr)) - - assertEqual "The new array still contains the original value" - (T.Text (TA.ByteArray newArr) originalOff originalLen) - original - where - !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" - !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original - -t330 :: IO () -t330 = do - let decodeL = LE.decodeUtf8With E.lenientDecode - assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked" - (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]])) - (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]])) - --- Stream decoders should not loop on incomplete code points -t525 :: IO () -t525 = do - let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) - decodeUtf8With E.lenientDecode "\xC0" @?= "\65533" - LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533" - LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533" - LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533" - LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533" - --- Stream decoders skip one invalid byte at a time -t528 :: IO () -t528 = do - let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) - decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536" - LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536" - LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536" - LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0" - LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280" - -t529 :: IO () -t529 = do - let decode = TE.decodeUtf8With E.lenientDecode - -- https://github.com/haskell/bytestring/issues/575 - assertEqual "Data.ByteString.isValidUtf8 should work correctly" - (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0])) - (decode (B.pack (33 : replicate 31 0 ++ [128, 0]))) - --- See Github #559 --- filter/filter fusion rules should apply predicates in the right order. -t559 :: IO () -t559 = do - T.filter undefined (T.filter (const False) "a") @?= "" - LT.filter undefined (LT.filter (const False) "a") @?= "" - -tests :: F.TestTree -tests = F.testGroup "Regressions" - [ F.testCase "hGetContents_crash" hGetContents_crash - , F.testCase "lazy_encode_crash" lazy_encode_crash - , F.testCase "mapAccumL_resize" mapAccumL_resize - , F.testCase "replicate_crash" replicate_crash - , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe - , F.testCase "t197" t197 - , F.testCase "t221" t221 - , F.testCase "t227" t227 - , F.testCase "t280/fromString" t280_fromString - , F.testCase "t280/singleton" t280_singleton - , F.testCase "t301" t301 - , F.testCase "t330" t330 - , F.testCase "t525" t525 - , F.testCase "t528" t528 - , F.testCase "t529" t529 - , F.testCase "t559" t559 - ] +-- | Regression tests for specific bugs. +-- +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Tests.Regressions + ( + tests + ) where + +import Control.Exception (ErrorCall, SomeException, handle, evaluate, displayException, try) +import Data.Char (isLetter, chr) +import GHC.Exts (Int(..), sizeofByteArray#) +import System.IO +import System.IO.Temp (withSystemTempFile) +import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure, (@?=)) +import qualified Data.ByteString as B +import Data.ByteString.Char8 () +import qualified Data.ByteString.Lazy as LB +import Data.Semigroup (stimes) +import qualified Data.Text as T +import qualified Data.Text.Array as TA +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as E +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as LF +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as LT +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Text.Lazy.Encoding as LE +import qualified Data.Text.Unsafe as T +import qualified Test.Tasty as F +import qualified Test.Tasty.HUnit as F +import Tests.Utils (withTempFile) +import System.IO.Error (isFullError) + +-- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring +-- caused either a segfault or attempt to allocate a negative number +-- of bytes. +lazy_encode_crash :: IO () +lazy_encode_crash = withTempFile $ \ _ h -> do + putRes <- try $ LB.hPut h $ LE.encodeUtf8 $ LT.pack $ replicate 100000 'a' + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> pure () + +-- Reported by Pieter Laeremans: attempting to read an incorrectly +-- encoded file can result in a crash in the RTS (i.e. not merely an +-- exception). +hGetContents_crash :: IO () +hGetContents_crash = withSystemTempFile "crashy.txt" $ \path h -> do + putRes <- try $ B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> do + hClose h + h' <- openFile path ReadMode + hSetEncoding h' utf8 + handle (\(_::SomeException) -> pure ()) $ + T.hGetContents h' >> assertFailure "T.hGetContents should crash" + hClose h' + +-- Reported by Ian Lynagh: attempting to allocate a sufficiently large +-- string (via either Array.new or Text.replicate) could result in an +-- integer overflow. +replicate_crash :: IO () +replicate_crash = handle (\(_::SomeException) -> return ()) $ + T.replicate (2^power) "0123456789abcdef" `seq` + assertFailure "T.replicate should crash" + where + power | maxBound == (2147483647::Int) = 28 + | otherwise = 60 :: Int + +-- Reported by John Millikin: a UTF-8 decode error handler could +-- return a bogus substitution character, which we would write without +-- checking. +utf8_decode_unsafe :: IO () +utf8_decode_unsafe = do + let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" + assertBool "broken error recovery shouldn't break us" (t == "\xfffd") + +-- Reported by Eric Seidel: we mishandled mapping Chars that fit in a +-- single Word16 to Chars that require two. +mapAccumL_resize :: IO () +mapAccumL_resize = do + let f a _ = (a, '\65536') + count = 5 + val = T.mapAccumL f (0::Int) (T.replicate count "a") + assertEqual "mapAccumL should correctly fill buffers for four-byte results" + (0, T.replicate count "\65536") val + assertEqual "mapAccumL should correctly size buffers for four-byte results" + (count * 4) (T.lengthWord8 (snd val)) + +-- See GitHub #197 +t197 :: IO () +t197 = + assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00") + where + currencyParser x = cond == 1 + where + cond = length fltr + fltr = filter (== ',') x + +t221 :: IO () +t221 = + assertEqual "toLower of large input shouldn't crash" + (T.toLower (T.replicate 200000 "0") `seq` ()) + () + +t227 :: IO () +t227 = + assertEqual "take (-3) shouldn't crash with overflow" + (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?") + 0 + +t280_fromString :: IO () +t280_fromString = + assertEqual "TB.fromString performs replacement on invalid scalar values" + (TB.toLazyText (TB.fromString "\xD800")) + (LT.pack "\xFFFD") + +t280_singleton :: IO () +t280_singleton = + assertEqual "TB.singleton performs replacement on invalid scalar values" + (TB.toLazyText (TB.singleton '\xD800')) + (LT.pack "\xFFFD") + +-- See GitHub issue #301 +-- This tests whether the "TEXT take . drop -> unfused" rule is applied to the +-- slice function. When the slice function is fused, a new array will be +-- constructed that is shorter than the original array. Without fusion the +-- array remains unmodified. +t301 :: IO () +t301 = do + assertEqual "The length of the array remains the same despite slicing" + (I# (sizeofByteArray# originalArr)) + (I# (sizeofByteArray# newArr)) + + assertEqual "The new array still contains the original value" + (T.Text (TA.ByteArray newArr) originalOff originalLen) + original + where + !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" + !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original + +t330 :: IO () +t330 = do + let decodeL = LE.decodeUtf8With E.lenientDecode + assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked" + (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]])) + (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]])) + +-- Stream decoders should not loop on incomplete code points +t525 :: IO () +t525 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0" @?= "\65533" + LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533" + +-- Stream decoders skip one invalid byte at a time +t528 :: IO () +t528 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536" + LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536" + LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536" + LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0" + LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280" + +t529 :: IO () +t529 = do + let decode = TE.decodeUtf8With E.lenientDecode + -- https://github.com/haskell/bytestring/issues/575 + assertEqual "Data.ByteString.isValidUtf8 should work correctly" + (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0])) + (decode (B.pack (33 : replicate 31 0 ++ [128, 0]))) + +-- See Github #559 +-- filter/filter fusion rules should apply predicates in the right order. +t559 :: IO () +t559 = do + T.filter undefined (T.filter (const False) "a") @?= "" + LT.filter undefined (LT.filter (const False) "a") @?= "" + +-- Github #633 +-- stimes checked for an `a` to `Int` to `a` roundtrip, but the `a` and `Int` values could represent different integers. +t633 :: IO () +t633 = + handle (\(_ :: ErrorCall) -> return ()) $ do + _ <- evaluate (stimes (maxBound :: Word) "a" :: T.Text) + assertFailure "should fail" + +t648 :: IO () +t648 = withTempFile $ \_ h -> do + hSetEncoding h utf8 + hSetNewlineMode h (NewlineMode LF CRLF) + hSetBuffering h (BlockBuffering $ Just 4) + let line = T.replicate 2047 "_" + T.hPutStrLn h line + hSeek h AbsoluteSeek 0 + line' <- T.hGetLine h + T.append line "\r" @?= line' + +tests :: F.TestTree +tests = F.testGroup "Regressions" + [ F.testCase "hGetContents_crash" hGetContents_crash + , F.testCase "lazy_encode_crash" lazy_encode_crash + , F.testCase "mapAccumL_resize" mapAccumL_resize + , F.testCase "replicate_crash" replicate_crash + , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe + , F.testCase "t197" t197 + , F.testCase "t221" t221 + , F.testCase "t227" t227 + , F.testCase "t280/fromString" t280_fromString + , F.testCase "t280/singleton" t280_singleton + , F.testCase "t301" t301 + , F.testCase "t330" t330 + , F.testCase "t525" t525 + , F.testCase "t528" t528 + , F.testCase "t529" t529 + , F.testCase "t559" t559 + , F.testCase "t633" t633 + , F.testCase "t648" t648 + ] diff --git a/tests/Tests/ShareEmpty.hs b/tests/Tests/ShareEmpty.hs --- a/tests/Tests/ShareEmpty.hs +++ b/tests/Tests/ShareEmpty.hs @@ -1,126 +1,138 @@ -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TemplateHaskell #-} -{-# LANGUAGE BangPatterns #-} - -module Tests.ShareEmpty - ( tests - ) where - -import Control.Exception (evaluate) -import Data.Text -import Language.Haskell.TH.Syntax (lift) -import Test.Tasty.HUnit (testCase, assertFailure, assertEqual) -import Test.Tasty (TestTree, testGroup) -import GHC.Exts -import GHC.Stack -import qualified Data.List as L -import qualified Data.Text as T - - --- | assert that a text value is represented by the same pointer --- as the 'empty' value. -assertPtrEqEmpty :: HasCallStack => Text -> IO () -assertPtrEqEmpty t = do - t' <- evaluate t - empty' <- evaluate empty - assertEqual "" empty' t' - case reallyUnsafePtrEquality# empty' t' of - 1# -> pure () - _ -> assertFailure "Pointers are not equal" -{-# NOINLINE assertPtrEqEmpty #-} - -tests :: TestTree -tests = testGroup "empty Text values are shared" - [ testCase "empty = empty" $ assertPtrEqEmpty T.empty - , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack "" - , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString "" - , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack "")) - , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a" - , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b" - , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty - , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " [] - , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' "" - , testCase "reverse empty = empty" $ assertPtrEqEmpty $ - T.reverse empty - , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $ - T.replace "needle" "replacement" empty - , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold "" - , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower "" - , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper "" - , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle "" - , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $ - justifyLeft 0 ' ' empty - , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $ - justifyRight 0 ' ' empty - , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $ - T.center 0 ' ' empty - , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $ - T.transpose [empty] - , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat [] - , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty] - , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x" - , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty - , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $ - T.unfoldr (const Nothing) () - , testCase "take 0 _ = empty" $ assertPtrEqEmpty $ - T.take 0 "xyz" - , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $ - T.takeEnd 0 "xyz" - , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $ - T.takeWhile (const False) "xyz" - , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $ - T.takeWhileEnd (const False) "xyz" - , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $ - T.drop 5 "xyz" - , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $ - T.dropEnd 5 "xyz" - , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $ - T.dropWhile (const True) "xyz" - , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $ - dropWhileEnd (const True) "xyz" - , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $ - dropAround (const True) empty - , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty - , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty - , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty - , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123" - , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $ - snd $ T.splitAt 5 "123" - , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $ - fst $ T.span (const False) "123" - , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $ - snd $ T.span (const True) "123" - , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $ - fst $ T.span (const False) "123" - , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $ - snd $ T.span (const True) "123" - , testCase "fst (spanM (const $ pure False) _) = empty" $ - assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123" - , testCase "snd (spanM (const $ pure True) _) = empty" $ - assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123" - , testCase "fst (spanEndM (const $ pure True) _) = empty" $ - assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123" - , testCase "snd (spanEndM (const $ pure False) _) = empty" $ - assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123" - , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty - , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty - , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123" - , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty - , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" - , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" - , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') "" - , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234" - , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" "" - , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines [] - , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords [] - , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ - T.stripPrefix empty empty - , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ - T.stripSuffix empty empty - , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $ - mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123" - , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $ - mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz" - , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy "" - ] +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE BangPatterns #-} + +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.ShareEmpty + ( tests + ) where + +import Control.Exception (evaluate) +import Data.Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertFailure, assertEqual) +import Test.Tasty (TestTree, testGroup) +import GHC.Exts +import GHC.Stack +import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList +import qualified Data.Text as T + + +-- | assert that a text value is represented by the same pointer +-- as the 'empty' value. +assertPtrEqEmpty :: HasCallStack => Text -> IO () +assertPtrEqEmpty t = do + t' <- evaluate t + empty' <- evaluate empty + assertEqual "" empty' t' + case reallyUnsafePtrEquality# empty' t' of + 1# -> pure () + _ -> assertFailure "Pointers are not equal" +{-# NOINLINE assertPtrEqEmpty #-} + +tests :: TestTree +tests = testGroup "empty Text values are shared" + [ testCase "empty = empty" $ assertPtrEqEmpty T.empty + , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack "" + , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString "" + , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack "")) + , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a" + , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b" + , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty + , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " [] + , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' "" + , testCase "reverse empty = empty" $ assertPtrEqEmpty $ + T.reverse empty + , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $ + T.replace "needle" "replacement" empty + , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold "" + , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower "" + , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper "" + , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle "" + , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyLeft 0 ' ' empty + , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyRight 0 ' ' empty + , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $ + T.center 0 ' ' empty + , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $ + T.transpose [empty] + , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat [] + , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty] + , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x" + , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty + , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $ + T.unfoldr (const Nothing) () + , testCase "take 0 _ = empty" $ assertPtrEqEmpty $ + T.take 0 "xyz" + , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $ + T.takeEnd 0 "xyz" + , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhile (const False) "xyz" + , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhileEnd (const False) "xyz" + , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $ + T.drop 5 "xyz" + , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $ + T.dropEnd 5 "xyz" + , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $ + T.dropWhile (const True) "xyz" + , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $ + dropWhileEnd (const True) "xyz" + , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $ + dropAround (const True) empty + , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty + , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty + , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty + , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123" + , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $ + snd $ T.splitAt 5 "123" + , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (spanM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123" + , testCase "snd (spanM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123" + , testCase "fst (spanEndM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123" + , testCase "snd (spanEndM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123" + , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty + , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty + , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty + , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123" + , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123" + , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty + , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty + , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" + , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123" + , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') "" + , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234" + , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" "" + , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines [] + , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords [] + , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripPrefix empty empty + , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripSuffix empty empty + , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $ + mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123" + , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $ + mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz" + , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy "" + ] diff --git a/tests/Tests/Utils.hs b/tests/Tests/Utils.hs --- a/tests/Tests/Utils.hs +++ b/tests/Tests/Utils.hs @@ -1,51 +1,50 @@ --- | Miscellaneous testing utilities --- -{-# LANGUAGE ScopedTypeVariables #-} -module Tests.Utils - ( - (=^=) - , withRedirect - , withTempFile - ) where - -import Control.Exception (SomeException, bracket, bracket_, evaluate, try) -import Control.Monad (when) -import GHC.IO.Handle.Internals (withHandle) -import System.Directory (removeFile) -import System.IO (Handle, hClose, hFlush, hIsOpen, hIsWritable, openTempFile) -import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) - --- Ensure that two potentially bottom values (in the sense of crashing --- for some inputs, not looping infinitely) either both crash, or both --- give comparable results for some input. -(=^=) :: (Eq a, Show a) => a -> a -> Property -i =^= j = ioProperty $ do - x <- try (evaluate i) - y <- try (evaluate j) - return $ case (x, y) of - (Left (_ :: SomeException), Left (_ :: SomeException)) - -> property True - (Right a, Right b) -> a === b - e -> counterexample ("Divergence: " ++ show e) $ property False -infix 4 =^= -{-# NOINLINE (=^=) #-} - -withTempFile :: (FilePath -> Handle -> IO a) -> IO a -withTempFile = bracket (openTempFile "." "crashy.txt") cleanupTemp . uncurry - where - cleanupTemp (path,h) = do - open <- hIsOpen h - when open (hClose h) - removeFile path - -withRedirect :: Handle -> Handle -> IO a -> IO a -withRedirect tmp h = bracket_ swap swap - where - whenM p a = p >>= (`when` a) - swap = do - whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp - whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h - withHandle "spam" tmp $ \tmph -> do - hh <- withHandle "spam" h $ \hh -> - return (tmph,hh) - return (hh,()) +-- | Miscellaneous testing utilities +-- +{-# LANGUAGE ScopedTypeVariables #-} +module Tests.Utils + ( + (=^=) + , withRedirect + , withTempFile + , emptyTempFile + ) where + +import Control.Exception (SomeException, bracket_, evaluate, try) +import Control.Monad (when) +import System.IO.Temp (withSystemTempFile, emptySystemTempFile) +import GHC.IO.Handle.Internals (withHandle) +import System.IO (Handle, hFlush, hIsOpen, hIsWritable) +import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) + +-- Ensure that two potentially bottom values (in the sense of crashing +-- for some inputs, not looping infinitely) either both crash, or both +-- give comparable results for some input. +(=^=) :: (Eq a, Show a) => a -> a -> Property +i =^= j = ioProperty $ do + x <- try (evaluate i) + y <- try (evaluate j) + return $ case (x, y) of + (Left (_ :: SomeException), Left (_ :: SomeException)) + -> property True + (Right a, Right b) -> a === b + e -> counterexample ("Divergence: " ++ show e) $ property False +infix 4 =^= +{-# NOINLINE (=^=) #-} + +withTempFile :: (FilePath -> Handle -> IO a) -> IO a +withTempFile = withSystemTempFile "crashy.txt" + +emptyTempFile :: IO FilePath +emptyTempFile = emptySystemTempFile "crashy.txt" + +withRedirect :: Handle -> Handle -> IO a -> IO a +withRedirect tmp h = bracket_ swap swap + where + whenM p a = p >>= (`when` a) + swap = do + whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp + whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h + withHandle "spam" tmp $ \tmph -> do + hh <- withHandle "spam" h $ \hh -> + return (tmph,hh) + return (hh,()) diff --git a/text.cabal b/text.cabal --- a/text.cabal +++ b/text.cabal @@ -1,352 +1,385 @@ -cabal-version: 2.2 -name: text -version: 2.1.1 - -homepage: https://github.com/haskell/text -bug-reports: https://github.com/haskell/text/issues -synopsis: An efficient packed Unicode text type. -description: - . - An efficient packed, immutable Unicode text type (both strict and - lazy). - . - The 'Text' type represents Unicode character strings, in a time and - space-efficient manner. This package provides text processing - capabilities that are optimized for performance critical use, both - in terms of large data quantities and high speed. - . - The 'Text' type provides character-encoding, type-safe case - conversion via whole-string case conversion functions (see "Data.Text"). - It also provides a range of functions for converting 'Text' values to - and from 'ByteStrings', using several standard encodings - (see "Data.Text.Encoding"). - . - Efficient locale-sensitive support for text IO is also supported - (see "Data.Text.IO"). - . - These modules are intended to be imported qualified, to avoid name - clashes with Prelude functions, e.g. - . - > import qualified Data.Text as T - . - == ICU Support - . - To use an extended and very rich family of functions for working - with Unicode text (including normalization, regular expressions, - non-standard encodings, text breaking, and locales), see - the [text-icu package](https://hackage.haskell.org/package/text-icu) - based on the well-respected and liberally - licensed [ICU library](http://site.icu-project.org/). - -license: BSD-2-Clause -license-file: LICENSE -author: Bryan O'Sullivan -maintainer: Haskell Text Team , Core Libraries Committee -copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko -category: Data, Text -build-type: Simple -tested-with: - GHC == 8.2.2 - GHC == 8.4.4 - GHC == 8.6.5 - GHC == 8.8.4 - GHC == 8.10.7 - GHC == 9.0.2 - GHC == 9.2.8 - GHC == 9.4.6 - GHC == 9.6.2 - GHC == 9.8.1 - -extra-source-files: - -- scripts/CaseFolding.txt - -- scripts/SpecialCasing.txt - README.md - changelog.md - scripts/*.hs - simdutf/LICENSE-APACHE - simdutf/LICENSE-MIT - simdutf/simdutf.h - tests/literal-rule-test.sh - tests/LiteralRuleTest.hs - -flag developer - description: operate in developer mode - default: False - manual: True - -flag simdutf - description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed - default: True - manual: True - -flag pure-haskell - description: Don't use text's standard C routines - NB: This feature is not fully implemented. Several C routines are still in - use. - - When this flag is true, text will use pure Haskell variants of the - routines. This is not recommended except for use with GHC's JavaScript - backend. - - This flag also disables simdutf. - - default: False - manual: True - -library - if arch(javascript) || flag(pure-haskell) - cpp-options: -DPURE_HASKELL - else - c-sources: cbits/is_ascii.c - cbits/measure_off.c - cbits/reverse.c - cbits/utils.c - - hs-source-dirs: src - - if flag(simdutf) && !(arch(javascript) || flag(pure-haskell)) - exposed-modules: Data.Text.Internal.Validate.Simd - include-dirs: simdutf - cxx-sources: simdutf/simdutf.cpp - cbits/validate_utf8.cpp - cxx-options: -std=c++17 - cpp-options: -DSIMDUTF - if impl(ghc >= 9.4) - build-depends: system-cxx-std-lib == 1.0 - elif os(darwin) || os(freebsd) - extra-libraries: c++ - elif os(openbsd) - extra-libraries: c++ c++abi pthread - elif os(windows) - -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later - if impl(ghc < 9.3) - extra-libraries: stdc++ - else - extra-libraries: c++ c++abi - elif arch(wasm32) - cpp-options: -DSIMDUTF_NO_THREADS - cxx-options: -fno-exceptions - extra-libraries: c++ c++abi - else - extra-libraries: stdc++ - - -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++. - -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417 - if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1) - build-depends: base < 0 - - -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker. - if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1) - build-depends: base < 0 - - -- GHC 8.10 has linking issues (probably TH-related) on ARM. - if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*) - build-depends: base < 0 - - -- Subword primitives in GHC 9.2.1 are broken on ARM platforms. - if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1) - build-depends: base < 0 - - -- NetBSD + GHC 9.2.1 + TH + C++ does not work together. - -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577 - if flag(simdutf) && os(netbsd) && impl(ghc < 9.4) - build-depends: base < 0 - - exposed-modules: - Data.Text - Data.Text.Array - Data.Text.Encoding - Data.Text.Encoding.Error - Data.Text.Foreign - Data.Text.IO - Data.Text.IO.Utf8 - Data.Text.Internal - Data.Text.Internal.ArrayUtils - Data.Text.Internal.Builder - Data.Text.Internal.Builder.Functions - Data.Text.Internal.Builder.Int.Digits - Data.Text.Internal.Builder.RealFloat.Functions - Data.Text.Internal.ByteStringCompat - Data.Text.Internal.PrimCompat - Data.Text.Internal.Encoding - Data.Text.Internal.Encoding.Fusion - Data.Text.Internal.Encoding.Fusion.Common - Data.Text.Internal.Encoding.Utf16 - Data.Text.Internal.Encoding.Utf32 - Data.Text.Internal.Encoding.Utf8 - Data.Text.Internal.Fusion - Data.Text.Internal.Fusion.CaseMapping - Data.Text.Internal.Fusion.Common - Data.Text.Internal.Fusion.Size - Data.Text.Internal.Fusion.Types - Data.Text.Internal.IO - Data.Text.Internal.Lazy - Data.Text.Internal.Lazy.Encoding.Fusion - Data.Text.Internal.Lazy.Fusion - Data.Text.Internal.Lazy.Search - Data.Text.Internal.Private - Data.Text.Internal.Read - Data.Text.Internal.Search - Data.Text.Internal.StrictBuilder - Data.Text.Internal.Unsafe - Data.Text.Internal.Unsafe.Char - Data.Text.Internal.Validate - Data.Text.Internal.Validate.Native - Data.Text.Lazy - Data.Text.Lazy.Builder - Data.Text.Lazy.Builder.Int - Data.Text.Lazy.Builder.RealFloat - Data.Text.Lazy.Encoding - Data.Text.Lazy.IO - Data.Text.Lazy.Internal - Data.Text.Lazy.Read - Data.Text.Read - Data.Text.Unsafe - - other-modules: - Data.Text.Show - Data.Text.Internal.Measure - Data.Text.Internal.Reverse - Data.Text.Internal.Transformation - Data.Text.Internal.IsAscii - - build-depends: - array >= 0.3 && < 0.6, - base >= 4.10 && < 5, - binary >= 0.5 && < 0.9, - bytestring >= 0.10.4 && < 0.13, - deepseq >= 1.1 && < 1.6, - ghc-prim >= 0.2 && < 0.12, - template-haskell >= 2.5 && < 2.22 - - if impl(ghc < 9.4) - build-depends: data-array-byte >= 0.1 && < 0.2 - - ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 - if flag(developer) - ghc-options: -fno-ignore-asserts - cpp-options: -DASSERTS - if impl(ghc >= 9.2.2) - ghc-options: -fcheck-prim-bounds - - default-language: Haskell2010 - default-extensions: - NondecreasingIndentation - other-extensions: - BangPatterns - CPP - DeriveDataTypeable - ExistentialQuantification - ForeignFunctionInterface - GeneralizedNewtypeDeriving - MagicHash - OverloadedStrings - Rank2Types - RankNTypes - RecordWildCards - Safe - ScopedTypeVariables - TemplateHaskellQuotes - Trustworthy - TypeFamilies - UnboxedTuples - UnliftedFFITypes - -source-repository head - type: git - location: https://github.com/haskell/text - -test-suite tests - type: exitcode-stdio-1.0 - ghc-options: - -Wall -threaded -rtsopts -with-rtsopts=-N - - hs-source-dirs: tests - main-is: Tests.hs - other-modules: - Tests.Lift - Tests.Properties - Tests.Properties.Basics - Tests.Properties.Builder - Tests.Properties.Folds - Tests.Properties.Instances - Tests.Properties.LowLevel - Tests.Properties.Read - Tests.Properties.Substrings - Tests.Properties.Text - Tests.Properties.Transcoding - Tests.Properties.Validate - Tests.QuickCheckUtils - Tests.RebindableSyntaxTest - Tests.Regressions - Tests.SlowFunctions - Tests.ShareEmpty - Tests.Utils - - build-depends: - QuickCheck >= 2.12.6 && < 2.15, - base <5, - bytestring, - deepseq, - directory, - ghc-prim, - tasty, - tasty-hunit, - tasty-quickcheck, - template-haskell, - transformers, - text - if impl(ghc < 9.4) - build-depends: data-array-byte >= 0.1 && < 0.2 - -- Plugin infrastructure does not work properly in 8.6.1, and - -- ghc-9.2.1 library depends on parsec, which causes a circular dependency. - if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2) - build-depends: tasty-inspection-testing - - default-language: Haskell2010 - default-extensions: NondecreasingIndentation - -benchmark text-benchmarks - type: exitcode-stdio-1.0 - - ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m" - if impl(ghc >= 8.6) - ghc-options: -fproc-alignment=64 - - build-depends: base, - bytestring >= 0.10.4, - containers, - deepseq, - directory, - filepath, - tasty-bench >= 0.2, - text, - transformers - - hs-source-dirs: benchmarks/haskell - main-is: Benchmarks.hs - other-modules: - Benchmarks.Builder - Benchmarks.Concat - Benchmarks.DecodeUtf8 - Benchmarks.EncodeUtf8 - Benchmarks.Equality - Benchmarks.FileRead - Benchmarks.FoldLines - Benchmarks.Multilang - Benchmarks.Programs.BigTable - Benchmarks.Programs.Cut - Benchmarks.Programs.Fold - Benchmarks.Programs.Sort - Benchmarks.Programs.StripTags - Benchmarks.Programs.Throughput - Benchmarks.Pure - Benchmarks.ReadNumbers - Benchmarks.Replace - Benchmarks.Search - Benchmarks.Stream - Benchmarks.WordFrequencies - - default-language: Haskell2010 - default-extensions: NondecreasingIndentation - other-extensions: DeriveGeneric +cabal-version: 2.2 +name: text +version: 2.1.4 + +homepage: https://github.com/haskell/text +bug-reports: https://github.com/haskell/text/issues +synopsis: An efficient packed Unicode text type. +description: + . + An efficient packed, immutable Unicode text type (both strict and + lazy). + . + The 'Text' type represents Unicode character strings, in a time and + space-efficient manner. This package provides text processing + capabilities that are optimized for performance critical use, both + in terms of large data quantities and high speed. + . + The 'Text' type provides character-encoding, type-safe case + conversion via whole-string case conversion functions (see "Data.Text"). + It also provides a range of functions for converting 'Text' values to + and from 'ByteStrings', using several standard encodings + (see "Data.Text.Encoding"). + . + Efficient locale-sensitive support for text IO is also supported + (see "Data.Text.IO"). + . + These modules are intended to be imported qualified, to avoid name + clashes with Prelude functions, e.g. + . + > import qualified Data.Text as T + . + == ICU Support + . + To use an extended and very rich family of functions for working + with Unicode text (including normalization, regular expressions, + non-standard encodings, text breaking, and locales), see + the [text-icu package](https://hackage.haskell.org/package/text-icu) + based on the well-respected and liberally + licensed [ICU library](http://site.icu-project.org/). + +license: BSD-2-Clause +license-file: LICENSE +author: Bryan O'Sullivan +maintainer: Haskell Text Team , Core Libraries Committee +copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko +category: Data, Text +build-type: Simple +tested-with: + GHC == 8.4.4 + GHC == 8.6.5 + GHC == 8.8.4 + GHC == 8.10.7 + GHC == 9.0.2 + GHC == 9.2.8 + GHC == 9.4.8 + GHC == 9.6.7 + GHC == 9.8.4 + GHC == 9.10.1 + GHC == 9.12.2 + +extra-source-files: + -- scripts/CaseFolding.txt + -- scripts/SpecialCasing.txt + scripts/*.hs + simdutf/LICENSE-APACHE + simdutf/LICENSE-MIT + simdutf/simdutf_c.h + simdutf/simdutf.h + tests/literal-rule-test.sh + tests/LiteralRuleTest.hs +extra-doc-files: + README.md + changelog.md + +flag developer + description: operate in developer mode + default: False + manual: True + +flag simdutf + description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed + default: True + manual: True + +flag pure-haskell + description: Don't use text's standard C routines + NB: This feature is not fully implemented. Several C routines are still in + use. + + When this flag is true, text will use pure Haskell variants of the + routines. This is not recommended except for use with GHC's JavaScript + backend. + + This flag also disables simdutf. + + default: False + manual: True + +flag ExtendedBenchmarks + description: Runs extra benchmarks which can be very slow. + default: False + manual: True + +library + if arch(javascript) || flag(pure-haskell) + cpp-options: -DPURE_HASKELL + else + c-sources: cbits/is_ascii.c + cbits/reverse.c + cbits/utils.c + if (arch(aarch64)) + c-sources: cbits/aarch64/measure_off.c + else + c-sources: cbits/measure_off.c + + hs-source-dirs: src + + if flag(simdutf) && !(arch(javascript) || flag(pure-haskell)) + exposed-modules: Data.Text.Internal.Validate.Simd + include-dirs: simdutf + c-sources: simdutf/hs_simdutf.c + cxx-sources: simdutf/simdutf.cpp + cxx-options: -std=c++17 + cpp-options: -DSIMDUTF + if impl(ghc >= 9.4) + build-depends: system-cxx-std-lib == 1.0 + elif os(darwin) || os(freebsd) + extra-libraries: c++ + elif os(openbsd) + extra-libraries: c++ c++abi pthread + elif os(windows) + -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later + if impl(ghc < 9.3) + extra-libraries: stdc++ + else + extra-libraries: c++ c++abi + elif arch(wasm32) + cpp-options: -DSIMDUTF_NO_THREADS + cxx-options: -fno-exceptions + extra-libraries: c++ c++abi + else + extra-libraries: stdc++ + + -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417 + if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1) + build-depends: base < 0 + + -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker. + if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1) + build-depends: base < 0 + + -- GHC 8.10 has linking issues (probably TH-related) on ARM. + if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*) + build-depends: base < 0 + + -- Subword primitives in GHC 9.2.1 are broken on ARM platforms. + if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1) + build-depends: base < 0 + + -- NetBSD + GHC 9.2.1 + TH + C++ does not work together. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577 + if flag(simdutf) && os(netbsd) && impl(ghc < 9.4) + build-depends: base < 0 + + exposed-modules: + Data.Text + Data.Text.Array + Data.Text.Encoding + Data.Text.Encoding.Error + Data.Text.Foreign + Data.Text.IO + Data.Text.IO.Utf8 + Data.Text.Internal + Data.Text.Internal.ArrayUtils + Data.Text.Internal.Builder + Data.Text.Internal.Builder.Functions + Data.Text.Internal.Builder.Int.Digits + Data.Text.Internal.Builder.RealFloat.Functions + Data.Text.Internal.ByteStringCompat + Data.Text.Internal.PrimCompat + Data.Text.Internal.Encoding + Data.Text.Internal.Encoding.Fusion + Data.Text.Internal.Encoding.Fusion.Common + Data.Text.Internal.Encoding.Utf16 + Data.Text.Internal.Encoding.Utf32 + Data.Text.Internal.Encoding.Utf8 + Data.Text.Internal.Fusion + Data.Text.Internal.Fusion.CaseMapping + Data.Text.Internal.Fusion.Common + Data.Text.Internal.Fusion.Size + Data.Text.Internal.Fusion.Types + Data.Text.Internal.IO + Data.Text.Internal.Lazy + Data.Text.Internal.Lazy.Encoding.Fusion + Data.Text.Internal.Lazy.Fusion + Data.Text.Internal.Lazy.Search + Data.Text.Internal.Private + Data.Text.Internal.Read + Data.Text.Internal.Search + Data.Text.Internal.StrictBuilder + Data.Text.Internal.Unsafe + Data.Text.Internal.Unsafe.Char + Data.Text.Internal.Validate + Data.Text.Internal.Validate.Native + Data.Text.Lazy + Data.Text.Lazy.Builder + Data.Text.Lazy.Builder.Int + Data.Text.Lazy.Builder.RealFloat + Data.Text.Lazy.Encoding + Data.Text.Lazy.IO + Data.Text.Lazy.Internal + Data.Text.Lazy.Read + Data.Text.Read + Data.Text.Unsafe + + other-modules: + Data.Text.Show + Data.Text.Internal.Measure + Data.Text.Internal.Reverse + Data.Text.Internal.Transformation + Data.Text.Internal.IsAscii + + build-depends: + array >= 0.3 && < 0.6, + base >= 4.11 && < 5, + binary >= 0.8.3 && < 0.9, + bytestring >= 0.10.4 && < 0.13, + deepseq >= 1.1 && < 1.6, + ghc-prim >= 0.2 && < 0.15, + + -- template-haskell-lift was added as a boot library in GHC-9.14 + -- once we no longer wish to backport releases to older major releases of GHC, + -- this conditional can be dropped + if impl(ghc < 9.14) + build-depends: template-haskell >= 2.5 && < 3 + else + build-depends: template-haskell-lift >= 0.1 && <0.2 + + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + + ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 + if flag(developer) + ghc-options: -fno-ignore-asserts + cpp-options: -DASSERTS + if impl(ghc >= 9.2.2) + ghc-options: -fcheck-prim-bounds + + default-language: Haskell2010 + default-extensions: + NondecreasingIndentation + other-extensions: + BangPatterns + CPP + DeriveDataTypeable + ExistentialQuantification + ForeignFunctionInterface + GeneralizedNewtypeDeriving + MagicHash + OverloadedStrings + Rank2Types + RankNTypes + RecordWildCards + Safe + ScopedTypeVariables + TemplateHaskellQuotes + Trustworthy + TypeFamilies + UnboxedTuples + UnliftedFFITypes + +source-repository head + type: git + location: https://github.com/haskell/text + +test-suite tests + type: exitcode-stdio-1.0 + ghc-options: + -Wall -threaded -rtsopts -with-rtsopts=-N + + hs-source-dirs: tests + main-is: Tests.hs + other-modules: + Tests.Lift + Tests.Properties + Tests.Properties.Basics + Tests.Properties.Builder + Tests.Properties.Folds + Tests.Properties.Instances + Tests.Properties.LowLevel + Tests.Properties.Read + Tests.Properties.Substrings + Tests.Properties.Text + Tests.Properties.Transcoding + Tests.Properties.CornerCases + Tests.Properties.Validate + Tests.QuickCheckUtils + Tests.RebindableSyntaxTest + Tests.Regressions + Tests.SlowFunctions + Tests.ShareEmpty + Tests.Utils + + build-depends: + QuickCheck >= 2.12.6 && < 2.18, + base <5, + binary, + bytestring, + deepseq, + ghc-prim, + tasty, + tasty-hunit, + tasty-quickcheck, + temporary, + transformers, + text + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + if impl(ghc < 9.14) + build-depends: template-haskell + else + build-depends: template-haskell-lift + + -- Plugin infrastructure does not work properly in 8.6.1, and + -- ghc-9.2.1 library depends on parsec, which causes a circular dependency. + if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2) + build-depends: tasty-inspection-testing + + -- https://github.com/haskellari/splitmix/issues/101 + if os(openbsd) + build-depends: splitmix < 0.1.3 || > 0.1.3.1 + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + +benchmark text-benchmarks + type: exitcode-stdio-1.0 + + ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m" + if impl(ghc >= 8.6) + ghc-options: -fproc-alignment=64 + if flag(ExtendedBenchmarks) + cpp-options: -DExtendedBenchmarks + + build-depends: base, + bytestring >= 0.10.4, + containers, + deepseq, + directory, + filepath, + tasty-bench >= 0.2, + temporary, + text, + transformers + + hs-source-dirs: benchmarks/haskell + main-is: Benchmarks.hs + other-modules: + Benchmarks.Builder + Benchmarks.Concat + Benchmarks.DecodeUtf8 + Benchmarks.EncodeUtf8 + Benchmarks.Equality + Benchmarks.FileRead + Benchmarks.FileWrite + Benchmarks.FoldLines + Benchmarks.Micro + Benchmarks.Multilang + Benchmarks.Programs.BigTable + Benchmarks.Programs.Cut + Benchmarks.Programs.Fold + Benchmarks.Programs.Sort + Benchmarks.Programs.StripTags + Benchmarks.Programs.Throughput + Benchmarks.Pure + Benchmarks.ReadNumbers + Benchmarks.Replace + Benchmarks.Search + Benchmarks.Stream + Benchmarks.WordFrequencies + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + other-extensions: DeriveGeneric