text 2.1.2 → 2.1.4
raw patch · 53 files changed
Files
- benchmarks/haskell/Benchmarks.hs +3/−2
- benchmarks/haskell/Benchmarks/FileWrite.hs +6/−3
- benchmarks/haskell/Benchmarks/Micro.hs +2/−1
- benchmarks/haskell/Benchmarks/Programs/Fold.hs +5/−4
- benchmarks/haskell/Benchmarks/Pure.hs +18/−0
- benchmarks/haskell/Benchmarks/ReadNumbers.hs +2/−1
- benchmarks/haskell/Benchmarks/WordFrequencies.hs +3/−2
- cbits/measure_off.c +231/−229
- cbits/validate_utf8.cpp +0/−11
- changelog.md +440/−380
- scripts/Arsec.hs +48/−49
- scripts/CaseFolding.hs +47/−46
- scripts/CaseMapping.hs +66/−41
- scripts/SpecialCasing.hs +61/−63
- scripts/UnicodeData.hs +51/−0
- simdutf/hs_simdutf.c +9/−0
- simdutf/simdutf.cpp too large to diff
- simdutf/simdutf.h +13573/−3757
- simdutf/simdutf_c.h +339/−0
- src/Data/Text.hs +2276/−2143
- src/Data/Text/Array.hs +361/−339
- src/Data/Text/Encoding.hs +588/−581
- src/Data/Text/Encoding/Error.hs +119/−120
- src/Data/Text/Foreign.hs +212/−206
- src/Data/Text/IO.hs +211/−189
- src/Data/Text/IO/Utf8.hs +94/−94
- src/Data/Text/Internal.hs +274/−275
- src/Data/Text/Internal/Builder.hs +350/−346
- src/Data/Text/Internal/Encoding/Utf8.hs +318/−298
- src/Data/Text/Internal/Fusion/CaseMapping.hs +7984/−7706
- src/Data/Text/Internal/IO.hs +308/−306
- src/Data/Text/Internal/Lazy.hs +151/−151
- src/Data/Text/Internal/Transformation.hs +340/−340
- src/Data/Text/Lazy.hs +1911/−1850
- src/Data/Text/Lazy/Builder/RealFloat.hs +3/−0
- src/Data/Text/Lazy/Encoding.hs +254/−244
- src/Data/Text/Lazy/Internal.hs +0/−1
- src/Data/Text/Show.hs +25/−4
- tests/Tests/Lift.hs +39/−34
- tests/Tests/Properties.hs +2/−0
- tests/Tests/Properties/CornerCases.hs +39/−0
- tests/Tests/Properties/Folds.hs +357/−342
- tests/Tests/Properties/Instances.hs +9/−0
- tests/Tests/Properties/LowLevel.hs +176/−164
- tests/Tests/Properties/Substrings.hs +2/−3
- tests/Tests/Properties/Text.hs +491/−483
- tests/Tests/Properties/Transcoding.hs +4/−1
- tests/Tests/QuickCheckUtils.hs +326/−307
- tests/Tests/RebindableSyntaxTest.hs +18/−14
- tests/Tests/Regressions.hs +236/−204
- tests/Tests/ShareEmpty.hs +138/−133
- tests/Tests/Utils.hs +50/−51
- text.cabal +385/−365
benchmarks/haskell/Benchmarks.hs view
@@ -10,6 +10,7 @@ import System.IO #ifdef mingw32_HOST_OS+import System.IO.Temp (emptySystemTempFile) import System.Directory (removeFile) #endif @@ -40,11 +41,11 @@ mkSink :: IO (FilePath, Handle) mkSink = do #ifdef mingw32_HOST_OS- (sinkFn, sink) <- openTempFile "." "dev.null"+ sinkFn <- emptySystemTempFile "dev.null" #else let sinkFn = "/dev/null"- sink <- openFile sinkFn WriteMode #endif+ sink <- openFile sinkFn WriteMode hSetEncoding sink utf8 pure (sinkFn, sink)
benchmarks/haskell/Benchmarks/FileWrite.hs view
@@ -15,11 +15,10 @@ import Control.DeepSeq (NFData, deepseq) import Data.Bifunctor (first) import Data.List (intercalate, intersperse)-import Data.Semigroup ((<>)) import Data.String (fromString) import Data.Text (StrictText) import Data.Text.Internal.Lazy (LazyText, defaultChunkSize)-import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(NoBuffering,LineBuffering,BlockBuffering), hSetBuffering, hSetNewlineMode)+import System.IO (Handle, Newline(CRLF,LF), NewlineMode(NewlineMode), BufferMode(..), hSetBuffering, hSetNewlineMode) import Test.Tasty.Bench (Benchmark, bgroup, bench, whnfAppIO) import qualified Data.Text as T import qualified Data.Text.IO as T@@ -112,11 +111,15 @@ #endif where- lazy, lazyNewlines, lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText)+ lazy, lazyNewlines :: (String, StrictText -> LazyText) lazy = ("lazy", L.fromChunks . T.chunksOf defaultChunkSize) lazyNewlines = ("lazy many newlines", snd lazy . snd strictNewlines)++#ifdef ExtendedBenchmarks+ lazySmallChunks, lazySmallChunksNewlines :: (String, StrictText -> LazyText) lazySmallChunks = ("lazy small chunks", L.fromChunks . T.chunksOf 10) lazySmallChunksNewlines = ("lazy small chunks many newlines", snd lazySmallChunks . snd strictNewlines)+#endif strict, strictNewlines :: (String, StrictText -> StrictText) strict = ("strict", id)
benchmarks/haskell/Benchmarks/Micro.hs view
@@ -2,6 +2,7 @@ module Benchmarks.Micro (benchmark) where +import qualified Data.List.NonEmpty as NE import qualified Data.Text.Lazy as TL import qualified Data.Text as T import Test.Tasty.Bench (Benchmark, Benchmarkable, bgroup, bcompareWithin, bench, nf)@@ -9,7 +10,7 @@ benchmark :: Benchmark benchmark = bgroup "Micro" [ blinear "lazy-inits--last" 500000 2 0.1 $ \len ->- nf (last . TL.inits) (chunks len)+ nf (NE.last . TL.initsNE) (chunks len) , blinear "lazy-inits--map-take1" 500000 2 0.1 $ \len -> nf (map (TL.take 1) . TL.inits) (chunks len) ]
benchmarks/haskell/Benchmarks/Programs/Fold.hs view
@@ -17,8 +17,9 @@ ( benchmark ) where -import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.List (intersperse)+import Prelude hiding (Foldable(..)) import System.IO (Handle) import Test.Tasty.Bench (Benchmark, bench, whnfIO) import qualified Data.Text as T@@ -29,7 +30,7 @@ benchmark :: FilePath -> Handle -> Benchmark benchmark i o =- bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80+ bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . foldText 80 -- | We represent a paragraph by a word list --@@ -37,8 +38,8 @@ -- | Fold a text ---fold :: Int -> T.Text -> TL.Text-fold maxWidth = TLB.toLazyText . mconcat .+foldText :: Int -> T.Text -> TL.Text+foldText maxWidth = TLB.toLazyText . mconcat . intersperse "\n\n" . map (foldParagraph maxWidth) . paragraphs -- | Fold a paragraph
benchmarks/haskell/Benchmarks/Pure.hs view
@@ -26,6 +26,8 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import qualified Data.Text.Lazy.Encoding as TL+import Data.Semigroup+import Data.List.NonEmpty (NonEmpty((:|))) data Env = Env { bsa :: !BS.ByteString@@ -83,6 +85,14 @@ [ benchT $ nf T.concat tl , benchTL $ nf TL.concat tll ]+ , bgroup "sconcat"+ [ benchT $ nf sconcat (T.empty :| tl)+ , benchTL $ nf sconcat (TL.empty :| tll)+ ]+ , bgroup "stimes"+ [ benchT $ nf (stimes (10 :: Int)) ta+ , benchTL $ nf (stimes (10 :: Int)) tla+ ] , bgroup "cons" [ benchT $ nf (T.cons c) ta , benchTL $ nf (TL.cons c) tla@@ -207,6 +217,14 @@ , bgroup "zipWith" [ benchT $ nf (T.zipWith min tb) ta , benchTL $ nf (TL.zipWith min tlb) tla+ ]+ , bgroup "length . unpack" -- length should fuse with unpack+ [ benchT $ nf (L.length . T.unpack) ta+ , benchTL $ nf (L.length . TL.unpack) tla+ ]+ , bgroup "length . drop 1 . unpack" -- no list fusion because of drop 1+ [ benchT $ nf (L.length . L.drop 1 . T.unpack) ta+ , benchTL $ nf (L.length . L.drop 1 . TL.unpack) tla ] , bgroup "length" [ bgroup "cons"
benchmarks/haskell/Benchmarks/ReadNumbers.hs view
@@ -21,8 +21,9 @@ , benchmark ) where +import Data.Foldable (Foldable(..))+import Prelude hiding (Foldable(..)) import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf)-import Data.List (foldl') import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL
benchmarks/haskell/Benchmarks/WordFrequencies.hs view
@@ -13,9 +13,10 @@ , benchmark ) where -import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf)-import Data.List (foldl')+import Data.Foldable (Foldable(..)) import Data.Map (Map)+import Prelude hiding (Foldable(..))+import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T
cbits/measure_off.c view
@@ -1,229 +1,231 @@-/*- * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com>- */--#include <string.h>-#include <stdint.h>-#include <sys/types.h>-#ifdef __x86_64__-#include <emmintrin.h>-#include <xmmintrin.h>-#include <immintrin.h>-#include <cpuid.h>-#endif-#include <stdbool.h>--// stdatomic.h has been introduces in gcc 4.9-#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__))-#define __STDC_NO_ATOMICS__-#endif--#ifndef __STDC_NO_ATOMICS__-#include <stdatomic.h>-#endif--/*- Clang-6 does not enable proper -march flags for assembly modules- which leads to "error: instruction requires: AVX-512 ISA"- at the assembler phase.-- Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6- https://en.wikipedia.org/wiki/Xcode#Toolchain_versions- and it's the latest available version on macOS 10.13.-- Disable AVX-512 instructions as they are most likely not supported- on the hardware running clang-6.-*/-#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \- || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__)-#define COMPILER_SUPPORTS_AVX512-#endif---#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)-bool has_avx512_vl_bw() {-#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__)- uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;- __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);- // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features- const bool has_avx512_bw = ebx & (1 << 30);- const bool has_avx512_vl = ebx & (1 << 31);- // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl);- return has_avx512_bw && has_avx512_vl;-#else- return false;-#endif-}-#endif--/*- measure_off_naive / measure_off_avx / measure_off_sse- take a UTF-8 sequence between src and srcend, and a number of characters cnt.- If the sequence is long enough to contain cnt characters, then return how many bytes- remained unconsumed. Otherwise, if the sequence is shorter, return- negated count of lacking characters. Cf. _hs_text_measure_off below.-*/--static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{- // Count leading bytes in 8 byte sequence- while (src < srcend - 7){- uint64_t w64;- memcpy(&w64, src, sizeof(uint64_t));- // find leading bytes by finding every byte that is not a continuation- // byte. The bit twiddle only results in a 0 if the original byte starts- // with 0b11...- w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL;- // compute the popcount of w64 with two bit shifts and a multiplication- size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1})- * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i- // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) - // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) - // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k- // and each i satisfies this equation for exactly one such j- // So each byte of the result contains the sum we want.- ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j- if (cnt < leads) break;- cnt-= leads;- src+= 8;- }-- // Skip until next leading byte- while (src < srcend){- uint8_t w8 = *src;- if ((int8_t)w8 >= -0x40) break;- src++;- }-- // Finish up with tail- while (src < srcend && cnt > 0){- uint8_t leadByte = *src++;- cnt--;- src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0);- }-- return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt);-}--#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)-__attribute__((target("avx512vl,avx512bw")))-static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{- while (src < srcend - 63){- __m512i w512 = _mm512_loadu_si512((__m512i *)src);- // Which bytes are either < 128 or >= 192?- uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF));- size_t leads = __builtin_popcountll(mask);- if (cnt < leads) break;- cnt-= leads;- src+= 64;- }-- // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties- // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html-- if (src < srcend - 31){- __m256i w256 = _mm256_loadu_si256((__m256i *)src);- uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF));- size_t leads = __builtin_popcountl(mask);- if (cnt >= leads){- cnt-= leads;- src+= 32;- }- }-- if (src < srcend - 15){- __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16- uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8- size_t leads = __builtin_popcountl(mask);- if (cnt >= leads){- cnt-= leads;- src+= 16;- }- }-- return measure_off_naive(src, srcend, cnt);-}-#endif--/* Count the number of bits set in the argument - * - This is a temporary workaround for the fact that- the GHC RTS linker does not recognize the `__builtin_popcountll`- symbol.- - See https://gitlab.haskell.org/ghc/ghc/-/issues/21787- https://gitlab.haskell.org/ghc/ghc/-/issues/19900- https://github.com/haskell/text/issues/450- - It can be removed and the usages of popcount64 replaced with- `__builtin_popcountll` once affected versions of the compiler- are no longer in widespread use.- - Once GHC learns to recognize the symbol, this can be gated- by CPP to call __builtin_popcountll when using the appropriate- version of GHC.-*/-static inline const size_t popcount16(uint16_t x) {-- // Taken from https://en.wikipedia.org/wiki/Hamming_weight- const uint16_t m1 = 0x5555; //binary: 0101...- const uint16_t m2 = 0x3333; //binary: 00110011..- const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ...- x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits- x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits - x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits - return (x >> 8) + (x & 0x00FF);-}--static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt)-{-#ifdef __x86_64__- while (src < srcend - 15){- __m128i w128 = _mm_loadu_si128((__m128i *)src);- // Which bytes are either < 128 or >= 192?- uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF)));- size_t leads = popcount16(mask);- if (cnt < leads) break;- cnt-= leads;- src+= 16;- }-#endif-- return measure_off_naive(src, srcend, cnt);-}--typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t);--/*- _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len),- and a number of code points (aka characters) cnt. If the buffer is long enough- to contain cnt characters, then _hs_text_measure_off returns a non-negative number,- measuring their size in code units (aka bytes). If the buffer is shorter,- _hs_text_measure_off returns a non-positive number, which is a negated total count- of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0- as well.-- This scheme allows us to implement both take/drop and length with the same C function.-- The input buffer (src, off, len) must be a valid UTF-8 sequence,- this condition is not checked.-*/-ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) {-#ifndef __STDC_NO_ATOMICS__- static _Atomic measure_off_t s_impl = (measure_off_t)NULL;- measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed);- if (!impl) {-#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512)- impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse;-#else- impl = measure_off_sse;-#endif- atomic_store_explicit(&s_impl, impl, memory_order_relaxed);- }-#else- measure_off_t impl = measure_off_sse;-#endif- ssize_t ret = (*impl)(src + off, src + off + len, cnt);- return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret));-}+/* + * Copyright (c) 2021 Andrew Lelechenko <andrew.lelechenko@gmail.com> + */ + +#include <string.h> +#include <stdint.h> +#include <sys/types.h> +#ifdef __x86_64__ +#include <emmintrin.h> +#include <xmmintrin.h> +#include <immintrin.h> +#include <cpuid.h> +#endif +#include <stdbool.h> + +// stdatomic.h has been introduces in gcc 4.9 +#if !(__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || defined(__clang_major__)) +#ifndef __STDC_NO_ATOMICS__ +#define __STDC_NO_ATOMICS__ +#endif +#endif + +#ifndef __STDC_NO_ATOMICS__ +#include <stdatomic.h> +#endif + +/* + Clang-6 does not enable proper -march flags for assembly modules + which leads to "error: instruction requires: AVX-512 ISA" + at the assembler phase. + + Apple LLVM version 10.0.0 (clang-1000.11.45.5) is based on clang-6 + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions + and it's the latest available version on macOS 10.13. + + Disable AVX-512 instructions as they are most likely not supported + on the hardware running clang-6. +*/ +#if !((defined(__apple_build_version__) && __apple_build_version__ <= 10001145) \ + || (defined(__clang_major__) && __clang_major__ <= 6)) && !defined(__STDC_NO_ATOMICS__) +#define COMPILER_SUPPORTS_AVX512 +#endif + + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +bool has_avx512_vl_bw() { +#if (__GNUC__ >= 7 || __GNUC__ == 6 && __GNUC_MINOR__ >= 3) || defined(__clang_major__) + uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; + __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); + // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features + const bool has_avx512_bw = ebx & (1 << 30); + const bool has_avx512_vl = ebx & (1 << 31); + // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); + return has_avx512_bw && has_avx512_vl; +#else + return false; +#endif +} +#endif + +/* + measure_off_naive / measure_off_avx / measure_off_sse + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + // find leading bytes by finding every byte that is not a continuation + // byte. The bit twiddle only results in a 0 if the original byte starts + // with 0b11... + w64 = ((w64 << 1) | ~w64) & 0x8080808080808080ULL; + // compute the popcount of w64 with two bit shifts and a multiplication + size_t leads = ( (w64 >> 7) // w64 >> 7 = Sum{0<= i <= 7} x_i * 256^i (x_i \in {0,1}) + * (0x0101010101010101ULL) // 0x0101010101010101 = Sum{0<= i <= 7} 256^i + // (Sum{0<= i <= 7} x_i * 256^i) * (Sum{0<= j <= 7} 256^j) + // =(mod 256^8) (Sum{0<= k <= 7} (256^k) * (Sum {0 <= l < 7} x_l) + // as the coefficients of 256^k in the result are the x_i such that i+j =(mod 8) k + // and each i satisfies this equation for exactly one such j + // So each byte of the result contains the sum we want. + ) >> 56; // bit shift to get a single byte which contains Sum {0 <= j < 7} x_j + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) +__attribute__((target("avx512vl,avx512bw"))) +static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + __m512i w512 = _mm512_loadu_si512((__m512i *)src); + // Which bytes are either < 128 or >= 192? + uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); + size_t leads = __builtin_popcountll(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties + // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html + + if (src < srcend - 31){ + __m256i w256 = _mm256_loadu_si256((__m256i *)src); + uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 32; + } + } + + if (src < srcend - 15){ + __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 + uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 16; + } + } + + return measure_off_naive(src, srcend, cnt); +} +#endif + +/* Count the number of bits set in the argument + * + This is a temporary workaround for the fact that + the GHC RTS linker does not recognize the `__builtin_popcountll` + symbol. + + See https://gitlab.haskell.org/ghc/ghc/-/issues/21787 + https://gitlab.haskell.org/ghc/ghc/-/issues/19900 + https://github.com/haskell/text/issues/450 + + It can be removed and the usages of popcount64 replaced with + `__builtin_popcountll` once affected versions of the compiler + are no longer in widespread use. + + Once GHC learns to recognize the symbol, this can be gated + by CPP to call __builtin_popcountll when using the appropriate + version of GHC. +*/ +static inline const size_t popcount16(uint16_t x) { + + // Taken from https://en.wikipedia.org/wiki/Hamming_weight + const uint16_t m1 = 0x5555; //binary: 0101... + const uint16_t m2 = 0x3333; //binary: 00110011.. + const uint16_t m4 = 0x0f0f; //binary: 4 zeros, 4 ones ... + x -= (x >> 1) & m1; //put count of each 2 bits into those 2 bits + x = (x & m2) + ((x >> 2) & m2); //put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & m4; //put count of each 8 bits into those 8 bits + return (x >> 8) + (x & 0x00FF); +} + +static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ +#ifdef __x86_64__ + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are either < 128 or >= 192? + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); + size_t leads = popcount16(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } +#endif + + return measure_off_naive(src, srcend, cnt); +} + +typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { +#ifndef __STDC_NO_ATOMICS__ + static _Atomic measure_off_t s_impl = (measure_off_t)NULL; + measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); + if (!impl) { +#if defined(__x86_64__) && defined(COMPILER_SUPPORTS_AVX512) + impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; +#else + impl = measure_off_sse; +#endif + atomic_store_explicit(&s_impl, impl, memory_order_relaxed); + } +#else + measure_off_t impl = measure_off_sse; +#endif + ssize_t ret = (*impl)(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +}
− cbits/validate_utf8.cpp
@@ -1,11 +0,0 @@-#include "simdutf.h"--extern "C"-int _hs_text_is_valid_utf8(const char* str, size_t len){- return simdutf::validate_utf8(str, len);-}--extern "C"-int _hs_text_is_valid_utf8_offset(const char* str, size_t off, size_t len){- return simdutf::validate_utf8(str + off, len);-}
changelog.md view
@@ -1,380 +1,440 @@-### 2.1.2--* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618)--* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581)--* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558)--* [Add `foldlM'`](https://github.com/haskell/text/pull/543)--* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599)--* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608)--* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619)--* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565)--* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579)--* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600)--* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589)--* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617)--* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575)--* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572)--* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611)--* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code.--### 2.1.1--* Add pure Haskell implementations as an alternative to C-based ones,- suitable for JavaScript backend.--* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547)--* [Share empty `Text` values](https://github.com/haskell/text/pull/493)--* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553)--* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551)--* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560)--### 2.1--* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474)--* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503)--* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483)--* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527)--* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528)--* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534)--### 2.0.2--* [Add decoding functions in `Data.Text.Encoding` that allow- more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge!- * `decodeASCIIPrefix`- * `decodeUtf8Chunk`- * `decodeUtf8More`- * `Utf8ValidState`- * `startUtf8ValidState`- * `StrictBuilder`- * `strictBuilderToText`- * `textToStrictBuilder`- * `validateUtf8Chunk`- * `validateUtf8More`--* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495)--* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497)--* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499)--* Add internal module `Data.Text.Internal.StrictBuilder`--* Add internal module `Data.Text.Internal.Encoding`--* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module.--* [Speed up case conversions](https://github.com/haskell/text/pull/460)--* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468)--* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485)--### 2.0.1--* Improve portability of C and C++ code.-* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413)-* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402)-* [Add `fromPtr0`](https://github.com/haskell/text/pull/423)-* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436)-* [Add `withCString`](https://github.com/haskell/text/pull/431)-* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437)--### 2.0--* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365):- * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`.- * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`.- * Rename functions and types in `Data.Text.Foreign` to reflect switch- from `Word16` to `Word8`.- * Rename slicing functions in `Data.Text.Unsafe` to reflect switch- from `Word16` to `Word8`.- * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`.- * Change semantics and order of arguments of `Data.Text.Array.copyI`:- pass length, not end offset.- * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines.- * Extend interface of `Data.Text.Array` with more utility functions.- * Add `instance Show Data.Text.Unsafe.Iter`.- * Add `Data.Text.measureOff`.- * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`.- * Export `Data.Text.Internal.Lazy.equal`.- * Export `Data.Text.Internal.append`.- * Add `Data.Text.Internal.Private.spanAscii_`.- * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane.-* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348)-* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342)-* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343)-* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354)-* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355)-* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355)-* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388)--### 1.2.5.0--* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305)-* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320)-* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274)-* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281)-* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325)-* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302)-* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334)-* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333)--### 1.2.4.1--* Support `template-haskell-2.17.0.0`-* Support `bytestring-0.11`-* Add `take . drop` related RULE--### 1.2.4.0--* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232)--* Update Haddock documentation to better reflect fusion eligibility; improve fusion- rules for `takeWhileEnd` and `length` (gh-241, ghc-202)--* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times,- call it only `O(log n)` times on chunks of increasing size. The total- asymptotic complexity remains `O(nm)`. (gh-209)--* Support `base-4.13.0.0`--### 1.2.3.1--* Make `decodeUtf8With` fail explicitly for unsupported non-BMP- replacement characters instead silent undefined behaviour (gh-213)--* Fix termination condition for file reads via `Data.Text.IO`- operations (gh-223)--* A serious correctness issue affecting uses of `take` and `drop` with- negative counts has been fixed (gh-227)--* A bug in the case-mapping functions resulting in unreasonably large- allocations with large arguments has been fixed (gh-221)--### 1.2.3.0--* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec- (updated from 8.0).--* Bug fix: the lazy `takeWhileEnd` function violated the- [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51)- (gh-184).--* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197).--* New function: `unsnoc` (gh-173).--* Reduce memory overhead in `encodeUTF8` (gh-194).--* Improve UTF-8 decoder error-recovery (gh-182).--* Minor documentation improvements (`@since` annotations, more- examples, clarifications).--#### 1.2.2.2--* The `toTitle` function now correctly handles letters that- immediately follow punctuation. Before, `"there's"` would turn into- `"There'S"`. Now, it becomes `"There's"`.--* The implementation of unstreaming is faster, resulting in operations- such as `map` and `intersperse` speeding up by up to 30%, with- smaller code generated.--* The optimised length comparison function is now more likely to be- used after some rewrite rule tweaking.--* Bug fix: an off-by-one bug in `takeEnd` is fixed.--* Bug fix: a logic error in `takeWord16` is fixed.--#### 1.2.2.1--* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.- The build flag has been renamed accordingly. Your army of diligent- maintainers apologizes for the churn.--* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec- (updated from 7.0)--* An STG lint error has been fixed--### 1.2.2.0--* The `integer-simple` package, upon which this package optionally- depended, has been replaced with `integer-pure`. The build flag has- been renamed accordingly.--* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a- `get`, the error is propagated via `fail` instead of an uncatchable- crash.--* New function: `takeWhileEnd`--* New instances for the `Text` types:- * if `base` >= 4.7: `PrintfArg`- * if `base` >= 4.9: `Semigroup`--#### 1.2.1.3--* Bug fix: As it turns out, moving the literal rewrite rules to simplifier- phase 2 does not prevent competition with the `unpack` rule, which is- also active in this phase. Unfortunately this was hidden due to a silly- test environment mistake. Moving literal rules back to phase 1 finally- fixes GHC Trac #10528 correctly.--#### 1.2.1.2--* Bug fix: Run literal rewrite rules in simplifier phase 2.- The behavior of the simplifier changed in GHC 7.10.2,- causing these rules to fail to fire, leading to poor code generation- and long compilation times. See- [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).--#### 1.2.1.1--* Expose unpackCString#, which you should never use.--### 1.2.1.0--* Added Binary instances for both Text types. (If you have previously- been using the text-binary package to get a Binary instance, it is- now obsolete.)--#### 1.2.0.6--* Fixed a space leak in UTF-8 decoding--#### 1.2.0.5--* Feature parity: repeat, cycle, iterate are now implemented for lazy- Text, and the Data instance is more complete--* Build speed: an inliner space explosion has been fixed with toCaseFold--* Bug fix: encoding Int to a Builder would infinite-loop if the- integer-simple package was used--* Deprecation: OnEncodeError and EncodeError are deprecated, as they- are never used--* Internals: some types that are used internally in fusion-related- functions have moved around, been renamed, or been deleted (we don't- bump the major version if .Internal modules change)--* Spec compliance: toCaseFold now follows the Unicode 7.0 spec- (updated from 6.3)--#### 1.2.0.4--* Fixed an incompatibility with base < 4.5--#### 1.2.0.3--* Update formatRealFloat to correspond to the definition in versions- of base newer than 4.5 (https://github.com/bos/text/issues/105)--#### 1.2.0.2--* Bumped lower bound on deepseq to 1.4 for compatibility with the- upcoming GHC 7.10--#### 1.2.0.1--* Fixed a buffer overflow in rendering of large Integers- (https://github.com/bos/text/issues/99)--## 1.2.0.0--* Fixed an integer overflow in the replace function- (https://github.com/bos/text/issues/81)--* Fixed a hang in lazy decodeUtf8With- (https://github.com/bos/text/issues/87)--* Reduced codegen bloat caused by use of empty and single-character- literals--* Added an instance of IsList for GHC 7.8 and above--### 1.1.1.0--* The Data.Data instance now allows gunfold to work, via a virtual- pack constructor--* dropEnd, takeEnd: new functions--* Comparing the length of a Text against a number can now- short-circuit in more cases--#### 1.1.0.1--* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes- in single-byte chunks--## 1.1.0.0--* encodeUtf8: Performance is improved by up to 4x.--* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions,- available only if bytestring >= 0.10.4.0 is installed, that allow- very fast and flexible encoding of a Text value to a bytestring- Builder.-- As an example of the performance gain to be had, the- encodeUtf8BuilderEscaped function helps to double the speed of JSON- encoding in the latest version of aeson! (Note: if all you need is a- plain ByteString, encodeUtf8 is still the faster way to go.)--* All of the internal module hierarchy is now publicly exposed. If a- module is in the .Internal hierarchy, or is documented as internal,- use at your own risk - there are no API stability guarantees for- internal modules!--#### 1.0.0.1--* decodeUtf8: Fixed a regression that caused us to incorrectly- identify truncated UTF-8 as valid (gh-61)--# 1.0.0.0--* Added support for Unicode 6.3.0 to case conversion functions--* New function toTitle converts words in a string to title case--* New functions peekCStringLen and withCStringLen simplify- interoperability with C functions--* Added support for decoding UTF-8 in stream-friendly fashion--* Fixed a bug in mapAccumL--* Added trusted Haskell support--* Removed support for GHC 6.10 (released in 2008) and older+### 2.1.4 - 2026-01-27 + +* [Upgrade to Unicode 17.0](https://github.com/haskell/text/pull/658) + [Fix `CaseMapping` generation script to not depend on GHC's Unicode data](https://github.com/haskell/text/pull/687) + +* [simdutf: update to 8.0.0](https://github.com/haskell/text/pull/685) + +* [Add `decodeUtf8Lenient` for lazy `Text`](https://github.com/haskell/text/pull/690) + +* [`scanl`/`scanr` should replace invalid `Char` in the initial value](https://github.com/haskell/text/pull/669) + +* [Shave off redundant field of `Text.Internal.Buffer`](https://github.com/haskell/text/pull/659) + +* [Switch from template-haskell to template-haskell-lift](https://github.com/haskell/text/pull/661) + +#### Minor changes + +* [Avoid calling `length` on chunks in lazy `splitAt`](https://github.com/haskell/text/pull/676) + +* [Check for zero length in internal `isSingleton`](https://github.com/haskell/text/pull/675) + +* [Implement folds directly, without resorting to streaming framework](https://github.com/haskell/text/pull/667) + +* [Implement `cons`, `snoc`, `head`, `isSingleton`, `isPrefixOf` directly, without resorting to streaming framework](https://github.com/haskell/text/pull/666) + +* [Mark `caseConvert` (the underlying implementation of `toUpper` / `toLower` / `toTitle`) as `INLINABLE`, not `INLINE`](https://github.com/haskell/text/pull/664) + +* [Express `index` via `measureOff` instead of going through fusion framework](https://github.com/haskell/text/pull/663) + +* [Guard `#define __STDC_NO_ATOMICS__` by `#ifndef`](https://github.com/haskell/text/pull/657) + +* [Support QuickCheck-2.17](https://github.com/haskell/text/pull/662) + +* [Bump lower bound of binary to >= 0.8.3](https://github.com/haskell/text/pull/673) + +#### Documentation + +* [A bit more documentation for `Data.Text.Internal.Encoding.Utf8`](https://github.com/haskell/text/pull/691) + +* [Clarify documentation of `Data.Text.Foreign`](https://github.com/haskell/text/pull/681) + +* [Haddocks: Hyperlink some identifiers and modules](https://github.com/haskell/text/pull/677) + +* [`since` pragmas for type synonyms](https://github.com/haskell/text/pull/671) + +* [Improve documentation for `streamDecodeUtf8With`](https://github.com/haskell/text/pull/665) + +* [Add comprehensive documentation for `hGetChunk`](https://github.com/haskell/text/pull/655) + +### 2.1.3 - 2025-08-01 + +* [Fix CRLF handling in IO functions](https://github.com/haskell/text/pull/649) + +* [Change `utf8LengthByLeader` to a branching implementation](https://github.com/haskell/text/pull/635) + +* [Define `stimes 0` for lazy text](https://github.com/haskell/text/pull/641) + +* [Add implementation of `sconcat` and `stimes` for strict `Text`](https://github.com/haskell/text/pull/580) and [Fix `stimes` for strict text when size wraps around `Int`](https://github.com/haskell/text/pull/639) + +* [Allow list fusion for `unpack` over both strict and lazy `Text`](https://github.com/haskell/text/pull/629) + +### 2.1.2 + +* [Update case mappings for Unicode 16.0](https://github.com/haskell/text/pull/618) + +* [Add type synonym for lazy builders. Deprecated `StrictBuilder` for `StrictTextBuilder`](https://github.com/haskell/text/pull/581) + +* [Add `initsNE` and `tailsNE`](https://github.com/haskell/text/pull/558) + +* [Add `foldlM'`](https://github.com/haskell/text/pull/543) + +* [Add `Data.Text.Foreign.peekCString`](https://github.com/haskell/text/pull/599) + +* [Add `Data.Text.show` and `Data.Text.Lazy.show`](https://github.com/haskell/text/pull/608) + +* [Add pattern synonyms `Empty`, `(:<)`, and `(:>)`](https://github.com/haskell/text/pull/619) + +* [Improve precision of `Data.Text.Read.rational`](https://github.com/haskell/text/pull/565) + +* [`Data.Text.IO.Utf8`: use `B.putStrLn` instead of `B.putStr t >> B.putStr "\n"`](https://github.com/haskell/text/pull/579) + +* [`Data.Text.IO` and `Data.Text.Lazy.IO`: Make `putStrLn` more atomic with line or block buffering](https://github.com/haskell/text/pull/600) + +* [Integrate UTF-8 `hPutStr` to standard `hPutStr`](https://github.com/haskell/text/pull/589) + +* [Serialise `Text` without going through `ByteString`](https://github.com/haskell/text/pull/617) + +* [Make `splitAt` strict in its first argument, even if input is empty](https://github.com/haskell/text/pull/575) + +* [Improve lazy performance of `Data.Text.Lazy.inits`](https://github.com/haskell/text/pull/572) + +* [Implement `Data.Text.unpack` and `Data.Text.toTitle` directly, without streaming](https://github.com/haskell/text/pull/611) + +* [Make `fromString` `INLINEABLE` instead of `INLINE`](https://github.com/haskell/text/pull/571) to reduce the size of generated code. + +### 2.1.1 + +* Add pure Haskell implementations as an alternative to C-based ones, + suitable for JavaScript backend. + +* [Add type synonyms for lazy and strict text flavours](https://github.com/haskell/text/pull/547) + +* [Share empty `Text` values](https://github.com/haskell/text/pull/493) + +* [Fix bug in `isValidUtf8ByteArray`](https://github.com/haskell/text/pull/553) + +* [Optimize the implementation of `Data.Text.concat`](https://github.com/haskell/text/pull/551) + +* [Fix `filter/filter` rules for `Text` and lazy `Text`](https://github.com/haskell/text/pull/560) + +### 2.1 + +* [Switch `Data.Text.Array` to `Data.Array.Byte`](https://github.com/haskell/text/pull/474) + +* [Add `Text.IO.Utf8` module](https://github.com/haskell/text/pull/503) + +* [Expose UTF-8 validation functions from internal module](https://github.com/haskell/text/pull/483) + +* [Fix handling of incomplete input in stream decoders](https://github.com/haskell/text/pull/527) + +* [Fix handling of invalid bytes in stream decoders](https://github.com/haskell/text/pull/528) + +* [Make Lift Text work under RebindableSyntax](https://github.com/haskell/text/pull/534) + +### 2.0.2 + +* [Add decoding functions in `Data.Text.Encoding` that allow + more control for error handling and for how to allocate text](https://github.com/haskell/text/pull/448). Thanks to David Sledge! + * `decodeASCIIPrefix` + * `decodeUtf8Chunk` + * `decodeUtf8More` + * `Utf8ValidState` + * `startUtf8ValidState` + * `StrictBuilder` + * `strictBuilderToText` + * `textToStrictBuilder` + * `validateUtf8Chunk` + * `validateUtf8More` + +* [Fix quadratic slowdown when decoding invalid UTF-8 bytestrings](https://github.com/haskell/text/issues/495) + +* [Add `isAscii :: Text -> Bool`](https://github.com/haskell/text/issues/497) + +* [Add `decodeASCII' :: ByteString -> Maybe Text`](https://github.com/haskell/text/issues/499) + +* Add internal module `Data.Text.Internal.StrictBuilder` + +* Add internal module `Data.Text.Internal.Encoding` + +* Add `Data.Text.Internal.Encoding.Utf8.updateDecoderState` and export `utf8{Accept,Reject}State` from the same module. + +* [Speed up case conversions](https://github.com/haskell/text/pull/460) + +* [Reduce code bloat for literal strings](https://github.com/haskell/text/pull/468) + +* [Remove support for GHC 8.0](https://github.com/haskell/text/pull/485) + +### 2.0.1 + +* Improve portability of C and C++ code. +* [Make `Lift` instance more efficient](https://github.com/haskell/text/pull/413) +* [Make `toCaseFold` idempotent](https://github.com/haskell/text/pull/402) +* [Add `fromPtr0`](https://github.com/haskell/text/pull/423) +* [Add `Data.Text.foldr'`](https://github.com/haskell/text/pull/436) +* [Add `withCString`](https://github.com/haskell/text/pull/431) +* [Add `spanM` and `spanEndM`](https://github.com/haskell/text/pull/437) + +### 2.0 + +* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): + * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. + * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. + * Rename functions and types in `Data.Text.Foreign` to reflect switch + from `Word16` to `Word8`. + * Rename slicing functions in `Data.Text.Unsafe` to reflect switch + from `Word16` to `Word8`. + * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. + * Change semantics and order of arguments of `Data.Text.Array.copyI`: + pass length, not end offset. + * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. + * Extend interface of `Data.Text.Array` with more utility functions. + * Add `instance Show Data.Text.Unsafe.Iter`. + * Add `Data.Text.measureOff`. + * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. + * Export `Data.Text.Internal.Lazy.equal`. + * Export `Data.Text.Internal.append`. + * Add `Data.Text.Internal.Private.spanAscii_`. + * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. +* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) +* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) +* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) +* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) +* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) +* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) +* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) + +### 1.2.5.0 + +* [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) +* [Allow `template-haskell-2.18.0.0`](https://github.com/haskell/text/pull/320) +* [Add `elem :: Char -> Text -> Bool` to `Data.Text` and `Data.Text.Lazy`](https://github.com/haskell/text/pull/274) +* [Replace surrogate code points in `Data.Text.Internal.Builder.{singleton,fromString}`](https://github.com/haskell/text/pull/281) +* [Use `unsafeWithForeignPtr` when available](https://github.com/haskell/text/pull/325) +* [Use vectorized CPU instructions for decoding and encoding](https://github.com/haskell/text/pull/302) +* [Regenerate case mapping in accordance to Unicode 13.0](https://github.com/haskell/text/pull/334) +* [Fix UTF-8 decoding of lazy bytestrings](https://github.com/haskell/text/pull/333) + +### 1.2.4.1 + +* Support `template-haskell-2.17.0.0` +* Support `bytestring-0.11` +* Add `take . drop` related RULE + +### 1.2.4.0 + +* Add TH `Lift` instances for `Data.Text.Text` and `Data.Text.Lazy.Text` (gh-232) + +* Update Haddock documentation to better reflect fusion eligibility; improve fusion + rules for `takeWhileEnd` and `length` (gh-241, ghc-202) + +* Optimise `Data.Text.replicate`. Rather than calling `memcpy` `n` times, + call it only `O(log n)` times on chunks of increasing size. The total + asymptotic complexity remains `O(nm)`. (gh-209) + +* Support `base-4.13.0.0` + +### 1.2.3.1 + +* Make `decodeUtf8With` fail explicitly for unsupported non-BMP + replacement characters instead silent undefined behaviour (gh-213) + +* Fix termination condition for file reads via `Data.Text.IO` + operations (gh-223) + +* A serious correctness issue affecting uses of `take` and `drop` with + negative counts has been fixed (gh-227) + +* A bug in the case-mapping functions resulting in unreasonably large + allocations with large arguments has been fixed (gh-221) + +### 1.2.3.0 + +* Spec compliance: `toCaseFold` now follows the Unicode 9.0 spec + (updated from 8.0). + +* Bug fix: the lazy `takeWhileEnd` function violated the + [lazy text invariant](https://github.com/bos/text/blob/1.2.3.0/Data/Text/Internal/Lazy.hs#L51) + (gh-184). + +* Bug fix: Fixed usage of size hints causing incorrect behavior (gh-197). + +* New function: `unsnoc` (gh-173). + +* Reduce memory overhead in `encodeUTF8` (gh-194). + +* Improve UTF-8 decoder error-recovery (gh-182). + +* Minor documentation improvements (`@since` annotations, more + examples, clarifications). + +#### 1.2.2.2 + +* The `toTitle` function now correctly handles letters that + immediately follow punctuation. Before, `"there's"` would turn into + `"There'S"`. Now, it becomes `"There's"`. + +* The implementation of unstreaming is faster, resulting in operations + such as `map` and `intersperse` speeding up by up to 30%, with + smaller code generated. + +* The optimised length comparison function is now more likely to be + used after some rewrite rule tweaking. + +* Bug fix: an off-by-one bug in `takeEnd` is fixed. + +* Bug fix: a logic error in `takeWord16` is fixed. + +#### 1.2.2.1 + +* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken. + The build flag has been renamed accordingly. Your army of diligent + maintainers apologizes for the churn. + +* Spec compliance: `toCaseFold` now follows the Unicode 8.0 spec + (updated from 7.0) + +* An STG lint error has been fixed + +### 1.2.2.0 + +* The `integer-simple` package, upon which this package optionally + depended, has been replaced with `integer-pure`. The build flag has + been renamed accordingly. + +* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a + `get`, the error is propagated via `fail` instead of an uncatchable + crash. + +* New function: `takeWhileEnd` + +* New instances for the `Text` types: + * if `base` >= 4.7: `PrintfArg` + * if `base` >= 4.9: `Semigroup` + +#### 1.2.1.3 + +* Bug fix: As it turns out, moving the literal rewrite rules to simplifier + phase 2 does not prevent competition with the `unpack` rule, which is + also active in this phase. Unfortunately this was hidden due to a silly + test environment mistake. Moving literal rules back to phase 1 finally + fixes GHC Trac #10528 correctly. + +#### 1.2.1.2 + +* Bug fix: Run literal rewrite rules in simplifier phase 2. + The behavior of the simplifier changed in GHC 7.10.2, + causing these rules to fail to fire, leading to poor code generation + and long compilation times. See + [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528). + +#### 1.2.1.1 + +* Expose unpackCString#, which you should never use. + +### 1.2.1.0 + +* Added Binary instances for both Text types. (If you have previously + been using the text-binary package to get a Binary instance, it is + now obsolete.) + +#### 1.2.0.6 + +* Fixed a space leak in UTF-8 decoding + +#### 1.2.0.5 + +* Feature parity: repeat, cycle, iterate are now implemented for lazy + Text, and the Data instance is more complete + +* Build speed: an inliner space explosion has been fixed with toCaseFold + +* Bug fix: encoding Int to a Builder would infinite-loop if the + integer-simple package was used + +* Deprecation: OnEncodeError and EncodeError are deprecated, as they + are never used + +* Internals: some types that are used internally in fusion-related + functions have moved around, been renamed, or been deleted (we don't + bump the major version if .Internal modules change) + +* Spec compliance: toCaseFold now follows the Unicode 7.0 spec + (updated from 6.3) + +#### 1.2.0.4 + +* Fixed an incompatibility with base < 4.5 + +#### 1.2.0.3 + +* Update formatRealFloat to correspond to the definition in versions + of base newer than 4.5 (https://github.com/bos/text/issues/105) + +#### 1.2.0.2 + +* Bumped lower bound on deepseq to 1.4 for compatibility with the + upcoming GHC 7.10 + +#### 1.2.0.1 + +* Fixed a buffer overflow in rendering of large Integers + (https://github.com/bos/text/issues/99) + +## 1.2.0.0 + +* Fixed an integer overflow in the replace function + (https://github.com/bos/text/issues/81) + +* Fixed a hang in lazy decodeUtf8With + (https://github.com/bos/text/issues/87) + +* Reduced codegen bloat caused by use of empty and single-character + literals + +* Added an instance of IsList for GHC 7.8 and above + +### 1.1.1.0 + +* The Data.Data instance now allows gunfold to work, via a virtual + pack constructor + +* dropEnd, takeEnd: new functions + +* Comparing the length of a Text against a number can now + short-circuit in more cases + +#### 1.1.0.1 + +* streamDecodeUtf8: fixed gh-70, did not return all unconsumed bytes + in single-byte chunks + +## 1.1.0.0 + +* encodeUtf8: Performance is improved by up to 4x. + +* encodeUtf8Builder, encodeUtf8BuilderEscaped: new functions, + available only if bytestring >= 0.10.4.0 is installed, that allow + very fast and flexible encoding of a Text value to a bytestring + Builder. + + As an example of the performance gain to be had, the + encodeUtf8BuilderEscaped function helps to double the speed of JSON + encoding in the latest version of aeson! (Note: if all you need is a + plain ByteString, encodeUtf8 is still the faster way to go.) + +* All of the internal module hierarchy is now publicly exposed. If a + module is in the .Internal hierarchy, or is documented as internal, + use at your own risk - there are no API stability guarantees for + internal modules! + +#### 1.0.0.1 + +* decodeUtf8: Fixed a regression that caused us to incorrectly + identify truncated UTF-8 as valid (gh-61) + +# 1.0.0.0 + +* Added support for Unicode 6.3.0 to case conversion functions + +* New function toTitle converts words in a string to title case + +* New functions peekCStringLen and withCStringLen simplify + interoperability with C functions + +* Added support for decoding UTF-8 in stream-friendly fashion + +* Fixed a bug in mapAccumL + +* Added trusted Haskell support + +* Removed support for GHC 6.10 (released in 2008) and older
scripts/Arsec.hs view
@@ -1,49 +1,48 @@-module Arsec- (- Comment- , comment- , semi- , showC- , unichar- , unichars- , module Control.Applicative- , module Control.Monad- , module Data.Char- , module Text.ParserCombinators.Parsec.Char- , module Text.ParserCombinators.Parsec.Combinator- , module Text.ParserCombinators.Parsec.Error- , module Text.ParserCombinators.Parsec.Prim- ) where--import Prelude hiding (head, tail)-import Control.Monad-import Control.Applicative-import Data.Char-import Numeric (readHex, showHex)-import Text.ParserCombinators.Parsec.Char hiding (lower, upper)-import Text.ParserCombinators.Parsec.Combinator hiding (optional)-import Text.ParserCombinators.Parsec.Error-import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many)--type Comment = String--unichar :: Parser Char-unichar = do- digits <- many1 hexDigit- case readHex digits of- [] -> error "unichar: cannot parse hex digits"- (hd, _) : _ -> pure $ chr hd--unichars :: Parser [Char]-unichars = manyTill (unichar <* spaces) semi--semi :: Parser ()-semi = char ';' *> spaces *> pure ()--comment :: Parser Comment-comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n"--showC :: Char -> String-showC c = "'\\x" ++ d ++ "'"- where h = showHex (ord c) ""- d = replicate (4 - length h) '0' ++ h+module Arsec + ( + Comment + , comment + , semi + , showC + , unichar + , unichars + , module Control.Applicative + , module Control.Monad + , module Text.ParserCombinators.Parsec.Char + , module Text.ParserCombinators.Parsec.Combinator + , module Text.ParserCombinators.Parsec.Error + , module Text.ParserCombinators.Parsec.Prim + ) where + +import Prelude hiding (head, tail) +import Control.Monad +import Control.Applicative +import Data.Char +import Numeric (readHex, showHex) +import Text.ParserCombinators.Parsec.Char hiding (lower, upper) +import Text.ParserCombinators.Parsec.Combinator hiding (optional) +import Text.ParserCombinators.Parsec.Error +import Text.ParserCombinators.Parsec.Prim hiding ((<|>), many) + +type Comment = String + +unichar :: Parser Char +unichar = do + digits <- many1 hexDigit + case readHex digits of + [] -> error "unichar: cannot parse hex digits" + (hd, _) : _ -> pure $ chr hd + +unichars :: Parser [Char] +unichars = manyTill (unichar <* spaces) semi + +semi :: Parser () +semi = char ';' *> spaces *> pure () + +comment :: Parser Comment +comment = (char '#' *> manyTill anyToken (char '\n')) <|> string "\n" + +showC :: Char -> String +showC c = "'\\x" ++ d ++ "'" + where h = showHex (ord c) "" + d = replicate (4 - length h) '0' ++ h
scripts/CaseFolding.hs view
@@ -1,46 +1,47 @@--- This script processes the following source file:------ http://unicode.org/Public/UNIDATA/CaseFolding.txt--module CaseFolding- (- CaseFolding(..)- , Fold(..)- , parseCF- , mapCF- ) where--import Arsec-import Data.Bits--data Fold = Fold {- code :: Char- , status :: Char- , mapping :: [Char]- , name :: String- } deriving (Eq, Ord, Show)--data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] }- deriving (Show)--entries :: Parser CaseFolding-entries = CF <$> many comment <*> many (entry <* many comment)- where- entry = Fold <$> unichar <* semi- <*> oneOf "CFST" <* semi- <*> unichars- <*> (string "# " *> manyTill anyToken (char '\n'))--parseCF :: FilePath -> IO (Either ParseError CaseFolding)-parseCF name = parse entries name <$> readFile name--mapCF :: CaseFolding -> [String]-mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last]- where- typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}"- ,"{-# NOINLINE foldMapping #-}"- ,"foldMapping = \\case"]- last = " _ -> unI64 0"- printUnusual c = " -- " ++ name c ++ "\n" ++- " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))- where x:y:z:_ = mapping c ++ repeat '\0'+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/CaseFolding.txt + +module CaseFolding + ( + CaseFolding(..) + , Fold(..) + , parseCF + , mapCF + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data Fold = Fold { + code :: Char + , status :: Char + , mapping :: [Char] + , name :: String + } deriving (Eq, Ord, Show) + +data CaseFolding = CF { cfComments :: [Comment], cfFolding :: [Fold] } + deriving (Show) + +entries :: Parser CaseFolding +entries = CF <$> many comment <*> many (entry <* many comment) + where + entry = Fold <$> unichar <* semi + <*> oneOf "CFST" <* semi + <*> unichars + <*> (string "# " *> manyTill anyToken (char '\n')) + +parseCF :: FilePath -> IO (Either ParseError CaseFolding) +parseCF name = parse entries name <$> readFile name + +mapCF :: CaseFolding -> [String] +mapCF (CF _ ms) = typ ++ map printUnusual (filter (\f -> status f `elem` "CF") ms) ++ [last] + where + typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE foldMapping #-}" + ,"foldMapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = mapping c ++ repeat '\0'
scripts/CaseMapping.hs view
@@ -1,41 +1,66 @@-import System.Environment-import System.IO--import Arsec-import CaseFolding-import SpecialCasing--main = do- args <- getArgs- let oname = case args of- [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs"- [o] -> o- psc <- parseSC "SpecialCasing.txt"- pcf <- parseCF "CaseFolding.txt"- scs <- case psc of- Left err -> print err >> return undefined- Right ms -> return ms- cfs <- case pcf of- Left err -> print err >> return undefined- Right ms -> return ms- h <- openFile oname WriteMode- let comments = map ("--" ++) $- take 2 (cfComments cfs) ++ take 2 (scComments scs)- mapM_ (hPutStrLn h) $- ["-- AUTOMATICALLY GENERATED - DO NOT EDIT"- ,"-- Generated by scripts/CaseMapping.hs"] ++- comments ++- [""- ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}"- ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}"- ,"module Data.Text.Internal.Fusion.CaseMapping where"- ,"import GHC.Int"- ,"import GHC.Exts"- ,"unI64 :: Int64 -> _ {- unboxed Int64 -}"- ,"unI64 (I64# n) = n"- ,""]- mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs)- mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs)- mapM_ (hPutStrLn h) (mapSC "title" title toTitle scs)- mapM_ (hPutStrLn h) (mapCF cfs)- hClose h+import Data.Char (isDigit) +import Data.Foldable (toList) +import Data.List (stripPrefix) +import Data.Maybe (fromJust) +import System.Environment +import System.IO + +import Arsec +import CaseFolding +import SpecialCasing +import UnicodeData + +main = do + args <- getArgs + let oname = case args of + [] -> "../src/Data/Text/Internal/Fusion/CaseMapping.hs" + [o] -> o + psc <- parseSC "SpecialCasing.txt" + pcf <- parseCF "CaseFolding.txt" + ud <- parseUD "UnicodeData.txt" + scs <- case psc of + Left err -> print err >> return undefined + Right ms -> return ms + cfs <- case pcf of + Left err -> print err >> return undefined + Right ms -> return ms + ud <- case ud of + Left err -> print err >> return undefined + Right ms -> return ms + h <- openFile oname WriteMode + let comments = map ("--" ++) $ + take 2 (cfComments cfs) ++ take 2 (scComments scs) + version = parseVersion (cfComments cfs) + mapM_ (hPutStrLn h) $ + ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ,"-- Generated by scripts/CaseMapping.hs"] ++ + comments ++ + ["" + ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" + ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" + ,"module Data.Text.Internal.Fusion.CaseMapping where" + ,"import GHC.Int" + ,"import GHC.Exts" + ,"import Data.Version (Version, makeVersion)" + ,"unicodeVersion :: Version" + ,"unicodeVersion = makeVersion " ++ version + ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" + ,"unI64 (I64# n) = n" + ,""] + let get f = [(k, d) | c <- toList ud, Just d <- [f c], let k = charUD c, k /= d] + mapM_ (hPutStrLn h) (mapSC "upper" upper (get toUpperUD) scs) + mapM_ (hPutStrLn h) (mapSC "lower" lower (get toLowerUD) scs) + mapM_ (hPutStrLn h) (mapSC "title" title (get toTitleUD) scs) + mapM_ (hPutStrLn h) (mapCF cfs) + hClose h + +-- Parse version from CaseFolding comments +-- and render it as a list (an argument of makeVersion) +parseVersion :: [String] -> String +parseVersion comments = fromJust $ do + line' : _ <- pure comments + line'' <- stripPrefix " CaseFolding-" line' + let (v1, line1) = span isDigit line'' + (v2, line2) = span isDigit (drop 1 line1) + (v3, _) = span isDigit (drop 1 line2) + pure $ "[" ++ v1 ++ ", " ++ v2 ++ ", " ++ v3 ++ "]"
scripts/SpecialCasing.hs view
@@ -1,63 +1,61 @@--- This script processes the following source file:------ http://unicode.org/Public/UNIDATA/SpecialCasing.txt--module SpecialCasing- (- SpecialCasing(..)- , Case(..)- , parseSC- , mapSC- ) where--import Arsec-import Data.Bits--data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] }- deriving (Show)--data Case = Case {- code :: Char- , lower :: [Char]- , title :: [Char]- , upper :: [Char]- , conditions :: String- , name :: String- } deriving (Eq, Ord, Show)--entries :: Parser SpecialCasing-entries = SC <$> many comment <*> many (entry <* many comment)- where- entry = Case <$> unichar <* semi- <*> unichars- <*> unichars- <*> unichars- <*> manyTill anyToken (string "# ")- <*> manyTill anyToken (char '\n')--parseSC :: FilePath -> IO (Either ParseError SpecialCasing)-parseSC name = parse entries name <$> readFile name--mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing- -> [String]-mapSC which access twiddle (SC _ ms) =- typ ++ map printUnusual ms' ++ map printUsual usual ++ [last]- where- ms' = filter p ms- p c = [k] /= a && a /= [twiddle k] && null (conditions c)- where a = access c- k = code c- unusual = map code ms'- usual = filter (\c -> twiddle c /= c && c `notElem` unusual) [minBound..maxBound]-- typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}"- ,"{-# NOINLINE " ++ which ++ "Mapping #-}"- ,which ++ "Mapping = \\case"]- last = " _ -> unI64 0"- printUnusual c = " -- " ++ name c ++ "\n" ++- " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42))- where x:y:z:_ = access c ++ repeat '\0'- printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (twiddle c))--ucFirst (c:cs) = toUpper c : cs-ucFirst [] = []+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/SpecialCasing.txt + +module SpecialCasing + ( + SpecialCasing(..) + , Case(..) + , parseSC + , mapSC + ) where + +import Arsec +import Data.Bits +import Data.Char (ord) + +data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } + deriving (Show) + +data Case = Case { + code :: Char + , lower :: [Char] + , title :: [Char] + , upper :: [Char] + , conditions :: String + , name :: String + } deriving (Eq, Ord, Show) + +entries :: Parser SpecialCasing +entries = SC <$> many comment <*> many (entry <* many comment) + where + entry = Case <$> unichar <* semi + <*> unichars + <*> unichars + <*> unichars + <*> manyTill anyToken (string "# ") + <*> manyTill anyToken (char '\n') + +parseSC :: FilePath -> IO (Either ParseError SpecialCasing) +parseSC name = parse entries name <$> readFile name + +mapSC :: String -> (Case -> String) -> [(Char, Char)] -> SpecialCasing + -> [String] +mapSC which access twiddle (SC _ ms) = + typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] + where + ms' = filter p ms + p c = [k] /= a && null (conditions c) + where a = access c + k = code c + unusual = map code ms' + usual = filter (\(c, _) -> c `notElem` unusual) twiddle + + typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE " ++ which ++ "Mapping #-}" + ,which ++ "Mapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = access c ++ repeat '\0' + printUsual (c, c') = " " ++ showC c ++ "# -> unI64 " ++ show (ord c')
+ scripts/UnicodeData.hs view
@@ -0,0 +1,51 @@+-- This script processes the following source file: +-- +-- http://unicode.org/Public/UNIDATA/UnicodeData.txt +-- +-- Format description: https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt + +module UnicodeData + ( UnicodeData + , Data(..) + , toTitleUD + , parseUD + ) where + +import Debug.Trace +import Arsec hiding (semi) +import Data.Array +import Data.Functor (void) +import Data.List (sort) +import Data.Maybe (fromMaybe) + +type UnicodeData = Array Int Data + +-- "Simple_Titlecase_Mapping: If this field is null, then the Simple_Titlecase_Mapping +-- is the same as the Simple_Uppercase_Mapping for this character." +-- -- https://www.unicode.org/reports/tr44/tr44-36.html#UnicodeData.txt +toTitleUD :: Data -> Maybe Char +toTitleUD d = toTitleUD_ d <|> toUpperUD d + +data Data = Data { + charUD :: {-# UNPACK #-} !Char + , toUpperUD :: {-# UNPACK #-} !(Maybe Char) + , toLowerUD :: {-# UNPACK #-} !(Maybe Char) + , toTitleUD_ :: {-# UNPACK #-} !(Maybe Char) + } deriving (Eq, Ord, Show) + +-- I'm pretty sure UnicodeData.txt is sorted but still sort it to be 100% certain. +entries :: Parser UnicodeData +entries = (\xs -> listArray (0, length xs - 1) xs) <$> many entry <* eof + where + entry = Data <$> unichar <* semi + <* replicateM_ 11 (ignoreField <* semi) + <*> optional unichar <* semi + <*> optional unichar <* semi + <*> optional unichar <* char '\n' + semi = char ';' + +ignoreField :: Parser () +ignoreField = void (many (satisfy (\c -> c /= ';'))) + +parseUD :: FilePath -> IO (Either ParseError UnicodeData) +parseUD name = parse entries name <$> readFile name
+ simdutf/hs_simdutf.c view
@@ -0,0 +1,9 @@+#include "simdutf_c.h" + +int _hs_text_is_valid_utf8(const char *buf, size_t len) { + return simdutf_validate_utf8(buf, len); +} + +int _hs_text_is_valid_utf8_offset(const char *buf, size_t off, size_t len) { + return simdutf_validate_utf8(buf + off, len); +}
simdutf/simdutf.cpp view
file too large to diff
simdutf/simdutf.h view
@@ -1,3757 +1,13573 @@-/* auto-generated on 2024-05-07 22:33:11 -0400. Do not edit! */-/* begin file include/simdutf.h */-#ifndef SIMDUTF_H-#define SIMDUTF_H-#include <cstring>--/* begin file include/simdutf/compiler_check.h */-#ifndef SIMDUTF_COMPILER_CHECK_H-#define SIMDUTF_COMPILER_CHECK_H--#ifndef __cplusplus-#error simdutf requires a C++ compiler-#endif--#ifndef SIMDUTF_CPLUSPLUS-#if defined(_MSVC_LANG) && !defined(__clang__)-#define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)-#else-#define SIMDUTF_CPLUSPLUS __cplusplus-#endif-#endif--// C++ 17-#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L)-#define SIMDUTF_CPLUSPLUS17 1-#endif--// C++ 14-#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L)-#define SIMDUTF_CPLUSPLUS14 1-#endif--// C++ 11-#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L)-#define SIMDUTF_CPLUSPLUS11 1-#endif--#ifndef SIMDUTF_CPLUSPLUS11-#error simdutf requires a compiler compliant with the C++11 standard-#endif--#endif // SIMDUTF_COMPILER_CHECK_H-/* end file include/simdutf/compiler_check.h */-/* begin file include/simdutf/common_defs.h */-#ifndef SIMDUTF_COMMON_DEFS_H-#define SIMDUTF_COMMON_DEFS_H--#include <cassert>-/* begin file include/simdutf/portability.h */-#ifndef SIMDUTF_PORTABILITY_H-#define SIMDUTF_PORTABILITY_H--#include <cstddef>-#include <cstdint>-#include <cstdlib>-#include <cfloat>-#include <cassert>-#ifndef _WIN32-// strcasecmp, strncasecmp-#include <strings.h>-#endif--/**- * We want to check that it is actually a little endian system at- * compile-time.- */--#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)-#define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)-#elif defined(_WIN32)-#define SIMDUTF_IS_BIG_ENDIAN 0-#else-#if defined(__APPLE__) || defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__-#include <machine/endian.h>-#elif defined(sun) || defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__)-#include <sys/byteorder.h>-#else // defined(__APPLE__) || defined(__FreeBSD__)--#ifdef __has_include-#if __has_include(<endian.h>)-#include <endian.h>-#endif //__has_include(<endian.h>)-#endif //__has_include--#endif // defined(__APPLE__) || defined(__FreeBSD__)---#ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__)-#define SIMDUTF_IS_BIG_ENDIAN 0-#endif--#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__-#define SIMDUTF_IS_BIG_ENDIAN 0-#else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__-#define SIMDUTF_IS_BIG_ENDIAN 1-#endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__--#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__---/**- * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined.- */--#ifdef _MSC_VER-#define SIMDUTF_VISUAL_STUDIO 1-/**- * We want to differentiate carefully between- * clang under visual studio and regular visual- * studio.- *- * Under clang for Windows, we enable:- * * target pragmas so that part and only part of the- * code gets compiled for advanced instructions.- *- */-#ifdef __clang__-// clang under visual studio-#define SIMDUTF_CLANG_VISUAL_STUDIO 1-#else-// just regular visual studio (best guess)-#define SIMDUTF_REGULAR_VISUAL_STUDIO 1-#endif // __clang__-#endif // _MSC_VER--#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO-// https://en.wikipedia.org/wiki/C_alternative_tokens-// This header should have no effect, except maybe-// under Visual Studio.-#include <iso646.h>-#endif--#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC)-#define SIMDUTF_IS_X86_64 1-#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)-#define SIMDUTF_IS_ARM64 1-#elif defined(__PPC64__) || defined(_M_PPC64)-//#define SIMDUTF_IS_PPC64 1-// The simdutf library does yet support SIMD acceleration under-// POWER processors. Please see https://github.com/lemire/simdutf/issues/51-#elif defined(__s390__)-// s390 IBM system. Big endian.-#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64-// RISC-V 64-bit-#define SIMDUTF_IS_RISCV64 1--#if __clang_major__ >= 19-// Does the compiler support target regions for RISC-V-#define SIMDUTF_HAS_RVV_TARGET_REGION 1-#endif--#if __riscv_v_intrinsic >= 11000-#define SIMDUTF_HAS_RVV_INTRINSICS 1-#endif--#define SIMDUTF_HAS_ZVBB_INTRINSICS 0 // there is currently no way to detect this--#if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector && __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64-// RISC-V V extension-#define SIMDUTF_IS_RVV 1-#if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000-// RISC-V Vector Basic Bit-manipulation-#define SIMDUTF_IS_ZVBB 1-#endif-#endif--#elif defined(__loongarch_lp64)-// LoongArch 64-bit-#else-// The simdutf library is designed-// for 64-bit processors and it seems that you are not-// compiling for a known 64-bit platform. Please-// use a 64-bit target such as x64 or 64-bit ARM for best performance.-#define SIMDUTF_IS_32BITS 1--// We do not support 32-bit platforms, but it can be-// handy to identify them.-#if defined(_M_IX86) || defined(__i386__)-#define SIMDUTF_IS_X86_32BITS 1-#elif defined(__arm__) || defined(_M_ARM)-#define SIMDUTF_IS_ARM_32BITS 1-#elif defined(__PPC__) || defined(_M_PPC)-#define SIMDUTF_IS_PPC_32BITS 1-#endif--#endif // defined(__x86_64__) || defined(_M_AMD64)--#ifdef SIMDUTF_IS_32BITS-#ifndef SIMDUTF_NO_PORTABILITY_WARNING-// In the future, we may want to warn users of 32-bit systems that-// the simdutf does not support accelerated kernels for such systems.-#endif // SIMDUTF_NO_PORTABILITY_WARNING-#endif // SIMDUTF_IS_32BITS--// this is almost standard?-#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a-#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a)--// Our fast kernels require 64-bit systems.-//-// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions.-// Furthermore, the number of SIMD registers is reduced.-//-// On 32-bit ARM, we would have smaller registers.-//-// The simdutf users should still have the fallback kernel. It is-// slower, but it should run everywhere.--//-// Enable valid runtime implementations, and select SIMDUTF_BUILTIN_IMPLEMENTATION-//--// We are going to use runtime dispatch.-#ifdef SIMDUTF_IS_X86_64-#ifdef __clang__-// clang does not have GCC push pop-// warning: clang attribute push can't be used within a namespace in clang up-// til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be *outside* of a-// namespace.-#define SIMDUTF_TARGET_REGION(T) \- _Pragma(SIMDUTF_STRINGIFY( \- clang attribute push(__attribute__((target(T))), apply_to = function)))-#define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop")-#elif defined(__GNUC__)-// GCC is easier-#define SIMDUTF_TARGET_REGION(T) \- _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T)))-#define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options")-#endif // clang then gcc--#endif // x86--// Default target region macros don't do anything.-#ifndef SIMDUTF_TARGET_REGION-#define SIMDUTF_TARGET_REGION(T)-#define SIMDUTF_UNTARGET_REGION-#endif--// Is threading enabled?-#if defined(_REENTRANT) || defined(_MT)-#ifndef SIMDUTF_THREADS_ENABLED-#define SIMDUTF_THREADS_ENABLED-#endif-#endif--// workaround for large stack sizes under -O0.-// https://github.com/simdutf/simdutf/issues/691-#ifdef __APPLE__-#ifndef __OPTIMIZE__-// Apple systems have small stack sizes in secondary threads.-// Lack of compiler optimization may generate high stack usage.-// Users may want to disable threads for safety, but only when-// in debug mode which we detect by the fact that the __OPTIMIZE__-// macro is not defined.-#undef SIMDUTF_THREADS_ENABLED-#endif-#endif--#ifdef SIMDUTF_VISUAL_STUDIO-// This is one case where we do not distinguish between-// regular visual studio and clang under visual studio.-// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has)-#define simdutf_strcasecmp _stricmp-#define simdutf_strncasecmp _strnicmp-#else-// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8).-// So they are only useful for ASCII in our context.-// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings-#define simdutf_strcasecmp strcasecmp-#define simdutf_strncasecmp strncasecmp-#endif--#ifdef NDEBUG--#ifdef SIMDUTF_VISUAL_STUDIO-#define SIMDUTF_UNREACHABLE() __assume(0)-#define SIMDUTF_ASSUME(COND) __assume(COND)-#else-#define SIMDUTF_UNREACHABLE() __builtin_unreachable();-#define SIMDUTF_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0)-#endif--#else // NDEBUG--#define SIMDUTF_UNREACHABLE() assert(0);-#define SIMDUTF_ASSUME(COND) assert(COND)--#endif---#if defined(__GNUC__) && !defined(__clang__)-#if __GNUC__ >= 11-#define SIMDUTF_GCC11ORMORE 1-#endif // __GNUC__ >= 11-#endif // defined(__GNUC__) && !defined(__clang__)---#endif // SIMDUTF_PORTABILITY_H-/* end file include/simdutf/portability.h */-/* begin file include/simdutf/avx512.h */-#ifndef SIMDUTF_AVX512_H_-#define SIMDUTF_AVX512_H_--/*- It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS.-- All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`,- where a feature is a code name for extensions.-- Please see the listing below to find which are supported.-*/--#ifndef SIMDUTF_HAS_AVX512F-# if defined(__AVX512F__) && __AVX512F__ == 1-# define SIMDUTF_HAS_AVX512F 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512DQ-# if defined(__AVX512DQ__) && __AVX512DQ__ == 1-# define SIMDUTF_HAS_AVX512DQ 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512IFMA-# if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1-# define SIMDUTF_HAS_AVX512IFMA 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512CD-# if defined(__AVX512CD__) && __AVX512CD__ == 1-# define SIMDUTF_HAS_AVX512CD 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512BW-# if defined(__AVX512BW__) && __AVX512BW__ == 1-# define SIMDUTF_HAS_AVX512BW 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512VL-# if defined(__AVX512VL__) && __AVX512VL__ == 1-# define SIMDUTF_HAS_AVX512VL 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512VBMI-# if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1-# define SIMDUTF_HAS_AVX512VBMI 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512VBMI2-# if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1-# define SIMDUTF_HAS_AVX512VBMI2 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512VNNI-# if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1-# define SIMDUTF_HAS_AVX512VNNI 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512BITALG-# if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1-# define SIMDUTF_HAS_AVX512BITALG 1-# endif-#endif--#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ-# if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1-# define SIMDUTF_HAS_AVX512VPOPCNTDQ 1-# endif-#endif--#endif // SIMDUTF_AVX512_H_-/* end file include/simdutf/avx512.h */---#if defined(__GNUC__)- // Marks a block with a name so that MCA analysis can see it.- #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);- #define SIMDUTF_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);- #define SIMDUTF_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);-#else- #define SIMDUTF_BEGIN_DEBUG_BLOCK(name)- #define SIMDUTF_END_DEBUG_BLOCK(name)- #define SIMDUTF_DEBUG_BLOCK(name, block)-#endif--// Align to N-byte boundary-#define SIMDUTF_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))-#define SIMDUTF_ROUNDDOWN_N(a, n) ((a) & ~((n)-1))--#define SIMDUTF_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)--#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO)-- #define simdutf_really_inline __forceinline- #define simdutf_never_inline __declspec(noinline)-- #define simdutf_unused- #define simdutf_warn_unused-- #ifndef simdutf_likely- #define simdutf_likely(x) x- #endif- #ifndef simdutf_unlikely- #define simdutf_unlikely(x) x- #endif-- #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning( push ))- #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 ))- #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER ))- // Get rid of Intellisense-only warnings (Code Analysis)- // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910).- #ifdef __has_include- #if __has_include(<CppCoreCheck\Warnings.h>)- #include <CppCoreCheck\Warnings.h>- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS)- #endif- #endif-- #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #endif-- #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996)- #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING- #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning( pop ))--#else // SIMDUTF_REGULAR_VISUAL_STUDIO-- #define simdutf_really_inline inline __attribute__((always_inline))- #define simdutf_never_inline inline __attribute__((noinline))-- #define simdutf_unused __attribute__((unused))- #define simdutf_warn_unused __attribute__((warn_unused_result))-- #ifndef simdutf_likely- #define simdutf_likely(x) __builtin_expect(!!(x), 1)- #endif- #ifndef simdutf_unlikely- #define simdutf_unlikely(x) __builtin_expect(!!(x), 0)- #endif-- #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")- // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary- #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS SIMDUTF_PUSH_DISABLE_WARNINGS \- SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \- SIMDUTF_DISABLE_GCC_WARNING(-Wall) \- SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \- SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \- SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \- SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \- SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \- SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \- SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \- SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \- SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable)- #define SIMDUTF_PRAGMA(P) _Pragma(#P)- #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING)- #if defined(SIMDUTF_CLANG_VISUAL_STUDIO)- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include)- #else- #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS- #endif- #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations)- #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow)- #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")----#endif // MSC_VER--#ifndef SIMDUTF_DLLIMPORTEXPORT- #if defined(SIMDUTF_VISUAL_STUDIO)- /**- * It does not matter here whether you are using- * the regular visual studio or clang under visual- * studio.- */- #if SIMDUTF_USING_LIBRARY- #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport)- #else- #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport)- #endif- #else- #define SIMDUTF_DLLIMPORTEXPORT- #endif-#endif--/// If EXPR is an error, returns it.-#define SIMDUTF_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } }---#endif // SIMDUTF_COMMON_DEFS_H-/* end file include/simdutf/common_defs.h */-/* begin file include/simdutf/encoding_types.h */-#include <string>--namespace simdutf {--enum encoding_type {- UTF8 = 1, // BOM 0xef 0xbb 0xbf- UTF16_LE = 2, // BOM 0xff 0xfe- UTF16_BE = 4, // BOM 0xfe 0xff- UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00- UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff- Latin1 = 32,-- unspecified = 0-};--enum endianness {- LITTLE = 0,- BIG = 1-};--bool match_system(endianness e);--std::string to_string(encoding_type bom);--// Note that BOM for UTF8 is discouraged.-namespace BOM {--/**- * Checks for a BOM. If not, returns unspecified- * @param input the string to process- * @param length the length of the string in code units- * @return the corresponding encoding- */--encoding_type check_bom(const uint8_t* byte, size_t length);-encoding_type check_bom(const char* byte, size_t length);-/**- * Returns the size, in bytes, of the BOM for a given encoding type.- * Note that UTF8 BOM are discouraged.- * @param bom the encoding type- * @return the size in bytes of the corresponding BOM- */-size_t bom_byte_size(encoding_type bom);--} // BOM namespace-} // simdutf namespace-/* end file include/simdutf/encoding_types.h */-/* begin file include/simdutf/error.h */-#ifndef SIMDUTF_ERROR_H-#define SIMDUTF_ERROR_H-namespace simdutf {--enum error_code {- SUCCESS = 0,- HEADER_BITS, // Any byte must have fewer than 5 header bits.- TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, where N is the UTF-8 character length- // This is also the error when the input is truncated.- TOO_LONG, // We either have too many consecutive continuation bytes or the string starts with a continuation byte.- OVERLONG, // The decoded character must be above U+7F for two-byte characters, U+7FF for three-byte characters,- // and U+FFFF for four-byte characters.- TOO_LARGE, // The decoded character must be less than or equal to U+10FFFF,less than or equal than U+7F for ASCII OR less than equal than U+FF for Latin1- SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or UTF-32) OR- // a high surrogate must be followed by a low surrogate and a low surrogate must be preceded by a high surrogate (UTF-16) OR- // there must be no surrogate at all (Latin1)- INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid base64 string.- BASE64_INPUT_REMAINDER, // The base64 input terminates with a single character, excluding padding (=).- OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small.- OTHER // Not related to validation/transcoding.-};--struct result {- error_code error;- size_t count; // In case of error, indicates the position of the error. In case of success, indicates the number of code units validated/written.-- simdutf_really_inline result() : error{error_code::SUCCESS}, count{0} {}-- simdutf_really_inline result(error_code _err, size_t _pos) : error{_err}, count{_pos} {}-};--}-#endif-/* end file include/simdutf/error.h */--SIMDUTF_PUSH_DISABLE_WARNINGS-SIMDUTF_DISABLE_UNDESIRED_WARNINGS--// Public API-/* begin file include/simdutf/simdutf_version.h */-// /include/simdutf/simdutf_version.h automatically generated by release.py,-// do not change by hand-#ifndef SIMDUTF_SIMDUTF_VERSION_H-#define SIMDUTF_SIMDUTF_VERSION_H--/** The version of simdutf being used (major.minor.revision) */-#define SIMDUTF_VERSION "5.2.8"--namespace simdutf {-enum {- /**- * The major version (MAJOR.minor.revision) of simdutf being used.- */- SIMDUTF_VERSION_MAJOR = 5,- /**- * The minor version (major.MINOR.revision) of simdutf being used.- */- SIMDUTF_VERSION_MINOR = 2,- /**- * The revision (major.minor.REVISION) of simdutf being used.- */- SIMDUTF_VERSION_REVISION = 8-};-} // namespace simdutf--#endif // SIMDUTF_SIMDUTF_VERSION_H-/* end file include/simdutf/simdutf_version.h */-/* begin file include/simdutf/implementation.h */-#ifndef SIMDUTF_IMPLEMENTATION_H-#define SIMDUTF_IMPLEMENTATION_H-#include <string>-#if !defined(SIMDUTF_NO_THREADS)-#include <atomic>-#endif-#include <vector>-#include <tuple>-/* begin file include/simdutf/internal/isadetection.h */-/* From-https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h-Highly modified.--Copyright (c) 2016- Facebook, Inc (Adam Paszke)-Copyright (c) 2014- Facebook, Inc (Soumith Chintala)-Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)-Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)-Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)-Copyright (c) 2011-2013 NYU (Clement Farabet)-Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,-Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute-(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,-Samy Bengio, Johnny Mariethoz)--All rights reserved.--Redistribution and use in source and binary forms, with or without-modification, are permitted provided that the following conditions are met:--1. Redistributions of source code must retain the above copyright- notice, this list of conditions and the following disclaimer.--2. Redistributions in binary form must reproduce the above copyright- notice, this list of conditions and the following disclaimer in the- documentation and/or other materials provided with the distribution.--3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories-America and IDIAP Research Institute nor the names of its contributors may be- used to endorse or promote products derived from this software without- specific prior written permission.--THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE-POSSIBILITY OF SUCH DAMAGE.-*/--#ifndef SIMDutf_INTERNAL_ISADETECTION_H-#define SIMDutf_INTERNAL_ISADETECTION_H--#include <cstdint>-#include <cstdlib>-#if defined(_MSC_VER)-#include <intrin.h>-#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)-#include <cpuid.h>-#endif---// RISC-V ISA detection utilities-#if SIMDUTF_IS_RISCV64 && defined(__linux__)-#include <unistd.h> // for syscall-// We define these ourselves, for backwards compatibility-struct simdutf_riscv_hwprobe { int64_t key; uint64_t value; };-#define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__)-#define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4-#define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2)-#define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17)-#endif // SIMDUTF_IS_RISCV64 && defined(__linux__)--namespace simdutf {-namespace internal {--enum instruction_set {- DEFAULT = 0x0,- NEON = 0x1,- AVX2 = 0x4,- SSE42 = 0x8,- PCLMULQDQ = 0x10,- BMI1 = 0x20,- BMI2 = 0x40,- ALTIVEC = 0x80,- AVX512F = 0x100,- AVX512DQ = 0x200,- AVX512IFMA = 0x400,- AVX512PF = 0x800,- AVX512ER = 0x1000,- AVX512CD = 0x2000,- AVX512BW = 0x4000,- AVX512VL = 0x8000,- AVX512VBMI2 = 0x10000,- AVX512VPOPCNTDQ = 0x2000,- RVV = 0x4000,- ZVBB = 0x8000,-};--#if defined(__PPC64__)--static inline uint32_t detect_supported_architectures() {- return instruction_set::ALTIVEC;-}--#elif SIMDUTF_IS_RISCV64--static inline uint32_t detect_supported_architectures() {- uint32_t host_isa = instruction_set::DEFAULT;-#if SIMDUTF_IS_RVV- host_isa |= instruction_set::RVV;-#endif-#if SIMDUTF_IS_ZVBB- host_isa |= instruction_set::ZVBB;-#endif-#if defined(__linux__)- simdutf_riscv_hwprobe probes[] = { { SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0 } };- long ret = simdutf_riscv_hwprobe(&probes, sizeof probes/sizeof *probes, 0, nullptr, 0);- if (ret == 0) {- uint64_t extensions = probes[0].value;- if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V)- host_isa |= instruction_set::RVV;- if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB)- host_isa |= instruction_set::ZVBB;- }-#endif-#if defined(RUN_IN_SPIKE_SIMULATOR)- // Proxy Kernel does not implement yet hwprobe syscall- host_isa |= instruction_set::RVV;-#endif- return host_isa;-}--#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)--static inline uint32_t detect_supported_architectures() {- return instruction_set::NEON;-}--#elif defined(__x86_64__) || defined(_M_AMD64) // x64---namespace {-namespace cpuid_bit {- // Can be found on Intel ISA Reference for CPUID-- // EAX = 0x01- constexpr uint32_t pclmulqdq = uint32_t(1) << 1; ///< @private bit 1 of ECX for EAX=0x1- constexpr uint32_t sse42 = uint32_t(1) << 20; ///< @private bit 20 of ECX for EAX=0x1- constexpr uint32_t osxsave = (uint32_t(1) << 26) | (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1-- // EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf)- // See: "Table 3-8. Information Returned by CPUID Instruction"- namespace ebx {- constexpr uint32_t bmi1 = uint32_t(1) << 3;- constexpr uint32_t avx2 = uint32_t(1) << 5;- constexpr uint32_t bmi2 = uint32_t(1) << 8;- constexpr uint32_t avx512f = uint32_t(1) << 16;- constexpr uint32_t avx512dq = uint32_t(1) << 17;- constexpr uint32_t avx512ifma = uint32_t(1) << 21;- constexpr uint32_t avx512cd = uint32_t(1) << 28;- constexpr uint32_t avx512bw = uint32_t(1) << 30;- constexpr uint32_t avx512vl = uint32_t(1) << 31;- }-- namespace ecx {- constexpr uint32_t avx512vbmi = uint32_t(1) << 1;- constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6;- constexpr uint32_t avx512vnni = uint32_t(1) << 11;- constexpr uint32_t avx512bitalg = uint32_t(1) << 12;- constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14;- }- namespace edx {- constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8;- }- namespace xcr0_bit {- constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX- constexpr uint64_t avx512_saved = uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM- }- }-}----static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,- uint32_t *edx) {-#if defined(_MSC_VER)- int cpu_info[4];- __cpuidex(cpu_info, *eax, *ecx);- *eax = cpu_info[0];- *ebx = cpu_info[1];- *ecx = cpu_info[2];- *edx = cpu_info[3];-#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)- uint32_t level = *eax;- __get_cpuid(level, eax, ebx, ecx, edx);-#else- uint32_t a = *eax, b, c = *ecx, d;- asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));- *eax = a;- *ebx = b;- *ecx = c;- *edx = d;-#endif-}--static inline uint64_t xgetbv() {- #if defined(_MSC_VER)- return _xgetbv(0);- #else- uint32_t xcr0_lo, xcr0_hi;- asm volatile("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));- return xcr0_lo | ((uint64_t)xcr0_hi << 32);- #endif- }--static inline uint32_t detect_supported_architectures() {- uint32_t eax;- uint32_t ebx = 0;- uint32_t ecx = 0;- uint32_t edx = 0;- uint32_t host_isa = 0x0;-- // EBX for EAX=0x1- eax = 0x1;- cpuid(&eax, &ebx, &ecx, &edx);-- if (ecx & cpuid_bit::sse42) {- host_isa |= instruction_set::SSE42;- }-- if (ecx & cpuid_bit::pclmulqdq) {- host_isa |= instruction_set::PCLMULQDQ;- }-- if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) {- return host_isa;- }-- // xgetbv for checking if the OS saves registers- uint64_t xcr0 = xgetbv();-- if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) {- return host_isa;- }- // ECX for EAX=0x7- eax = 0x7;- ecx = 0x0; // Sub-leaf = 0- cpuid(&eax, &ebx, &ecx, &edx);- if (ebx & cpuid_bit::ebx::avx2) {- host_isa |= instruction_set::AVX2;- }- if (ebx & cpuid_bit::ebx::bmi1) {- host_isa |= instruction_set::BMI1;- }- if (ebx & cpuid_bit::ebx::bmi2) {- host_isa |= instruction_set::BMI2;- }- if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == cpuid_bit::xcr0_bit::avx512_saved)) {- return host_isa;- }- if (ebx & cpuid_bit::ebx::avx512f) {- host_isa |= instruction_set::AVX512F;- }- if (ebx & cpuid_bit::ebx::avx512bw) {- host_isa |= instruction_set::AVX512BW;- }- if (ebx & cpuid_bit::ebx::avx512cd) {- host_isa |= instruction_set::AVX512CD;- }- if (ebx & cpuid_bit::ebx::avx512dq) {- host_isa |= instruction_set::AVX512DQ;- }- if (ebx & cpuid_bit::ebx::avx512vl) {- host_isa |= instruction_set::AVX512VL;- }- if (ecx & cpuid_bit::ecx::avx512vbmi2) {- host_isa |= instruction_set::AVX512VBMI2;- }- if (ecx & cpuid_bit::ecx::avx512vpopcnt) {- host_isa |= instruction_set::AVX512VPOPCNTDQ;- }- return host_isa;-}-#else // fallback--// includes 32-bit ARM.-static inline uint32_t detect_supported_architectures() {- return instruction_set::DEFAULT;-}---#endif // end SIMD extension detection code--} // namespace internal-} // namespace simdutf--#endif // SIMDutf_INTERNAL_ISADETECTION_H-/* end file include/simdutf/internal/isadetection.h */---namespace simdutf {--/**- * Autodetect the encoding of the input, a single encoding is recommended.- * E.g., the function might return simdutf::encoding_type::UTF8,- * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or- * simdutf::encoding_type::UTF32_LE.- *- * @param input the string to analyze.- * @param length the length of the string in bytes.- * @return the detected encoding type- */-simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * input, size_t length) noexcept;-simdutf_really_inline simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const uint8_t * input, size_t length) noexcept {- return autodetect_encoding(reinterpret_cast<const char *>(input), length);-}--/**- * Autodetect the possible encodings of the input in one pass.- * E.g., if the input might be UTF-16LE or UTF-8, this function returns- * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE).- *- * Overridden by each implementation.- *- * @param input the string to analyze.- * @param length the length of the string in bytes.- * @return the detected encoding type- */-simdutf_warn_unused int detect_encodings(const char * input, size_t length) noexcept;-simdutf_really_inline simdutf_warn_unused int detect_encodings(const uint8_t * input, size_t length) noexcept {- return detect_encodings(reinterpret_cast<const char *>(input), length);-}--/**- * Validate the UTF-8 string. This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf8_with_errors.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid UTF-8.- */-simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept;--/**- * Validate the UTF-8 string and stop on error.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_utf8_with_errors(const char *buf, size_t len) noexcept;--/**- * Validate the ASCII string.- *- * Overridden by each implementation.- *- * @param buf the ASCII string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid ASCII.- */-simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept;--/**- * Validate the ASCII string and stop on error. It might be faster than- * validate_utf8 when an error is expected to occur early.- *- * Overridden by each implementation.- *- * @param buf the ASCII string to validate.- * @param len the length of the string in bytes.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_ascii_with_errors(const char *buf, size_t len) noexcept;--/**- * Using native endianness; Validate the UTF-16 string.- * This function may be best when you expect the input to be almost always valid.- * Otherwise, consider using validate_utf16_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16 string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return true if and only if the string is valid UTF-16.- */-simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept;--/**- * Validate the UTF-16LE string. This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf16le_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return true if and only if the string is valid UTF-16LE.- */-simdutf_warn_unused bool validate_utf16le(const char16_t *buf, size_t len) noexcept;--/**- * Validate the UTF-16BE string. This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf16be_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16BE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return true if and only if the string is valid UTF-16BE.- */-simdutf_warn_unused bool validate_utf16be(const char16_t *buf, size_t len) noexcept;--/**- * Using native endianness; Validate the UTF-16 string and stop on error.- * It might be faster than validate_utf16 when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16 string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, size_t len) noexcept;--/**- * Validate the UTF-16LE string and stop on error. It might be faster than- * validate_utf16le when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, size_t len) noexcept;--/**- * Validate the UTF-16BE string and stop on error. It might be faster than- * validate_utf16be when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16BE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, size_t len) noexcept;--/**- * Validate the UTF-32 string. This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf32_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-32 string to validate.- * @param len the length of the string in number of 4-byte code units (char32_t).- * @return true if and only if the string is valid UTF-32.- */-simdutf_warn_unused bool validate_utf32(const char32_t *buf, size_t len) noexcept;--/**- * Validate the UTF-32 string and stop on error. It might be faster than- * validate_utf32 when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-32 string to validate.- * @param len the length of the string in number of 4-byte code units (char32_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */-simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, size_t len) noexcept;-- /**- * Convert Latin1 string into UTF8 string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if conversion is not possible- */- simdutf_warn_unused size_t convert_latin1_to_utf8(const char * input, size_t length, char* utf8_output) noexcept;--- /**- * Convert possibly Latin1 string into UTF-16LE string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if conversion is not possible- */- simdutf_warn_unused size_t convert_latin1_to_utf16le(const char * input, size_t length, char16_t* utf16_output) noexcept;-- /**- * Convert Latin1 string into UTF-16BE string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if conversion is not possible- */- simdutf_warn_unused size_t convert_latin1_to_utf16be(const char * input, size_t length, char16_t* utf16_output) noexcept;-- /**- * Convert Latin1 string into UTF-32 string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return the number of written char32_t; 0 if conversion is not possible- */- simdutf_warn_unused size_t convert_latin1_to_utf32(const char * input, size_t length, char32_t* utf32_buffer) noexcept;-- /**- * Convert possibly broken UTF-8 string into latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if the input was not valid UTF-8 string or if it cannot be represented as Latin1- */- simdutf_warn_unused size_t convert_utf8_to_latin1(const char * input, size_t length, char* latin1_output) noexcept;--/**- * Using native endianness, convert possibly broken UTF-8 string into a UTF-16 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */-simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_output) noexcept;---/**- * Using native endianness, convert a Latin1 string into a UTF-16 string.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t.- */-simdutf_warn_unused size_t convert_latin1_to_utf16(const char * input, size_t length, char16_t* utf16_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */-simdutf_warn_unused size_t convert_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-16BE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */-simdutf_warn_unused size_t convert_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_output) noexcept;--- /**- * Convert possibly broken UTF-8 string into latin1 string with errors.- * If the string cannot be represented as Latin1, an error- * code is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused result convert_utf8_to_latin1_with_errors(const char * input, size_t length, char* latin1_output) noexcept;--/**- * Using native endianness, convert possibly broken UTF-8 string into UTF-16- * string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf8_to_utf16_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf8_to_utf16le_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf8_to_utf16be_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return the number of written char32_t; 0 if the input was not valid UTF-8 string- */-simdutf_warn_unused size_t convert_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_output) noexcept;--/**- * Convert possibly broken UTF-8 string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */-simdutf_warn_unused result convert_utf8_to_utf32_with_errors(const char * input, size_t length, char32_t* utf32_output) noexcept;-- /**- * Convert valid UTF-8 string into latin1 string.- *- * This function assumes that the input string is valid UTF-8.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused size_t convert_valid_utf8_to_latin1(const char * input, size_t length, char* latin1_output) noexcept;---/**- * Using native endianness, convert valid UTF-8 string into a UTF-16 string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */-simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert valid UTF-8 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */-simdutf_warn_unused size_t convert_valid_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert valid UTF-8 string into UTF-16BE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */-simdutf_warn_unused size_t convert_valid_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert valid UTF-8 string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return the number of written char32_t- */-simdutf_warn_unused size_t convert_valid_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_buffer) noexcept;---/**- * Return the number of bytes that this Latin1 string would require in UTF-8 format.- *- * @param input the Latin1 string to convert- * @param length the length of the string bytes- * @return the number of bytes required to encode the Latin1 string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_latin1(const char * input, size_t length) noexcept;--/**- * Compute the number of bytes that this UTF-8 string would require in Latin1 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in byte- * @return the number of bytes required to encode the UTF-8 string as Latin1- */-simdutf_warn_unused size_t latin1_length_from_utf8(const char * input, size_t length) noexcept;--/**- * Compute the number of 2-byte code units that this UTF-8 string would require in UTF-16LE format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char16_t code units required to encode the UTF-8 string as UTF-16LE- */-simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept;--/**- * Compute the number of 4-byte code units that this UTF-8 string would require in UTF-32 format.- *- * This function is equivalent to count_utf8- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char32_t code units required to encode the UTF-8 string as UTF-32- */-simdutf_warn_unused size_t utf32_length_from_utf8(const char * input, size_t length) noexcept;--/**- * Using native endianness, convert possibly broken UTF-16 string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;----/**- * Using native endianness, convert possibly broken UTF-16 string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16 string or if it cannot be represented as Latin1- */-simdutf_warn_unused size_t convert_utf16_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert possibly broken UTF-16LE string into Latin1 string.- * If the string cannot be represented as Latin1, an error- * is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string or if it cannot be represented as Latin1- */-simdutf_warn_unused size_t convert_utf16le_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16BE string or if it cannot be represented as Latin1- */-simdutf_warn_unused size_t convert_utf16be_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;---/**- * Convert possibly broken UTF-16LE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Using native endianness, convert possibly broken UTF-16 string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16_to_latin1_with_errors(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert possibly broken UTF-16LE string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16le_to_latin1_with_errors(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into Latin1 string.- * If the string cannot be represented as Latin1, an error- * is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16be_to_latin1_with_errors(const char16_t * input, size_t length, char* latin1_buffer) noexcept;---/**- * Using native endianness, convert possibly broken UTF-16 string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16le_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf16be_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Using native endianness, convert valid UTF-16 string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;---/**- * Using native endianness, convert UTF-16 string into Latin1 string.- *- * This function assumes that the input string is valid UTF-8.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert valid UTF-16LE string into Latin1 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16le_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert valid UTF-16BE string into Latin1 string.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16be_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) noexcept;---/**- * Convert valid UTF-16LE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16le_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert valid UTF-16BE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Using native endianness, convert possibly broken UTF-16 string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert possibly broken UTF-16LE string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */-simdutf_warn_unused size_t convert_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Using native endianness, convert possibly broken UTF-16 string into- * UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */-simdutf_warn_unused result convert_utf16_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */-simdutf_warn_unused result convert_utf16le_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */-simdutf_warn_unused result convert_utf16be_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Using native endianness, convert valid UTF-16 string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-16 (native endianness).- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert valid UTF-16LE string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;--/**- * Convert valid UTF-16BE string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept;---/*- * Compute the number of bytes that this UTF-16LE/BE string would require in Latin1 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as Latin1- */-simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept;---/**- * Using native endianness; Compute the number of bytes that this UTF-16- * string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept;--/**- * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t * input, size_t length) noexcept;--/**- * Compute the number of bytes that this UTF-16BE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16BE string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t * input, size_t length) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */-simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf32_to_utf8_with_errors(const char32_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Convert valid UTF-32 string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) noexcept;--/**- * Using native endianness, convert possibly broken UTF-32 string into a UTF-16 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */-simdutf_warn_unused size_t convert_utf32_to_utf16(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */-simdutf_warn_unused size_t convert_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string or if it cannot be represented as Latin1- */-simdutf_warn_unused size_t convert_utf32_to_latin1(const char32_t * input, size_t length, char* latin1_buffer) noexcept;---/**- * Convert possibly broken UTF-32 string into Latin1 string and stop on error.- * If the string cannot be represented as Latin1, an error is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */-simdutf_warn_unused result convert_utf32_to_latin1_with_errors(const char32_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert valid UTF-32 string into Latin1 string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf32_to_latin1(const char32_t * input, size_t length, char* latin1_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-16BE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */-simdutf_warn_unused size_t convert_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Using native endianness, convert possibly broken UTF-32 string into UTF-16- * string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf32_to_utf16_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf32_to_utf16le_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */-simdutf_warn_unused result convert_utf32_to_utf16be_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Using native endianness, convert valid UTF-32 string into a UTF-16 string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf32_to_utf16(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert valid UTF-32 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Convert valid UTF-32 string into UTF-16BE string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */-simdutf_warn_unused size_t convert_valid_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept;--/**- * Change the endianness of the input. Can be used to go from UTF-16LE to UTF-16BE or- * from UTF-16BE to UTF-16LE.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @param output the pointer to buffer that can hold the conversion result- */-void change_endianness_utf16(const char16_t * input, size_t length, char16_t * output) noexcept;--/**- * Compute the number of bytes that this UTF-32 string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as UTF-8- */-simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t * input, size_t length) noexcept;--/**- * Compute the number of two-byte code units that this UTF-32 string would require in UTF-16 format.- *- * This function does not validate the input.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as UTF-16- */-simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t * input, size_t length) noexcept;--/**- * Using native endianness; Compute the number of bytes that this UTF-16- * string would require in UTF-32 format.- *- * This function is equivalent to count_utf16.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-32- */-simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t * input, size_t length) noexcept;--/**- * Compute the number of bytes that this UTF-16LE string would require in UTF-32 format.- *- * This function is equivalent to count_utf16le.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-32- */-simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t * input, size_t length) noexcept;--/**- * Compute the number of bytes that this UTF-16BE string would require in UTF-32 format.- *- * This function is equivalent to count_utf16be.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16BE string as UTF-32- */-simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16 (native endianness).- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @return number of code points- */-simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @return number of code points- */-simdutf_warn_unused size_t count_utf16le(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @return number of code points- */-simdutf_warn_unused size_t count_utf16be(const char16_t * input, size_t length) noexcept;--/**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return number of code points- */-simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept;--/**- * Given a valid UTF-8 string having a possibly truncated last character,- * this function checks the end of string. If the last character is truncated (or partial),- * then it returns a shorter length (shorter by 1 to 3 bytes) so that the short UTF-8- * strings only contain complete characters. If there is no truncated character,- * the original length is returned.- *- * This function assumes that the input string is valid UTF-8, but possibly truncated.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes- */-simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length);--/**- * Given a valid UTF-16BE string having a possibly truncated last character,- * this function checks the end of string. If the last character is truncated (or partial),- * then it returns a shorter length (shorter by 1 unit) so that the short UTF-16BE- * strings only contain complete characters. If there is no truncated character,- * the original length is returned.- *- * This function assumes that the input string is valid UTF-16BE, but possibly truncated.- *- * @param input the UTF-16BE string to process- * @param length the length of the string in bytes- * @return the length of the string in bytes, possibly shorter by 1 unit- */-simdutf_warn_unused size_t trim_partial_utf16be(const char16_t* input, size_t length);--/**- * Given a valid UTF-16LE string having a possibly truncated last character,- * this function checks the end of string. If the last character is truncated (or partial),- * then it returns a shorter length (shorter by 1 unit) so that the short UTF-16LE- * strings only contain complete characters. If there is no truncated character,- * the original length is returned.- *- * This function assumes that the input string is valid UTF-16LE, but possibly truncated.- *- * @param input the UTF-16LE string to process- * @param length the length of the string in bytes- * @return the length of the string in unit, possibly shorter by 1 unit- */-simdutf_warn_unused size_t trim_partial_utf16le(const char16_t* input, size_t length);---/**- * Given a valid UTF-16 string having a possibly truncated last character,- * this function checks the end of string. If the last character is truncated (or partial),- * then it returns a shorter length (shorter by 1 unit) so that the short UTF-16- * strings only contain complete characters. If there is no truncated character,- * the original length is returned.- *- * This function assumes that the input string is valid UTF-16, but possibly truncated.- * We use the native endianness.- *- * @param input the UTF-16 string to process- * @param length the length of the string in bytes- * @return the length of the string in unit, possibly shorter by 1 unit- */-simdutf_warn_unused size_t trim_partial_utf16(const char16_t* input, size_t length);--// base64_options are used to specify the base64 encoding options.-using base64_options = uint64_t;-enum : base64_options {- base64_default = 0, /* standard base64 format */- base64_url = 1 /* base64url format*/-};--/**- * Provide the maximal binary length in bytes given the base64 input.- * In general, if the input contains ASCII spaces, the result will be less than- * the maximum length.- *- * @param input the base64 input to process- * @param length the length of the base64 input in bytes- * @return maximum number of binary bytes- */-simdutf_warn_unused size_t maximal_binary_length_from_base64(const char * input, size_t length) noexcept;--/**- * Provide the maximal binary length in bytes given the base64 input.- * In general, if the input contains ASCII spaces, the result will be less than- * the maximum length.- *- * @param input the base64 input to process, in ASCII stored as 16-bit units- * @param length the length of the base64 input in 16-bit units- * @return maximal number of binary bytes- */-simdutf_warn_unused size_t maximal_binary_length_from_base64(const char16_t * input, size_t length) noexcept;--/**- * Convert a base64 input to a binary ouput.- *- * This function follows the WHATWG forgiving-base64 format, which means that it will- * ignore any ASCII spaces in the input. You may provide a padded input (with one or two- * equal signs at the end) or an unpadded input (without any equal signs at the end).- *- * See https://infra.spec.whatwg.org/#forgiving-base64-decode- *- * This function will fail in case of invalid input. There are two possible reasons for- * failure: the input contains a number of base64 characters that when divided by 4, leaves- * a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character- * that is not a valid base64 character (INVALID_BASE64_CHARACTER).- *- * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input- * where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then- * r.count contains the number of bytes decoded.- *- * You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long.- * If you fail to provide that much space, the function may cause a buffer overflow.- *- * @param input the base64 string to process- * @param length the length of the string in bytes- * @param output the pointer to buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in bytes) if any, or the number of bytes written if successful.- */-simdutf_warn_unused result base64_to_binary(const char * input, size_t length, char* output, base64_options options = base64_default) noexcept;--/**- * Provide the base64 length in bytes given the length of a binary input.- *- * @param length the length of the input in bytes- * @return number of base64 bytes- */-simdutf_warn_unused size_t base64_length_from_binary(size_t length, base64_options options = base64_default) noexcept;--/**- * Convert a binary input to a base64 ouput. The output is always padded with equal signs so that it is- * a multiple of 4 bytes long.- *- * This function always succeeds.- *- * @param input the binary to process- * @param length the length of the input in bytes- * @param output the pointer to buffer that can hold the conversion result (should be at least base64_length_from_binary(length) bytes long)- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return number of written bytes, will be equal to base64_length_from_binary(length, options)- */-size_t binary_to_base64(const char * input, size_t length, char* output, base64_options options = base64_default) noexcept;--/**- * Convert a base64 input to a binary ouput.- *- * This function follows the WHATWG forgiving-base64 format, which means that it will- * ignore any ASCII spaces in the input. You may provide a padded input (with one or two- * equal signs at the end) or an unpadded input (without any equal signs at the end).- *- * See https://infra.spec.whatwg.org/#forgiving-base64-decode- *- * This function will fail in case of invalid input. There are two possible reasons for- * failure: the input contains a number of base64 characters that when divided by 4, leaves- * a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character- * that is not a valid base64 character (INVALID_BASE64_CHARACTER).- *- * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input- * where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then- * r.count contains the number of bytes decoded.- *- * You should call this function with a buffer that is at least maximal_binary_length_from_utf6_base64(input, length) bytes long.- * If you fail to provide that much space, the function may cause a buffer overflow.- *- * @param input the base64 string to process, in ASCII stored as 16-bit units- * @param length the length of the string in 16-bit units- * @param output the pointer to buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and position of the INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number of bytes written if successful.- */-simdutf_warn_unused result base64_to_binary(const char16_t * input, size_t length, char* output, base64_options options = base64_default) noexcept;--/**- * Convert a base64 input to a binary ouput.- *- * This function follows the WHATWG forgiving-base64 format, which means that it will- * ignore any ASCII spaces in the input. You may provide a padded input (with one or two- * equal signs at the end) or an unpadded input (without any equal signs at the end).- *- * See https://infra.spec.whatwg.org/#forgiving-base64-decode- *- * This function will fail in case of invalid input. There are three possible reasons for- * failure: the input contains a number of base64 characters that when divided by 4, leaves- * a single remainder character (BASE64_INPUT_REMAINDER), the input contains a character- * that is not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer- * is too small (OUTPUT_BUFFER_TOO_SMALL).- *- * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written- * and the number of units processed, see description of the parameters and returned value.- *- * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input- * where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then- * r.count contains the number of bytes decoded.- *- * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected to discard- * the output.- *- * @param input the base64 string to process, in ASCII stored as 8-bit or 16-bit units- * @param length the length of the string in 8-bit or 16-bit units.- * @param output the pointer to buffer that can hold the conversion result.- * @param outlen the number of bytes that can be written in the output buffer. Upon return, it is modified to reflect how many bytes were written.- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and position of the INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number of units processed if successful.- */-simdutf_warn_unused result base64_to_binary_safe(const char * input, size_t length, char* output, size_t& outlen, base64_options options = base64_default) noexcept;-simdutf_warn_unused result base64_to_binary_safe(const char16_t * input, size_t length, char* output, size_t& outlen, base64_options options = base64_default) noexcept;--/**- * An implementation of simdutf for a particular CPU architecture.- *- * Also used to maintain the currently active implementation. The active implementation is- * automatically initialized on first use to the most advanced implementation supported by the host.- */-class implementation {-public:-- /**- * The name of this implementation.- *- * const implementation *impl = simdutf::active_implementation;- * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;- *- * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"- */- virtual std::string name() const { return std::string(_name); }-- /**- * The description of this implementation.- *- * const implementation *impl = simdutf::active_implementation;- * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;- *- * @return the name of the implementation, e.g. "haswell", "westmere", "arm64"- */- virtual std::string description() const { return std::string(_description); }-- /**- * The instruction sets this implementation is compiled against- * and the current CPU match. This function may poll the current CPU/system- * and should therefore not be called too often if performance is a concern.- *- *- * @return true if the implementation can be safely used on the current system (determined at runtime)- */- bool supported_by_runtime_system() const;-- /**- * This function will try to detect the encoding- * @param input the string to identify- * @param length the length of the string in bytes.- * @return the encoding type detected- */- virtual encoding_type autodetect_encoding(const char * input, size_t length) const noexcept;-- /**- * This function will try to detect the possible encodings in one pass- * @param input the string to identify- * @param length the length of the string in bytes.- * @return the encoding type detected- */- virtual int detect_encodings(const char * input, size_t length) const noexcept = 0;-- /**- * @private For internal implementation use- *- * The instruction sets this implementation is compiled against.- *- * @return a mask of all required `internal::instruction_set::` values- */- virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }--- /**- * Validate the UTF-8 string.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid UTF-8.- */- simdutf_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-8 string and stop on errors.- *- * Overridden by each implementation.- *- * @param buf the UTF-8 string to validate.- * @param len the length of the string in bytes.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0;-- /**- * Validate the ASCII string.- *- * Overridden by each implementation.- *- * @param buf the ASCII string to validate.- * @param len the length of the string in bytes.- * @return true if and only if the string is valid ASCII.- */- simdutf_warn_unused virtual bool validate_ascii(const char *buf, size_t len) const noexcept = 0;-- /**- * Validate the ASCII string and stop on error.- *- * Overridden by each implementation.- *- * @param buf the ASCII string to validate.- * @param len the length of the string in bytes.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-16LE string.This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf16le_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return true if and only if the string is valid UTF-16LE.- */- simdutf_warn_unused virtual bool validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-16BE string. This function may be best when you expect- * the input to be almost always valid. Otherwise, consider using- * validate_utf16be_with_errors.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16BE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return true if and only if the string is valid UTF-16BE.- */- simdutf_warn_unused virtual bool validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-16LE string and stop on error. It might be faster than- * validate_utf16le when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16LE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result validate_utf16le_with_errors(const char16_t *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-16BE string and stop on error. It might be faster than- * validate_utf16be when an error is expected to occur early.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-16BE string to validate.- * @param len the length of the string in number of 2-byte code units (char16_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result validate_utf16be_with_errors(const char16_t *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-32 string.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-32 string to validate.- * @param len the length of the string in number of 4-byte code units (char32_t).- * @return true if and only if the string is valid UTF-32.- */- simdutf_warn_unused virtual bool validate_utf32(const char32_t *buf, size_t len) const noexcept = 0;-- /**- * Validate the UTF-32 string and stop on error.- *- * Overridden by each implementation.- *- * This function is not BOM-aware.- *- * @param buf the UTF-32 string to validate.- * @param len the length of the string in number of 4-byte code units (char32_t).- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result validate_utf32_with_errors(const char32_t *buf, size_t len) const noexcept = 0;-- /**- * Convert Latin1 string into UTF8 string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_latin1_to_utf8(const char * input, size_t length, char* utf8_output) const noexcept = 0;--- /**- * Convert possibly Latin1 string into UTF-16LE string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_latin1_to_utf16le(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert Latin1 string into UTF-16BE string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_latin1_to_utf16be(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert Latin1 string into UTF-32 string.- *- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the Latin1 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return the number of written char32_t; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_latin1_to_utf32(const char * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if the input was not valid UTF-8 string or if it cannot be represented as Latin1- */- simdutf_warn_unused virtual size_t convert_utf8_to_latin1(const char * input, size_t length, char* latin1_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into latin1 string with errors.- * If the string cannot be represented as Latin1, an error- * code is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result convert_utf8_to_latin1_with_errors(const char * input, size_t length, char* latin1_output) const noexcept = 0;-- /**- * Convert valid UTF-8 string into latin1 string.- *- * This function assumes that the input string is valid UTF-8.- *- * This function is not BOM-aware.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param latin1_output the pointer to buffer that can hold conversion result- * @return the number of written char; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused virtual size_t convert_valid_utf8_to_latin1(const char * input, size_t length, char* latin1_output) const noexcept = 0;--- /**- * Convert possibly broken UTF-8 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused virtual size_t convert_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-16BE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused virtual size_t convert_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.- */- simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t; 0 if the input was not valid UTF-8 string- */- simdutf_warn_unused virtual size_t convert_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_output) const noexcept = 0;-- /**- * Convert possibly broken UTF-8 string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */- simdutf_warn_unused virtual result convert_utf8_to_utf32_with_errors(const char * input, size_t length, char32_t* utf32_output) const noexcept = 0;-- /**- * Convert valid UTF-8 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */- simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;--/**- * Convert valid UTF-8 string into UTF-16BE string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char16_t- */- simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert valid UTF-8 string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in bytes- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return the number of written char32_t- */- simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Compute the number of 2-byte code units that this UTF-8 string would require in UTF-16LE format.- *- * This function does not validate the input.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char16_t code units required to encode the UTF-8 string as UTF-16LE- */- simdutf_warn_unused virtual size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept = 0;-- /**- * Compute the number of 4-byte code units that this UTF-8 string would require in UTF-32 format.- *- * This function is equivalent to count_utf8.- *- * This function does not validate the input.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return the number of char32_t code units required to encode the UTF-8 string as UTF-32- */- simdutf_warn_unused virtual size_t utf32_length_from_utf8(const char * input, size_t length) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string or if it cannot be represented as Latin1- */- simdutf_warn_unused virtual size_t convert_utf16le_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16BE string or if it cannot be represented as Latin1- */- simdutf_warn_unused virtual size_t convert_utf16be_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into Latin1 string.- * If the string cannot be represented as Latin1, an error- * is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf16le_to_latin1_with_errors(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into Latin1 string.- * If the string cannot be represented as Latin1, an error- * is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf16be_to_latin1_with_errors(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16LE string into Latin1 string.- *- * This function assumes that the input string is valid UTF-8.-- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16le_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16BE string into Latin1 string.- *- * This function assumes that the input string is valid UTF-8.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16be_to_latin1(const char16_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */- simdutf_warn_unused virtual size_t convert_utf16le_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16BE string- */- simdutf_warn_unused virtual size_t convert_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf16le_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf16be_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16LE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16le_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16BE string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16LE string- */- simdutf_warn_unused virtual size_t convert_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into UTF-32 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-16BE string- */- simdutf_warn_unused virtual size_t convert_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */- simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.- */- simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16LE string into UTF-32 string.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Convert valid UTF-16LE string into UTF-32BE string.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @param utf32_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-8- */- simdutf_warn_unused virtual size_t utf8_length_from_utf16le(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-16BE string would require in UTF-8 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16BE string as UTF-8- */- simdutf_warn_unused virtual size_t utf8_length_from_utf16be(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into Latin1 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */-- simdutf_warn_unused virtual size_t convert_utf32_to_latin1(const char32_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into Latin1 string and stop on error.- * If the string cannot be represented as Latin1, an error is returned.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf32_to_latin1_with_errors(const char32_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert valid UTF-32 string into Latin1 string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param latin1_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf32_to_latin1(const char32_t * input, size_t length, char* latin1_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-8 string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */- simdutf_warn_unused virtual size_t convert_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-8 string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.- */- simdutf_warn_unused virtual result convert_utf32_to_utf8_with_errors(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0;-- /**- * Convert valid UTF-32 string into UTF-8 string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf8_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0;--- /**- * Return the number of bytes that this UTF-16 string would require in Latin1 format.- *- *- * @param input the UTF-16 string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16 string as Latin1- */- simdutf_warn_unused virtual size_t utf16_length_from_latin1(size_t length) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-16LE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */- simdutf_warn_unused virtual size_t convert_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-16BE string.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return number of written code units; 0 if input is not a valid UTF-32 string- */- simdutf_warn_unused virtual size_t convert_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */- simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error.- *- * During the conversion also validation of the input string is done.- * This function is suitable to work with inputs from untrusted sources.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold conversion result- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.- */- simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert valid UTF-32 string into UTF-16LE string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Convert valid UTF-32 string into UTF-16BE string.- *- * This function assumes that the input string is valid UTF-32.- *- * This function is not BOM-aware.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @param utf16_buffer the pointer to buffer that can hold the conversion result- * @return number of written code units; 0 if conversion is not possible- */- simdutf_warn_unused virtual size_t convert_valid_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0;-- /**- * Change the endianness of the input. Can be used to go from UTF-16LE to UTF-16BE or- * from UTF-16BE to UTF-16LE.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16 string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @param output the pointer to buffer that can hold the conversion result- */- virtual void change_endianness_utf16(const char16_t * input, size_t length, char16_t * output) const noexcept = 0;-- /**- * Return the number of bytes that this Latin1 string would require in UTF-8 format.- *- * @param input the Latin1 string to convert- * @param length the length of the string bytes- * @return the number of bytes required to encode the Latin1 string as UTF-8- */- simdutf_warn_unused virtual size_t utf8_length_from_latin1(const char * input, size_t length) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-32 string would require in UTF-8 format.- *- * This function does not validate the input.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as UTF-8- */- simdutf_warn_unused virtual size_t utf8_length_from_utf32(const char32_t * input, size_t length) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-32 string would require in Latin1 format.- *- * This function does not validate the input.- *- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as Latin1- */- simdutf_warn_unused virtual size_t latin1_length_from_utf32(size_t length) const noexcept = 0;-- /**- * Compute the number of bytes that this UTF-8 string would require in Latin1 format.- *- * This function does not validate the input.- *- * @param input the UTF-8 string to convert- * @param length the length of the string in byte- * @return the number of bytes required to encode the UTF-8 string as Latin1- */- simdutf_warn_unused virtual size_t latin1_length_from_utf8(const char * input, size_t length) const noexcept = 0;-- /*- * Compute the number of bytes that this UTF-16LE/BE string would require in Latin1 format.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as Latin1- */- simdutf_warn_unused virtual size_t latin1_length_from_utf16(size_t length) const noexcept = 0;-- /**- * Compute the number of two-byte code units that this UTF-32 string would require in UTF-16 format.- *- * This function does not validate the input.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as UTF-16- */- simdutf_warn_unused virtual size_t utf16_length_from_utf32(const char32_t * input, size_t length) const noexcept = 0;--- /**- * Return the number of bytes that this UTF-32 string would require in Latin1 format.- *- * This function does not validate the input.- *- * @param input the UTF-32 string to convert- * @param length the length of the string in 4-byte code units (char32_t)- * @return the number of bytes required to encode the UTF-32 string as Latin1- */- simdutf_warn_unused virtual size_t utf32_length_from_latin1(size_t length) const noexcept = 0;-- /*- * Compute the number of bytes that this UTF-16LE string would require in UTF-32 format.- *- * This function is equivalent to count_utf16le.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16LE string as UTF-32- */- simdutf_warn_unused virtual size_t utf32_length_from_utf16le(const char16_t * input, size_t length) const noexcept = 0;-- /*- * Compute the number of bytes that this UTF-16BE string would require in UTF-32 format.- *- * This function is equivalent to count_utf16be.- *- * This function does not validate the input.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to convert- * @param length the length of the string in 2-byte code units (char16_t)- * @return the number of bytes required to encode the UTF-16BE string as UTF-32- */- simdutf_warn_unused virtual size_t utf32_length_from_utf16be(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16LE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16LE string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @return number of code points- */- simdutf_warn_unused virtual size_t count_utf16le(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-16BE.- *- * This function is not BOM-aware.- *- * @param input the UTF-16BE string to process- * @param length the length of the string in 2-byte code units (char16_t)- * @return number of code points- */- simdutf_warn_unused virtual size_t count_utf16be(const char16_t * input, size_t length) const noexcept = 0;--- /**- * Count the number of code points (characters) in the string assuming that- * it is valid.- *- * This function assumes that the input string is valid UTF-8.- *- * @param input the UTF-8 string to process- * @param length the length of the string in bytes- * @return number of code points- */- simdutf_warn_unused virtual size_t count_utf8(const char * input, size_t length) const noexcept = 0;-- /**- * Provide the maximal binary length in bytes given the base64 input.- * In general, if the input contains ASCII spaces, the result will be less than- * the maximum length.- *- * @param input the base64 input to process- * @param length the length of the base64 input in bytes- * @return maximal number of binary bytes- */- simdutf_warn_unused virtual size_t maximal_binary_length_from_base64(const char * input, size_t length) const noexcept = 0;-- /**- * Provide the maximal binary length in bytes given the base64 input.- * In general, if the input contains ASCII spaces, the result will be less than- * the maximum length.- *- * @param input the base64 input to process, in ASCII stored as 16-bit units- * @param length the length of the base64 input in 16-bit units- * @return maximal number of binary bytes- */- simdutf_warn_unused virtual size_t maximal_binary_length_from_base64(const char16_t * input, size_t length) const noexcept = 0;-- /**- * Convert a base64 input to a binary ouput.- *- * This function follows the WHATWG forgiving-base64 format, which means that it will- * ignore any ASCII spaces in the input. You may provide a padded input (with one or two- * equal signs at the end) or an unpadded input (without any equal signs at the end).- *- * See https://infra.spec.whatwg.org/#forgiving-base64-decode- *- * This function will fail in case of invalid input. There are two possible reasons for- * failure: the input contains a number of base64 characters that when divided by 4, leaves- * a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character- * that is not a valid base64 character (INVALID_BASE64_CHARACTER).- *- * You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long.- * If you fail to provide that much space, the function may cause a buffer overflow.- *- * @param input the base64 string to process- * @param length the length of the string in bytes- * @param output the pointer to buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and either position of the error (in the input in bytes) if any, or the number of bytes written if successful.- */- simdutf_warn_unused virtual result base64_to_binary(const char * input, size_t length, char* output, base64_options options = base64_default) const noexcept = 0;-- /**- * Convert a base64 input to a binary ouput.- *- * This function follows the WHATWG forgiving-base64 format, which means that it will- * ignore any ASCII spaces in the input. You may provide a padded input (with one or two- * equal signs at the end) or an unpadded input (without any equal signs at the end).- *- * See https://infra.spec.whatwg.org/#forgiving-base64-decode- *- * This function will fail in case of invalid input. There are two possible reasons for- * failure: the input contains a number of base64 characters that when divided by 4, leaves- * a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character- * that is not a valid base64 character (INVALID_BASE64_CHARACTER).- *- * You should call this function with a buffer that is at least maximal_binary_length_from_utf6_base64(input, length) bytes long.- * If you fail to provide that much space, the function may cause a buffer overflow.- *- * @param input the base64 string to process, in ASCII stored as 16-bit units- * @param length the length of the string in 16-bit units- * @param output the pointer to buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return a result pair struct (of type simdutf::error containing the two fields error and count) with an error code and position of the INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number of bytes written if successful.- */- simdutf_warn_unused virtual result base64_to_binary(const char16_t * input, size_t length, char* output, base64_options options = base64_default) const noexcept = 0;-- /**- * Provide the base64 length in bytes given the length of a binary input.- *- * @param length the length of the input in bytes- * @parem options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return number of base64 bytes- */- simdutf_warn_unused virtual size_t base64_length_from_binary(size_t length, base64_options options = base64_default) const noexcept = 0;-- /**- * Convert a binary input to a base64 ouput. The output is always padded with equal signs so that it is- * a multiple of 4 bytes long.- *- * This function always succeeds.- *- * @param input the binary to process- * @param length the length of the input in bytes- * @param output the pointer to buffer that can hold the conversion result (should be at least base64_length_from_binary(length) bytes long)- * @param options the base64 options to use, can be base64_default or base64_url, is base64_default by default.- * @return number of written bytes, will be equal to base64_length_from_binary(length, options)- */- virtual size_t binary_to_base64(const char * input, size_t length, char* output, base64_options options = base64_default) const noexcept = 0;---protected:- /** @private Construct an implementation with the given name and description. For subclasses. */- simdutf_really_inline implementation(- const char* name,- const char* description,- uint32_t required_instruction_sets- ) :- _name(name),- _description(description),- _required_instruction_sets(required_instruction_sets)- {- }-protected:- ~implementation() = default;-private:- /**- * The name of this implementation.- */- const char* _name;-- /**- * The description of this implementation.- */- const char* _description;-- /**- * Instruction sets required for this implementation.- */- const uint32_t _required_instruction_sets;-};--/** @private */-namespace internal {--/**- * The list of available implementations compiled into simdutf.- */-class available_implementation_list {-public:- /** Get the list of available implementations compiled into simdutf */- simdutf_really_inline available_implementation_list() {}- /** Number of implementations */- size_t size() const noexcept;- /** STL const begin() iterator */- const implementation * const *begin() const noexcept;- /** STL const end() iterator */- const implementation * const *end() const noexcept;-- /**- * Get the implementation with the given name.- *- * Case sensitive.- *- * const implementation *impl = simdutf::available_implementations["westmere"];- * if (!impl) { exit(1); }- * if (!imp->supported_by_runtime_system()) { exit(1); }- * simdutf::active_implementation = impl;- *- * @param name the implementation to find, e.g. "westmere", "haswell", "arm64"- * @return the implementation, or nullptr if the parse failed.- */- const implementation * operator[](const std::string &name) const noexcept {- for (const implementation * impl : *this) {- if (impl->name() == name) { return impl; }- }- return nullptr;- }-- /**- * Detect the most advanced implementation supported by the current host.- *- * This is used to initialize the implementation on startup.- *- * const implementation *impl = simdutf::available_implementation::detect_best_supported();- * simdutf::active_implementation = impl;- *- * @return the most advanced supported implementation for the current host, or an- * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported- * implementation. Will never return nullptr.- */- const implementation *detect_best_supported() const noexcept;-};--template<typename T>-class atomic_ptr {-public:- atomic_ptr(T *_ptr) : ptr{_ptr} {}--#if defined(SIMDUTF_NO_THREADS)- operator const T*() const { return ptr; }- const T& operator*() const { return *ptr; }- const T* operator->() const { return ptr; }-- operator T*() { return ptr; }- T& operator*() { return *ptr; }- T* operator->() { return ptr; }- atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; }--#else- operator const T*() const { return ptr.load(); }- const T& operator*() const { return *ptr; }- const T* operator->() const { return ptr.load(); }-- operator T*() { return ptr.load(); }- T& operator*() { return *ptr; }- T* operator->() { return ptr.load(); }- atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; }--#endif--private:-#if defined(SIMDUTF_NO_THREADS)- T* ptr;-#else- std::atomic<T*> ptr;-#endif-};--class detect_best_supported_implementation_on_first_use;--} // namespace internal--/**- * The list of available implementations compiled into simdutf.- */-extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations();--/**- * The active implementation.- *- * Automatically initialized on first use to the most advanced implementation supported by this hardware.- */-extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation();---} // namespace simdutf--#endif // SIMDUTF_IMPLEMENTATION_H-/* end file include/simdutf/implementation.h */---// Implementation-internal files (must be included before the implementations themselves, to keep-// amalgamation working--otherwise, the first time a file is included, it might be put inside the-// #ifdef SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other implementations can't-// compile unless that implementation is turned on).---SIMDUTF_POP_DISABLE_WARNINGS--#endif // SIMDUTF_H-/* end file include/simdutf.h */+/* auto-generated on 2026-01-13 09:03:21 +0100. Do not edit! */ +/* begin file include/simdutf.h */ +#ifndef SIMDUTF_H +#define SIMDUTF_H +#include <cstring> + +/* begin file include/simdutf/compiler_check.h */ +#ifndef SIMDUTF_COMPILER_CHECK_H +#define SIMDUTF_COMPILER_CHECK_H + +#ifndef __cplusplus + #error simdutf requires a C++ compiler +#endif + +#ifndef SIMDUTF_CPLUSPLUS + #if defined(_MSVC_LANG) && !defined(__clang__) + #define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) + #else + #define SIMDUTF_CPLUSPLUS __cplusplus + #endif +#endif + +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + +// C++ 23 +#if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) + #define SIMDUTF_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDUTF_CPLUSPLUS20) && (SIMDUTF_CPLUSPLUS >= 202002L) + #define SIMDUTF_CPLUSPLUS20 1 +#endif + +// C++ 17 +#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) + #define SIMDUTF_CPLUSPLUS17 1 +#endif + +// C++ 14 +#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) + #define SIMDUTF_CPLUSPLUS14 1 +#endif + +// C++ 11 +#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) + #define SIMDUTF_CPLUSPLUS11 1 +#endif + +#ifndef SIMDUTF_CPLUSPLUS11 + #error simdutf requires a compiler compliant with the C++11 standard +#endif + +#endif // SIMDUTF_COMPILER_CHECK_H +/* end file include/simdutf/compiler_check.h */ +/* begin file include/simdutf/common_defs.h */ +#ifndef SIMDUTF_COMMON_DEFS_H +#define SIMDUTF_COMMON_DEFS_H + +/* begin file include/simdutf/portability.h */ +#ifndef SIMDUTF_PORTABILITY_H +#define SIMDUTF_PORTABILITY_H + + +#include <cfloat> +#include <cstddef> +#include <cstdint> +#include <cstdlib> +#ifndef _WIN32 + // strcasecmp, strncasecmp + #include <strings.h> +#endif + +#if defined(__apple_build_version__) + #if __apple_build_version__ < 14000000 + #define SIMDUTF_SPAN_DISABLED \ + 1 // apple-clang/13 doesn't support std::convertible_to + #endif +#endif + +#if SIMDUTF_CPLUSPLUS20 + #include <version> + #if __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L && \ + !defined(SIMDUTF_SPAN_DISABLED) + #define SIMDUTF_SPAN 1 + #endif // __cpp_concepts >= 201907L && __cpp_lib_span >= 202002L + #if __cpp_lib_atomic_ref >= 201806L + #define SIMDUTF_ATOMIC_REF 1 + #endif // __cpp_lib_atomic_ref + #if __has_cpp_attribute(maybe_unused) >= 201603L + #define SIMDUTF_MAYBE_UNUSED_AVAILABLE 1 + #endif // __has_cpp_attribute(maybe_unused) >= 201603L +#endif + +/** + * We want to check that it is actually a little endian system at + * compile-time. + */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) + #define SIMDUTF_IS_BIG_ENDIAN 0 +#else + #if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ + #include <machine/endian.h> + #elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) + #include <sys/byteorder.h> + #else // defined(__APPLE__) || defined(__FreeBSD__) + + #ifdef __has_include + #if __has_include(<endian.h>) + #include <endian.h> + #endif //__has_include(<endian.h>) + #endif //__has_include + + #endif // defined(__APPLE__) || defined(__FreeBSD__) + + #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) + #define SIMDUTF_IS_BIG_ENDIAN 0 + #endif + + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 0 + #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #define SIMDUTF_IS_BIG_ENDIAN 1 + #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ + +/** + * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined. + */ + +#ifdef _MSC_VER + #define SIMDUTF_VISUAL_STUDIO 1 + /** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + * + * Under clang for Windows, we enable: + * * target pragmas so that part and only part of the + * code gets compiled for advanced instructions. + * + */ + #ifdef __clang__ + // clang under visual studio + #define SIMDUTF_CLANG_VISUAL_STUDIO 1 + #else + // just regular visual studio (best guess) + #define SIMDUTF_REGULAR_VISUAL_STUDIO 1 + #endif // __clang__ +#endif // _MSC_VER + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + // https://en.wikipedia.org/wiki/C_alternative_tokens + // This header should have no effect, except maybe + // under Visual Studio. + #include <iso646.h> +#endif + +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) + #define SIMDUTF_IS_X86_64 1 +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + #define SIMDUTF_IS_ARM64 1 +#elif defined(__PPC64__) || defined(_M_PPC64) + #if defined(__VEC__) && defined(__ALTIVEC__) + #define SIMDUTF_IS_PPC64 1 + #endif +#elif defined(__s390__) +// s390 IBM system. Big endian. +#elif (defined(__riscv) || defined(__riscv__)) && __riscv_xlen == 64 + // RISC-V 64-bit + #define SIMDUTF_IS_RISCV64 1 + + // #if __riscv_v_intrinsic >= 1000000 + // #define SIMDUTF_HAS_RVV_INTRINSICS 1 + // #define SIMDUTF_HAS_RVV_TARGET_REGION 1 + // #elif ... + // Check for special compiler versions that implement pre v1.0 intrinsics + #if __riscv_v_intrinsic >= 11000 + #define SIMDUTF_HAS_RVV_INTRINSICS 1 + #endif + + #define SIMDUTF_HAS_ZVBB_INTRINSICS \ + 0 // there is currently no way to detect this + + #if SIMDUTF_HAS_RVV_INTRINSICS && __riscv_vector && \ + __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 + // RISC-V V extension + #define SIMDUTF_IS_RVV 1 + #if SIMDUTF_HAS_ZVBB_INTRINSICS && __riscv_zvbb >= 1000000 + // RISC-V Vector Basic Bit-manipulation + #define SIMDUTF_IS_ZVBB 1 + #endif + #endif + +#elif defined(__loongarch_lp64) + #if defined(__loongarch_sx) && defined(__loongarch_asx) + #define SIMDUTF_IS_LSX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both + #elif defined(__loongarch_sx) + #define SIMDUTF_IS_LSX 1 + #endif +#else + // The simdutf library is designed + // for 64-bit processors and it seems that you are not + // compiling for a known 64-bit platform. Please + // use a 64-bit target such as x64 or 64-bit ARM for best performance. + #define SIMDUTF_IS_32BITS 1 + + // We do not support 32-bit platforms, but it can be + // handy to identify them. + #if defined(_M_IX86) || defined(__i386__) + #define SIMDUTF_IS_X86_32BITS 1 + #elif defined(__arm__) || defined(_M_ARM) + #define SIMDUTF_IS_ARM_32BITS 1 + #elif defined(__PPC__) || defined(_M_PPC) + #define SIMDUTF_IS_PPC_32BITS 1 + #endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) + +#ifdef SIMDUTF_IS_32BITS + #ifndef SIMDUTF_NO_PORTABILITY_WARNING + // In the future, we may want to warn users of 32-bit systems that + // the simdutf does not support accelerated kernels for such systems. + #endif // SIMDUTF_NO_PORTABILITY_WARNING +#endif // SIMDUTF_IS_32BITS + +// this is almost standard? +#define SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) #a +#define SIMDUTF_STRINGIFY(a) SIMDUTF_STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The simdutf users should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select +// SIMDUTF_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + #ifdef __clang__ + // clang does not have GCC push pop + // warning: clang attribute push can't be used within a namespace in clang + // up til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be + // *outside* of a namespace. + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma(SIMDUTF_STRINGIFY(clang attribute push( \ + __attribute__((target(T))), apply_to = function))) + #define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") + #elif defined(__GNUC__) + // GCC is easier + #define SIMDUTF_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(SIMDUTF_STRINGIFY(GCC target(T))) + #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") + #endif // clang then gcc + +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) + +// Default target region macros don't do anything. +#ifndef SIMDUTF_TARGET_REGION + #define SIMDUTF_TARGET_REGION(T) + #define SIMDUTF_UNTARGET_REGION +#endif + +// Is threading enabled? +#if defined(_REENTRANT) || defined(_MT) + #ifndef SIMDUTF_THREADS_ENABLED + #define SIMDUTF_THREADS_ENABLED + #endif +#endif + +// workaround for large stack sizes under -O0. +// https://github.com/simdutf/simdutf/issues/691 +#ifdef __APPLE__ + #ifndef __OPTIMIZE__ + // Apple systems have small stack sizes in secondary threads. + // Lack of compiler optimization may generate high stack usage. + // Users may want to disable threads for safety, but only when + // in debug mode which we detect by the fact that the __OPTIMIZE__ + // macro is not defined. + #undef SIMDUTF_THREADS_ENABLED + #endif +#endif + +#ifdef SIMDUTF_VISUAL_STUDIO + // This is one case where we do not distinguish between + // regular visual studio and clang under visual studio. + // clang under Windows has _stricmp (like visual studio) but not strcasecmp + // (as clang normally has) + #define simdutf_strcasecmp _stricmp + #define simdutf_strncasecmp _strnicmp +#else + // The strcasecmp, strncasecmp, and strcasestr functions do not work with + // multibyte strings (e.g. UTF-8). So they are only useful for ASCII in our + // context. + // https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings + #define simdutf_strcasecmp strcasecmp + #define simdutf_strncasecmp strncasecmp +#endif + +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ >= 11 + #define SIMDUTF_GCC11ORMORE 1 + #endif // __GNUC__ >= 11 + #if __GNUC__ == 10 + #define SIMDUTF_GCC10 1 + #endif // __GNUC__ == 10 + #if __GNUC__ < 10 + #define SIMDUTF_GCC9OROLDER 1 + #endif // __GNUC__ == 10 +#endif // defined(__GNUC__) && !defined(__clang__) + +#endif // SIMDUTF_PORTABILITY_H +/* end file include/simdutf/portability.h */ +/* begin file include/simdutf/avx512.h */ +#ifndef SIMDUTF_AVX512_H_ +#define SIMDUTF_AVX512_H_ + +/* + It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS. + + All preprocessor directives has form `SIMDUTF_HAS_AVX512{feature}`, + where a feature is a code name for extensions. + + Please see the listing below to find which are supported. +*/ + +#ifndef SIMDUTF_HAS_AVX512F + #if defined(__AVX512F__) && __AVX512F__ == 1 + #define SIMDUTF_HAS_AVX512F 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512DQ + #if defined(__AVX512DQ__) && __AVX512DQ__ == 1 + #define SIMDUTF_HAS_AVX512DQ 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512IFMA + #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1 + #define SIMDUTF_HAS_AVX512IFMA 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512CD + #if defined(__AVX512CD__) && __AVX512CD__ == 1 + #define SIMDUTF_HAS_AVX512CD 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BW + #if defined(__AVX512BW__) && __AVX512BW__ == 1 + #define SIMDUTF_HAS_AVX512BW 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VL + #if defined(__AVX512VL__) && __AVX512VL__ == 1 + #define SIMDUTF_HAS_AVX512VL 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI + #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1 + #define SIMDUTF_HAS_AVX512VBMI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VBMI2 + #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1 + #define SIMDUTF_HAS_AVX512VBMI2 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VNNI + #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1 + #define SIMDUTF_HAS_AVX512VNNI 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512BITALG + #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1 + #define SIMDUTF_HAS_AVX512BITALG 1 + #endif +#endif + +#ifndef SIMDUTF_HAS_AVX512VPOPCNTDQ + #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1 + #define SIMDUTF_HAS_AVX512VPOPCNTDQ 1 + #endif +#endif + +#endif // SIMDUTF_AVX512_H_ +/* end file include/simdutf/avx512.h */ + +// Sometimes logging is useful, but we want it disabled by default +// and free of any logging code in release builds. +#ifdef SIMDUTF_LOGGING + #include <iostream> + #define simdutf_log(msg) \ + std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; + #define simdutf_log_assert(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cerr << "[" << __FUNCTION__ << "]: " << msg << std::endl \ + << "\t" << __FILE__ << ":" << __LINE__ << std::endl; \ + std::abort(); \ + } \ + } while (0) +#else + #define simdutf_log(msg) + #define simdutf_log_assert(cond, msg) +#endif + +#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) + #define SIMDUTF_DEPRECATED __declspec(deprecated) + + #define simdutf_really_inline __forceinline // really inline in release mode + #define simdutf_always_inline __forceinline // always inline, no matter what + #define simdutf_never_inline __declspec(noinline) + + #define simdutf_unused + #define simdutf_warn_unused + + #ifndef simdutf_likely + #define simdutf_likely(x) x + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) x + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning(push)) + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0)) + #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) \ + __pragma(warning(disable : WARNING_NUMBER)) + // Get rid of Intellisense-only warnings (Code Analysis) + // Though __has_include is C++17, it is supported in Visual Studio 2017 or + // better (_MSC_VER>=1910). + #ifdef __has_include + #if __has_include(<CppCoreCheck\Warnings.h>) + #include <CppCoreCheck\Warnings.h> + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) + #endif + #endif + + #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING + #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning(pop)) + #define SIMDUTF_DISABLE_UNUSED_WARNING +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + #if defined(__OPTIMIZE__) || defined(NDEBUG) + #define simdutf_really_inline inline __attribute__((always_inline)) + #else + #define simdutf_really_inline inline + #endif + #define simdutf_always_inline \ + inline __attribute__((always_inline)) // always inline, no matter what + #define SIMDUTF_DEPRECATED __attribute__((deprecated)) + #define simdutf_never_inline inline __attribute__((noinline)) + + #define simdutf_unused __attribute__((unused)) + #define simdutf_warn_unused __attribute__((warn_unused_result)) + + #ifndef simdutf_likely + #define simdutf_likely(x) __builtin_expect(!!(x), 1) + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) + #endif + // clang-format off + #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") + // gcc doesn't seem to disable all warnings with all and extra, add warnings + // here as necessary + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) + #define SIMDUTF_PRAGMA(P) _Pragma(#P) + #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) \ + SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) + #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) + #else + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + #define SIMDUTF_DISABLE_DEPRECATED_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING \ + SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) + #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") + #define SIMDUTF_DISABLE_UNUSED_WARNING \ + SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-function) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-const-variable) + // clang-format on + +#endif // MSC_VER + +// Conditional constexpr macro: expands to constexpr for C++17+, empty otherwise +#if SIMDUTF_CPLUSPLUS17 + #define simdutf_constexpr constexpr +#else + #define simdutf_constexpr +#endif + +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + +#ifndef SIMDUTF_DLLIMPORTEXPORT + #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio + /** + * Windows users need to do some extra work when building + * or using a dynamic library (DLL). When building, we need + * to set SIMDUTF_DLLIMPORTEXPORT to __declspec(dllexport). + * When *using* the DLL, the user needs to set + * SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport). + * + * Static libraries not need require such work. + * + * It does not matter here whether you are using + * the regular visual studio or clang under visual + * studio, you still need to handle these issues. + * + * Non-Windows systems do not have this complexity. + */ + #if SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY + + // We set SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL + // under Windows. It should never happen that both + // SIMDUTF_BUILDING_WINDOWS_DYNAMIC_LIBRARY and + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY are set. + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) + #elif SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY + // Windows user who call a dynamic library should set + // SIMDUTF_USING_WINDOWS_DYNAMIC_LIBRARY to 1. + + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) + #else + // We assume by default static linkage + #define SIMDUTF_DLLIMPORTEXPORT + #endif + #else // defined(SIMDUTF_VISUAL_STUDIO) + // Non-Windows systems do not have this complexity. + #define SIMDUTF_DLLIMPORTEXPORT + #endif // defined(SIMDUTF_VISUAL_STUDIO) +#endif + +#if SIMDUTF_MAYBE_UNUSED_AVAILABLE + #define simdutf_maybe_unused [[maybe_unused]] +#else + #define simdutf_maybe_unused +#endif + +#endif // SIMDUTF_COMMON_DEFS_H +/* end file include/simdutf/common_defs.h */ +/* begin file include/simdutf/encoding_types.h */ +#ifndef SIMDUTF_ENCODING_TYPES_H +#define SIMDUTF_ENCODING_TYPES_H +#include <string> + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include <text_encoding> +#endif + +namespace simdutf { + +enum encoding_type { + UTF8 = 1, // BOM 0xef 0xbb 0xbf + UTF16_LE = 2, // BOM 0xff 0xfe + UTF16_BE = 4, // BOM 0xfe 0xff + UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00 + UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff + Latin1 = 32, + + unspecified = 0 +}; + +#ifndef SIMDUTF_IS_BIG_ENDIAN + #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." +#endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = +#if SIMDUTF_IS_BIG_ENDIAN + BIG +#else + LITTLE +#endif +}; + +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; +} + +simdutf_warn_unused std::string to_string(encoding_type bom); + +// Note that BOM for UTF8 is discouraged. +namespace BOM { + +/** + * Checks for a BOM. If not, returns unspecified + * @param input the string to process + * @param length the length of the string in code units + * @return the corresponding encoding + */ + +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); +/** + * Returns the size, in bytes, of the BOM for a given encoding type. + * Note that UTF8 BOM are discouraged. + * @param bom the encoding type + * @return the size in bytes of the corresponding BOM + */ +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); + +} // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + +} // namespace simdutf +#endif +/* end file include/simdutf/encoding_types.h */ +/* begin file include/simdutf/error.h */ +#ifndef SIMDUTF_ERROR_H +#define SIMDUTF_ERROR_H +namespace simdutf { + +enum error_code { + SUCCESS = 0, + HEADER_BITS, // Any byte must have fewer than 5 header bits. + TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, + // where N is the UTF-8 character length This is also the error + // when the input is truncated. + TOO_LONG, // We either have too many consecutive continuation bytes or the + // string starts with a continuation byte. + OVERLONG, // The decoded character must be above U+7F for two-byte characters, + // U+7FF for three-byte characters, and U+FFFF for four-byte + // characters. + TOO_LARGE, // The decoded character must be less than or equal to + // U+10FFFF,less than or equal than U+7F for ASCII OR less than + // equal than U+FF for Latin1 + SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or + // UTF-32) + // OR + // a high surrogate must be followed by a low surrogate + // and a low surrogate must be preceded by a high surrogate + // (UTF-16) + // OR + // there must be no surrogate at all and one is + // found (Latin1 functions) + // OR + // *specifically* for the function + // utf8_length_from_utf16_with_replacement, a surrogate (whether + // in error or not) has been found (I.e., whether we are in the + // Basic Multilingual Plane or not). + INVALID_BASE64_CHARACTER, // Found a character that cannot be part of a valid + // base64 string. This may include a misplaced + // padding character ('='). + BASE64_INPUT_REMAINDER, // The base64 input terminates with a single + // character, excluding padding (=). It is also used + // in strict mode when padding is not adequate. + BASE64_EXTRA_BITS, // The base64 input terminates with non-zero + // padding bits. + OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. + OTHER // Not related to validation/transcoding. +}; +#if SIMDUTF_CPLUSPLUS17 +inline std::string_view error_to_string(error_code code) noexcept { + switch (code) { + case SUCCESS: + return "SUCCESS"; + case HEADER_BITS: + return "HEADER_BITS"; + case TOO_SHORT: + return "TOO_SHORT"; + case TOO_LONG: + return "TOO_LONG"; + case OVERLONG: + return "OVERLONG"; + case TOO_LARGE: + return "TOO_LARGE"; + case SURROGATE: + return "SURROGATE"; + case INVALID_BASE64_CHARACTER: + return "INVALID_BASE64_CHARACTER"; + case BASE64_INPUT_REMAINDER: + return "BASE64_INPUT_REMAINDER"; + case BASE64_EXTRA_BITS: + return "BASE64_EXTRA_BITS"; + case OUTPUT_BUFFER_TOO_SMALL: + return "OUTPUT_BUFFER_TOO_SMALL"; + default: + return "OTHER"; + } +} +#endif + +struct result { + error_code error; + size_t count; // In case of error, indicates the position of the error. In + // case of success, indicates the number of code units + // validated/written. + + simdutf_really_inline simdutf_constexpr23 result() noexcept + : error{error_code::SUCCESS}, count{0} {} + + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept + : error{err}, count{pos} {} + + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { + return error == error_code::SUCCESS; + } + + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { + return error != error_code::SUCCESS; + } +}; + +struct full_result { + error_code error; + size_t input_count; + size_t output_count; + bool padding_error = false; // true if the error is due to padding, only + // meaningful when error is not SUCCESS + + simdutf_really_inline simdutf_constexpr23 full_result() noexcept + : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} + + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out} {} + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept + : error{err}, input_count{pos_in}, output_count{pos_out}, + padding_error{padding_err} {} + + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { + if (error == error_code::SUCCESS) { + return result{error, output_count}; + } else { + return result{error, input_count}; + } + } +}; + +} // namespace simdutf +#endif +/* end file include/simdutf/error.h */ + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + +// Public API +/* begin file include/simdutf/simdutf_version.h */ +// /include/simdutf/simdutf_version.h automatically generated by release.py, +// do not change by hand +#ifndef SIMDUTF_SIMDUTF_VERSION_H +#define SIMDUTF_SIMDUTF_VERSION_H + +/** The version of simdutf being used (major.minor.revision) */ +#define SIMDUTF_VERSION "8.0.0" + +namespace simdutf { +enum { + /** + * The major version (MAJOR.minor.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MAJOR = 8, + /** + * The minor version (major.MINOR.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MINOR = 0, + /** + * The revision (major.minor.REVISION) of simdutf being used. + */ + SIMDUTF_VERSION_REVISION = 0 +}; +} // namespace simdutf + +#endif // SIMDUTF_SIMDUTF_VERSION_H +/* end file include/simdutf/simdutf_version.h */ +/* begin file include/simdutf/implementation.h */ +#ifndef SIMDUTF_IMPLEMENTATION_H +#define SIMDUTF_IMPLEMENTATION_H +#if !defined(SIMDUTF_NO_THREADS) + #include <atomic> +#endif +#include <string> +#ifdef SIMDUTF_INTERNAL_TESTS + #include <vector> +#endif +/* begin file include/simdutf/internal/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDutf_INTERNAL_ISADETECTION_H +#define SIMDutf_INTERNAL_ISADETECTION_H + +#include <cstdint> +#include <cstdlib> +#if defined(_MSC_VER) + #include <intrin.h> +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + #include <cpuid.h> +#endif + + +// RISC-V ISA detection utilities +#if SIMDUTF_IS_RISCV64 && defined(__linux__) + #include <unistd.h> // for syscall +// We define these ourselves, for backwards compatibility +struct simdutf_riscv_hwprobe { + int64_t key; + uint64_t value; +}; + #define simdutf_riscv_hwprobe(...) syscall(258, __VA_ARGS__) + #define SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0 4 + #define SIMDUTF_RISCV_HWPROBE_IMA_V (1 << 2) + #define SIMDUTF_RISCV_HWPROBE_EXT_ZVBB (1 << 17) +#endif // SIMDUTF_IS_RISCV64 && defined(__linux__) + +#if defined(__loongarch__) && defined(__linux__) + #include <sys/auxv.h> +// bits/hwcap.h +// #define HWCAP_LOONGARCH_LSX (1 << 4) +// #define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +namespace simdutf { +namespace internal { + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40, + ALTIVEC = 0x80, + AVX512F = 0x100, + AVX512DQ = 0x200, + AVX512IFMA = 0x400, + AVX512PF = 0x800, + AVX512ER = 0x1000, + AVX512CD = 0x2000, + AVX512BW = 0x4000, + AVX512VL = 0x8000, + AVX512VBMI2 = 0x10000, + AVX512VPOPCNTDQ = 0x2000, + RVV = 0x4000, + ZVBB = 0x8000, + LSX = 0x40000, + LASX = 0x80000, +}; + +#if defined(__PPC64__) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::ALTIVEC; +} + +#elif SIMDUTF_IS_RISCV64 + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if SIMDUTF_IS_RVV + host_isa |= instruction_set::RVV; + #endif + #if SIMDUTF_IS_ZVBB + host_isa |= instruction_set::ZVBB; + #endif + #if defined(__linux__) + simdutf_riscv_hwprobe probes[] = {{SIMDUTF_RISCV_HWPROBE_KEY_IMA_EXT_0, 0}}; + long ret = simdutf_riscv_hwprobe(&probes, sizeof probes / sizeof *probes, 0, + nullptr, 0); + if (ret == 0) { + uint64_t extensions = probes[0].value; + if (extensions & SIMDUTF_RISCV_HWPROBE_IMA_V) + host_isa |= instruction_set::RVV; + if (extensions & SIMDUTF_RISCV_HWPROBE_EXT_ZVBB) + host_isa |= instruction_set::ZVBB; + } + #endif + #if defined(RUN_IN_SPIKE_SIMULATOR) + // Proxy Kernel does not implement yet hwprobe syscall + host_isa |= instruction_set::RVV; + #endif + return host_isa; +} + +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#elif defined(__x86_64__) || defined(_M_AMD64) // x64 + +namespace { +namespace cpuid_bit { +// Can be found on Intel ISA Reference for CPUID + +// EAX = 0x01 +constexpr uint32_t pclmulqdq = uint32_t(1) + << 1; ///< @private bit 1 of ECX for EAX=0x1 +constexpr uint32_t sse42 = uint32_t(1) + << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t osxsave = + (uint32_t(1) << 26) | + (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + +// EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf) +// See: "Table 3-8. Information Returned by CPUID Instruction" +namespace ebx { +constexpr uint32_t bmi1 = uint32_t(1) << 3; +constexpr uint32_t avx2 = uint32_t(1) << 5; +constexpr uint32_t bmi2 = uint32_t(1) << 8; +constexpr uint32_t avx512f = uint32_t(1) << 16; +constexpr uint32_t avx512dq = uint32_t(1) << 17; +constexpr uint32_t avx512ifma = uint32_t(1) << 21; +constexpr uint32_t avx512cd = uint32_t(1) << 28; +constexpr uint32_t avx512bw = uint32_t(1) << 30; +constexpr uint32_t avx512vl = uint32_t(1) << 31; +} // namespace ebx + +namespace ecx { +constexpr uint32_t avx512vbmi = uint32_t(1) << 1; +constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6; +constexpr uint32_t avx512vnni = uint32_t(1) << 11; +constexpr uint32_t avx512bitalg = uint32_t(1) << 12; +constexpr uint32_t avx512vpopcnt = uint32_t(1) << 14; +} // namespace ecx +namespace edx { +constexpr uint32_t avx512vp2intersect = uint32_t(1) << 8; +} +namespace xcr0_bit { +constexpr uint64_t avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX +constexpr uint64_t avx512_saved = + uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM +} // namespace xcr0_bit +} // namespace cpuid_bit +} // namespace + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { + #if defined(_MSC_VER) + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; + #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); + #else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; + #endif +} + +static inline uint64_t xgetbv() { + #if defined(_MSC_VER) + return _xgetbv(0); + #else + uint32_t xcr0_lo, xcr0_hi; + asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); + #endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax; + uint32_t ebx = 0; + uint32_t ecx = 0; + uint32_t edx = 0; + uint32_t host_isa = 0x0; + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_bit::sse42) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_bit::pclmulqdq) { + host_isa |= instruction_set::PCLMULQDQ; + } + + if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) { + return host_isa; + } + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; // Sub-leaf = 0 + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_bit::ebx::avx2) { + host_isa |= instruction_set::AVX2; + } + if (ebx & cpuid_bit::ebx::bmi1) { + host_isa |= instruction_set::BMI1; + } + if (ebx & cpuid_bit::ebx::bmi2) { + host_isa |= instruction_set::BMI2; + } + if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == + cpuid_bit::xcr0_bit::avx512_saved)) { + return host_isa; + } + if (ebx & cpuid_bit::ebx::avx512f) { + host_isa |= instruction_set::AVX512F; + } + if (ebx & cpuid_bit::ebx::avx512bw) { + host_isa |= instruction_set::AVX512BW; + } + if (ebx & cpuid_bit::ebx::avx512cd) { + host_isa |= instruction_set::AVX512CD; + } + if (ebx & cpuid_bit::ebx::avx512dq) { + host_isa |= instruction_set::AVX512DQ; + } + if (ebx & cpuid_bit::ebx::avx512vl) { + host_isa |= instruction_set::AVX512VL; + } + if (ecx & cpuid_bit::ecx::avx512vbmi2) { + host_isa |= instruction_set::AVX512VBMI2; + } + if (ecx & cpuid_bit::ecx::avx512vpopcnt) { + host_isa |= instruction_set::AVX512VPOPCNTDQ; + } + return host_isa; +} +#elif defined(__loongarch__) + +static inline uint32_t detect_supported_architectures() { + uint32_t host_isa = instruction_set::DEFAULT; + #if defined(__linux__) + uint64_t hwcap = 0; + hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_LOONGARCH_LSX) { + host_isa |= instruction_set::LSX; + } + if (hwcap & HWCAP_LOONGARCH_LASX) { + host_isa |= instruction_set::LASX; + } + #endif + return host_isa; +} +#else // fallback + +// includes 32-bit ARM. +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif // end SIMD extension detection code + +} // namespace internal +} // namespace simdutf + +#endif // SIMDutf_INTERNAL_ISADETECTION_H +/* end file include/simdutf/internal/isadetection.h */ + +#if SIMDUTF_SPAN + #include <concepts> + #include <type_traits> + #include <span> + #include <tuple> +#endif +#if SIMDUTF_CPLUSPLUS17 + #include <string_view> +#endif +// The following defines are conditionally enabled/disabled during amalgamation. +// By default all features are enabled, regular code shouldn't check them. Only +// when user code really relies of a selected subset, it's good to verify these +// flags, like: +// +// #if !SIMDUTF_FEATURE_UTF16 +// # error("Please amalgamate simdutf with UTF-16 support") +// #endif +// +#define SIMDUTF_FEATURE_DETECT_ENCODING 1 +#define SIMDUTF_FEATURE_ASCII 1 +#define SIMDUTF_FEATURE_LATIN1 1 +#define SIMDUTF_FEATURE_UTF8 1 +#define SIMDUTF_FEATURE_UTF16 1 +#define SIMDUTF_FEATURE_UTF32 1 +#define SIMDUTF_FEATURE_BASE64 1 + +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include <cstddef> + +namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template <typename to, typename from> + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; + + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast<to>(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast<to>(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template <typename to, typename from> +constexpr constexpr_ptr<to, from> constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr<to, from>{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template <typename SrcType, typename TargetType> +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast<TargetType>(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template <typename SrcType, typename TargetType> struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy<SrcType, TargetType> operator*() const { + return constexpr_write_ptr_proxy<SrcType, TargetType>{p}; + } + + constexpr constexpr_write_ptr_proxy<SrcType, TargetType> + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy<SrcType, TargetType>{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template <typename SrcType, typename TargetType> +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr<SrcType, TargetType>{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif + +#if SIMDUTF_SPAN +/// helpers placed in namespace detail are not a part of the public API +namespace simdutf { +namespace detail { +/** + * matches a byte, in the many ways C++ allows. note that these + * are all distinct types. + */ +template <typename T> +concept byte_like = std::is_same_v<T, std::byte> || // + std::is_same_v<T, char> || // + std::is_same_v<T, signed char> || // + std::is_same_v<T, unsigned char> || // + std::is_same_v<T, char8_t>; + +template <typename T> +concept is_byte_like = byte_like<std::remove_cvref_t<T>>; + +template <typename T> +concept is_pointer = std::is_pointer_v<T>; + +/** + * matches anything that behaves like std::span and points to character-like + * data such as: std::byte, char, unsigned char, signed char, std::int8_t, + * std::uint8_t + */ +template <typename T> +concept input_span_of_byte_like = requires(const T &t) { + { t.size() } noexcept -> std::convertible_to<std::size_t>; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; +}; + +template <typename T> +concept is_mutable = !std::is_const_v<std::remove_reference_t<T>>; + +/** + * like span_of_byte_like, but for an output span (intended to be written to) + */ +template <typename T> +concept output_span_of_byte_like = requires(T &t) { + { t.size() } noexcept -> std::convertible_to<std::size_t>; + { t.data() } noexcept -> is_pointer; + { *t.data() } noexcept -> is_byte_like; + { *t.data() } noexcept -> is_mutable; +}; + +/** + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array<char,10> + * invalid examples: int*, std::array<int,10> + */ +template <class InputPtr> +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> simdutf::detail::byte_like; +}; +template <class InputPtr> +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<char16_t>; +}; +template <class InputPtr> +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<char32_t>; +}; + +template <class InputPtr> +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; + +/** + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* + */ +template <class InputPtr> +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t<decltype(p[0])>{} } -> std::same_as<std::uint32_t>; +}; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN + +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); +} + +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 +} + +namespace utf32 { +template <endianness big_endian> constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; +} +} // namespace utf32 + +namespace utf16 { +template <endianness big_endian> constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; +} +} // namespace utf16 + +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template <class InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template <class InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast<std::uint8_t>(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include <atomic> +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast<char *>(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref<char>(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast<std::uintptr_t>(src) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast<uint64_t *>(const_cast<char *>(src)); + const auto dst_value = + std::atomic_ref<uint64_t>(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref<char>(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast<std::uintptr_t>(dst) % alignment; + if (offset) { + size_t to_align = std::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast<uint64_t *>(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref<uint64_t>(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast<const uint8_t *>(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template <endianness big_endian> +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast<const unsigned char *>(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast<const unsigned char *>(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast<const unsigned char *>(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast<uint8_t>(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template <endianness big_endian> +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template <endianness big_endian> +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template <endianness big_endian> +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template <endianness big_endian> +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template <endianness big_endian> +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter++; // ASCII + counter += static_cast<size_t>( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast<size_t>((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template <endianness big_endian> +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template <endianness big_endian> +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed<big_endian>(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template <endianness big_endian> +simdutf_constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed<big_endian>(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template <endianness big_endian> +simdutf_constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed<big_endian>(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +simdutf_really_inline constexpr bool low_surrogate(char16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +template <endianness big_endian> +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate<big_endian>(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate<big_endian>(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate<big_endian>(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast<size_t>(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast<size_t>(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template <endianness big_endian> constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template <endianness big_endian> +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement<big_endian>(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate<big_endian>(c); + low_surrogate = is_low_surrogate<big_endian>(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include <cstring> // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if simdutf_constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if simdutf_constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template <endianness big_endian, class InputIterator, class OutputIterator> +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template <endianness big_endian> +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl<big_endian>(reinterpret_cast<const uint16_t *>(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template <endianness big_endian> +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16<InputPtr> +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template <endianness big_endian, bool check_output = false, typename InputPtr, + typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template <endianness big_endian> +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors<big_endian, false>(buf, len, utf8_output, 0); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template <endianness big_endian, typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if simdutf_constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast<const uint32_t *>(buf), len); +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast<const uint32_t *>(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast<size_t>(p[i] > 0x7F); // two-byte + counter += static_cast<size_t>(p[i] > 0x7FF); // three-byte + counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast<size_t>(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template <typename ReadPtr, typename WritePtr> +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast<const uint32_t *>(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template <endianness big_endian> +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template <endianness big_endian> +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32<InputPtr> && + simdutf::detail::index_assignable_from_char<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template <class BytePtr> +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast<const uint8_t *>(buf), len); +} + +template <class BytePtr> +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; + return res; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template <typename InputPtr, typename OutputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like<InputPtr> && + simdutf::detail::indexes_into_byte_like<OutputPtr>) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point)) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if simdutf_constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template <endianness endian> +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors<endian>(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template <endianness big_endian, typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if simdutf_constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if simdutf_constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast<std::ptrdiff_t>(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described + // in C Standard as <stddef.h>. C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template <typename InputPtr> +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like<InputPtr> +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + +#if SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const char *input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const uint8_t *input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast<const char *>(input), length); +} + #if SIMDUTF_SPAN +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. can be a anything span-like that has a + * data() and size() that points to character data: std::string, + * std::string_view, std::vector<char>, std::span<const std::byte> etc. + * @return the detected encoding type + */ +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding( + const detail::input_span_of_byte_like auto &input) noexcept { + return autodetect_encoding(reinterpret_cast<const char *>(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN + +/** + * Autodetect the possible encodings of the input in one pass. + * E.g., if the input might be UTF-16LE or UTF-8, this function returns + * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). + * + * Overridden by each implementation. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused int detect_encodings(const char *input, + size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const uint8_t *input, size_t length) noexcept { + return detect_encodings(reinterpret_cast<const char *>(input), length); +} + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { + return detect_encodings(reinterpret_cast<const char *>(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-8 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf8_with_errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool +validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf8(reinterpret_cast<const char *>(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 +/** + * Validate the UTF-8 string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf8_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +validate_utf8_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate_with_errors( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf8_with_errors( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate( + detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_ascii(reinterpret_cast<const char *>(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string and stop on error. It might be faster than + * validate_utf8 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_ascii_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_ascii_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate_with_errors( + detail::constexpr_cast_ptr<std::uint8_t>(input.data()), input.size()); + } else + #endif + { + return validate_ascii_with_errors( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string as a UTF-16 sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::NATIVE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::BIG>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ +simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16le_as_ascii(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii<endianness::LITTLE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le_as_ascii(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Validate the UTF-16 string. + * This function may be best when you expect the input to be almost always + * valid. Otherwise, consider using validate_utf16_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::NATIVE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-16LE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ +simdutf_warn_unused bool validate_utf16le(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool +validate_utf16le(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::LITTLE>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 +/** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ +simdutf_warn_unused bool validate_utf16be(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf16be(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate<endianness::BIG>(input.data(), input.size()); + } else + #endif + { + return validate_utf16be(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Validate the UTF-16 string and stop on error. + * It might be faster than validate_utf16 when an error is expected to occur + * early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::NATIVE>( + input.data(), input.size()); + } else + #endif + { + return validate_utf16_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16le_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::LITTLE>( + input.data(), input.size()); + } else + #endif + { + return validate_utf16le_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf16be_with_errors(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors<endianness::BIG>(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16le(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::LITTLE>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16le(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with + * the Unicode replacement character U+FFFD. If input and output points to + * different memory areas, the procedure copies string, and it's expected that + * output memory is at least as big as the input. It's also possible to set + * input equal output, that makes replacements an in-place operation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16be(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::BIG>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16be(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. If input and output points to different + * memory areas, the procedure copies string, and it's expected that output + * memory is at least as big as the input. It's also possible to set input equal + * output, that makes replacements an in-place operation. + * + * @param input the UTF-16 string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ +void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +to_well_formed_utf16(std::span<const char16_t> input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16(input.data(), input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-32 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf32_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ +simdutf_warn_unused bool validate_utf32(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool +validate_utf32(std::span<const char32_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate( + detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf32(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 +/** + * Validate the UTF-32 string and stop on error. It might be faster than + * validate_utf32 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_utf32_with_errors(std::span<const char32_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate_with_errors( + detail::constexpr_cast_ptr<std::uint32_t>(input.data()), input.size()); + } else + #endif + { + return validate_utf32_with_errors(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input, + size_t length, + char *utf8_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8( + const detail::input_span_of_byte_like auto &latin1_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert( + detail::constexpr_cast_ptr<char>(latin1_input.data()), + latin1_input.size(), + detail::constexpr_cast_writeptr<char>(utf8_output.data())); + } else + #endif + { + return convert_latin1_to_utf8( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-8 string with output limit. + * + * This function is suitable to work with inputs from untrusted sources. + * + * We write as many characters as possible. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t +convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert_safe_constexpr( + input.data(), input.size(), utf8_output.data(), utf8_output.size()); + } else + #endif + { + return convert_latin1_to_utf8_safe( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16le( + const detail::input_span_of_byte_like auto &latin1_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::LITTLE>( + latin1_input.data(), latin1_input.size(), utf16_output.data()); + } else + #endif + { + return convert_latin1_to_utf16le( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::BIG>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16be( + reinterpret_cast<const char *>(input.data()), input.size(), + output.data()); + } +} + #endif // SIMDUTF_SPAN +/** + * Compute the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in Latin1 code units (char) required to + * encode the UTF-16 string as Latin1 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf16(size_t length) noexcept { + return length; +} + +/** + * Compute the number of code units that this Latin1 string would require in + * UTF-16 format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 2-byte code units (char16_t) required to + * encode the Latin1 string as UTF-16 + */ +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_latin1_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf32( + const detail::input_span_of_byte_like auto &latin1_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf32::convert( + latin1_input.data(), latin1_input.size(), utf32_output.data()); + } else + #endif + { + return convert_latin1_to_utf32( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input, + size_t length, + char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_latin1( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert(input.data(), input.size(), + output.data()); + } else + #endif + { + return convert_utf8_to_latin1(reinterpret_cast<const char *>(input.data()), + input.size(), + reinterpret_cast<char *>(output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_utf8_to_utf16(reinterpret_cast<const char *>(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format even when the UTF-16LE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +utf8_length_from_utf16le_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format even when the UTF-16BE content contains mismatched surrogates + * that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16be_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert a Latin1 string into a UTF-16 string. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t. + */ +simdutf_warn_unused size_t convert_latin1_to_utf16( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, + std::span<char16_t> output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert<endianness::NATIVE>( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16(reinterpret_cast<const char *>(input.data()), + input.size(), output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::LITTLE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert<endianness::BIG>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_latin1_with_errors( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_latin1_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_with_errors( + utf8_input.data(), utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf8_to_latin1_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-8 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::NATIVE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16le_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::LITTLE>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, char16_t *utf16_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf16be_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors<endianness::BIG>( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if the input was not valid UTF-8 + * string + */ +simdutf_warn_unused size_t convert_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf8_to_utf32_with_errors( + const char *input, size_t length, char32_t *utf32_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf8_to_utf32_with_errors( + const detail::input_span_of_byte_like auto &utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32_with_errors( + reinterpret_cast<const char *>(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_valid_utf8_to_latin1( + const char *input, size_t length, char *latin1_output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_latin1( + const detail::input_span_of_byte_like auto &valid_utf8_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_valid_utf8_to_latin1( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), latin1_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert valid UTF-8 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::NATIVE>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16le( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::LITTLE>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16le( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( + const char *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16be( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid<endianness::BIG>( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16be( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf32( + const char *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf32( + const detail::input_span_of_byte_like auto &valid_utf8_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf32( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +/** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_latin1(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_latin1( + const detail::input_span_of_byte_like auto &latin1_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(), + latin1_input.size()); + } else + #endif + { + return utf8_length_from_latin1( + reinterpret_cast<const char *>(latin1_input.data()), + latin1_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ +simdutf_warn_unused size_t latin1_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return latin1_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Compute the number of 2-byte code units that this UTF-8 string would require + * in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 string + * as UTF-16LE + */ +simdutf_warn_unused size_t utf16_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf16_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of 4-byte code units that this UTF-8 string would require + * in UTF-32 format. + * + * This function is equivalent to count_utf8 + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 string + * as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf32_length_from_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string with output limit. + * + * We write as many characters as possible into the output buffer, + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 16-bit code units (char16_t) + * @param utf8_output the pointer to buffer that can hold conversion result + * @param utf8_len the maximum output length + * @return the number of written char; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input, + size_t length, + char *utf8_output, + size_t utf8_len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_safe( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + const full_result r = + scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE, true>( + utf16_input.data(), utf16_input.size(), utf8_output.data(), + utf8_output.size()); + if (r.error != error_code::SUCCESS && + r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + return 0; + } + return r.output_count; + } else + #endif + { + return convert_utf16_to_utf8_safe( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data()), utf8_output.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_latin1( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert possibly broken UTF-16 string into Latin1 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_latin1_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-8 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf8_with_errors( + std::span<const char16_t> utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +/** + * Using native endianness, convert UTF-16 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::NATIVE>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16LE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16le_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::LITTLE>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16le_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-16BE and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. The function may be removed from the + * library in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( + const char16_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf16be_to_latin1( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl<endianness::BIG>( + detail::constexpr_cast_ptr<uint16_t>(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16be_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf8( + std::span<const char16_t> valid_utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-16 string into UTF-32 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ +simdutf_warn_unused size_t convert_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf32(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-16 string into + * UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16_to_utf32_with_errors(std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::NATIVE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16le_to_utf32_with_errors( + std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::LITTLE>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ +simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf16be_to_utf32_with_errors( + std::span<const char16_t> utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors<endianness::BIG>( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-16 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16le_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-16BE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16be_to_utf32( + const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf32(std::span<const char16_t> valid_utf16_input, + std::span<char32_t> utf32_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; compute the number of bytes that this UTF-16 + * string would require in UTF-8 format even when the UTF-16LE content contains + * mismatched surrogates that have to be replaced by the replacement character + * (0xFFFD). + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or + * SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of this + * function: it indicates that at least one surrogate has been encountered: the + * surrogates may be matched or not (thus this function does not validate). If + * the returned error code is SUCCESS, then the input contains no surrogate, is + * in the Basic Multilingual Plane, and is necessarily valid. + */ +simdutf_warn_unused result utf8_length_from_utf16_with_replacement( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input, + size_t length, + char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf8( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf8_with_errors( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf8_with_errors( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf8( + const char32_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf8( + std::span<const char32_t> valid_utf32_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf8( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast<char *>(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Using native endianness, convert possibly broken UTF-32 string into a UTF-16 + * string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::NATIVE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16le(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::LITTLE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +/** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + * or if it cannot be represented as Latin1 + */ +simdutf_warn_unused size_t convert_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_latin1( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_latin1_with_errors( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_latin1_with_errors( + std::span<const char32_t> utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_with_errors( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. The function may be removed from the library + * in the future. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_latin1( + const char32_t *input, size_t length, char *latin1_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf32_to_latin1( + std::span<const char32_t> valid_utf32_input, + detail::output_span_of_byte_like auto &&latin1_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_valid( + detail::constexpr_cast_ptr<uint32_t>(valid_utf32_input.data()), + valid_utf32_input.size(), + detail::constexpr_cast_writeptr<char>(latin1_output.data())); + } + #endif + { + return convert_valid_utf32_to_latin1( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast<char *>(latin1_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +latin1_length_from_utf32(size_t length) noexcept { + return length; +} + +/** + * Compute the number of bytes that this Latin1 string would require in UTF-32 + * format. + * + * @param length the length of the string in Latin1 code units (char) + * @return the length of the string in 4-byte code units (char32_t) required to + * encode the Latin1 string as UTF-32 + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +utf32_length_from_latin1(size_t length) noexcept { + return length; +} +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf16be(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert<endianness::BIG>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert possibly broken UTF-32 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16_with_errors(std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::NATIVE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16le_with_errors( + std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::LITTLE>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +convert_utf32_to_utf16be_with_errors( + std::span<const char32_t> utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors<endianness::BIG>( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness, convert valid UTF-32 string into a UTF-16 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::NATIVE>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16le( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16le(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::LITTLE>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16le(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16be( + const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf16be(std::span<const char32_t> valid_utf32_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid<endianness::BIG>( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16be(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ +void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(std::span<const char16_t> utf16_input, + std::span<char16_t> utf16_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::change_endianness_utf16( + utf16_input.data(), utf16_input.size(), utf16_output.data()); + } else + #endif + { + return change_endianness_utf16(utf16_input.data(), utf16_input.size(), + utf16_output.data()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +/** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ +simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(std::span<const char32_t> valid_utf32_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-32 format. + * + * This function is equivalent to count_utf16. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16le( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 + * format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16be( + std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16 (native + * endianness). It is acceptable to pass invalid UTF-16 strings but in such + * cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16le(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16be(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +count_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ +simdutf_warn_unused size_t count_utf8(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return count_utf8(reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-8 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so + * that the short UTF-8 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-8, but possibly + * truncated. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 to 3 bytes + */ +simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8( + const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return trim_partial_utf8( + reinterpret_cast<const char *>(valid_utf8_input.data()), + valid_utf8_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_UTF16 +/** + * Given a valid UTF-16BE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16BE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16BE, but possibly + * truncated. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in bytes + * @return the length of the string in bytes, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16be(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::BIG>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16LE string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16LE strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16LE, but possibly + * truncated. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16le(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::LITTLE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16, but possibly + * truncated. We use the native endianness. + * + * @param input the UTF-16 string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(std::span<const char16_t> valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16<endianness::NATIVE>( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || \ + SIMDUTF_FEATURE_DETECT_ENCODING + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif +#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_BASE64 +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include <cstdint> + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <iostream> + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template <class char_type> bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template <class char_type> simdutf_constexpr23 bool is_eight_byte(char_type c) { + if simdutf_constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template <class char_type> +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template <class char_type> +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template <class char_type> +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template <class char_type> +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template <class char_type> +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template <bool check_capacity, class char_type> +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>(x & 0xFF); + *dst++ = static_cast<char>((x >> 8) & 0xFF); + *dst++ = static_cast<char>((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + *dst++ = static_cast<char>((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast<char>((triple >> 16) & 0xFF); + *dst++ = static_cast<char>((triple >> 8) & 0xFF); + *dst++ = static_cast<char>(triple & 0xFF); + } +} + +template <class char_type> +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl<false>(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template <class char_type> +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl<true>(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template <bool use_lines = false> +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if simdutf_constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if simdutf_constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if simdutf_constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +inline simdutf_constexpr23 size_t tail_encode_base64(char *dst, const char *src, + size_t srclen, + base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template <class InputPtr> +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +template <typename char_type> +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template <typename char_type> +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template <typename char_type> +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen +} + +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(base64_options options) { + switch (options) { + case base64_default: + return "base64_default"; + case base64_url: + return "base64_url"; + case base64_reverse_padding: + return "base64_reverse_padding"; + case base64_url_with_padding: + return "base64_url_with_padding"; + case base64_default_accept_garbage: + return "base64_default_accept_garbage"; + case base64_url_accept_garbage: + return "base64_url_accept_garbage"; + case base64_default_or_url: + return "base64_default_or_url"; + case base64_default_or_url_accept_garbage: + return "base64_default_or_url_accept_garbage"; + } + return "<unknown>"; +} + #endif // SIMDUTF_CPLUSPLUS17 + + #if SIMDUTF_CPLUSPLUS17 +inline std::string_view to_string(last_chunk_handling_options options) { + switch (options) { + case loose: + return "loose"; + case strict: + return "strict"; + case stop_before_partial: + return "stop_before_partial"; + case only_full_chunks: + return "only_full_chunks"; + } + return "<unknown>"; +} + #endif + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximum number of binary bytes + */ +simdutf_warn_unused size_t +maximal_binary_length_from_base64(const char *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr<uint8_t>(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast<const char *>(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ +simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(std::span<const char16_t> input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, usually base64_default or + * base64_url, and base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if successful. + */ +simdutf_warn_unused result base64_to_binary( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast<const char *>(input.data()), + input.size(), + reinterpret_cast<char *>(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} + +/** + * Provide the base64 length in bytes given the length of a binary input, + * taking into account line breaks. + * + * @param length the length of the input in bytes + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @return number of base64 bytes + */ +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( + size_t length, base64_options options = base64_default, + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} + +/** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a binary input to a base64 output with line breaks. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of lines, must be at least 4 (otherwise it is + * interpreted as 4), + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options) + */ +size_t +binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl<true>( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), line_length, options); + } +} + #endif // SIMDUTF_SPAN + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a binary input to a base64 output, using atomic accesses. + * This function comes with a potentially significant performance + * penalty, but it may be useful in some cases where the input + * buffers are shared between threads, to avoid undefined + * behavior in case of data races. + * + * The function is for advanced users. Its main use case is when + * to silence sanitizer warnings. We have no documented use case + * where this function is actually necessary in terms of practical correctness. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * The default option (simdutf::base64_default) uses the characters `+` and `/` + * as part of its alphabet. Further, it adds padding (`=`) at the end of the + * output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as part + * of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @brief atomic_binary_to_base64 + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t +atomic_binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused size_t +atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + return atomic_binary_to_base64( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), options); +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail + * to provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of bytes written if successful. + */ +simdutf_warn_unused result +base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span<const char16_t> input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast<char *>(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Check if a character is an ignorable base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is an ignorable base64 character, false + * otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} + +/** + * Check if a character is a valid base64 character. + * Checking a large input, character by character, is not computationally + * efficient. + * Note that padding characters are not considered valid base64 characters in + * this context, nor are spaces. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} + +/** + * Check if a character is a valid base64 character or the padding character + * ('='). Checking a large input, character by character, is not computationally + * efficient. + * + * @param input the character to check + * @param options the base64 options to use, is base64_default by default. + * @return true if the character is a base64 character, false otherwise. + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid_or_padding(char16_t input, + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are three possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer + * is too small (OUTPUT_BUFFER_TOO_SMALL). + * + * When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written + * and the number of units processed, see description of the parameters and + * returned value. + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected + * to discard the output unless the parameter decode_up_to_bad_char is set to + * true. In that case, the function will decode up to the first invalid + * character. Extra padding characters ('=') are considered invalid characters. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 8-bit + * or 16-bit units + * @param length the length of the string in 8-bit or 16-bit units. + * @param output the pointer to a buffer that can hold the conversion + * result. + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of units processed if successful. + */ +simdutf_warn_unused result +base64_to_binary_safe(const char *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +// the span overload has moved to the bottom of the file + +simdutf_warn_unused result +base64_to_binary_safe(const char16_t *input, size_t length, char *output, + size_t &outlen, base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; + // span overload moved to bottom of file + + #if SIMDUTF_ATOMIC_REF +/** + * Convert a base64 input to a binary output with a size limit and using atomic + * operations. + * + * Like `base64_to_binary_safe` but using atomic operations, this function is + * thread-safe for concurrent memory access, allowing the output + * buffers to be shared between threads without undefined behavior in case of + * data races. + * + * This function comes with a potentially significant performance penalty, but + * is useful when thread safety is needed during base64 decoding. + * + * This function is only available when simdutf is compiled with + * C++20 support and __cpp_lib_atomic_ref >= 201806L. You may check + * the availability of this function by checking the macro + * SIMDUTF_ATOMIC_REF. + * + * This function is considered experimental. It is not tested by default + * (see the CMake option SIMDUTF_ATOMIC_BASE64_TESTS) nor is it fuzz tested. + * It is not documented in the public API documentation (README). It is + * offered on a best effort basis. We rely on the community for further + * testing and feedback. + * + * @param input the base64 input to decode + * @param length the length of the input in bytes + * @param output the pointer to buffer that can hold the conversion + * result + * @param outlen the number of bytes that can be written in the output + * buffer. Upon return, it is modified to reflect how many bytes were written. + * @param options the base64 options to use (default, url, etc.) + * @param last_chunk_options the last chunk handling options (loose, strict, + * stop_before_partial) + * @param decode_up_to_bad_char if true, the function will decode up to the + * first invalid character. By default (false), it is assumed that the output + * buffer is to be discarded. When there are multiple errors in the input, + * using decode_up_to_bad_char might trigger a different error. + * @return a result struct with an error code and count indicating error + * position or success + */ +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept; +simdutf_warn_unused result atomic_base64_to_binary_safe( + const char16_t *input, size_t length, char *output, size_t &outlen, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept; + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline simdutf_warn_unused std::tuple<result, std::size_t> +atomic_base64_to_binary_safe( + const detail::input_span_of_byte_like auto &binary_input, + detail::output_span_of_byte_like auto &&output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = output.size(); + auto ret = atomic_base64_to_binary_safe( + reinterpret_cast<const char *>(binary_input.data()), binary_input.size(), + reinterpret_cast<char *>(output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_warn_unused std::tuple<result, std::size_t> +atomic_base64_to_binary_safe( + std::span<const char16_t> base64_input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + auto ret = atomic_base64_to_binary_safe( + base64_input.data(), base64_input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {ret, outlen}; +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +#endif // SIMDUTF_FEATURE_BASE64 + +/** + * An implementation of simdutf for a particular CPU architecture. + * + * Also used to maintain the currently active implementation. The active + * implementation is automatically initialized on first use to the most advanced + * implementation supported by the host. + */ +class implementation { +public: + /** + * The name of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string name() const { return std::string(_name); } + + /** + * The description of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << + * impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual std::string description() const { return std::string(_description); } + + /** + * The instruction sets this implementation is compiled against + * and the current CPU match. This function may poll the current CPU/system + * and should therefore not be called too often if performance is a concern. + * + * + * @return true if the implementation can be safely used on the current system + * (determined at runtime) + */ + bool supported_by_runtime_system() const; + +#if SIMDUTF_FEATURE_DETECT_ENCODING + /** + * This function will try to detect the encoding + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual encoding_type autodetect_encoding(const char *input, + size_t length) const noexcept; + + /** + * This function will try to detect the possible encodings in one pass + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual int detect_encodings(const char *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + + /** + * @private For internal implementation use + * + * The instruction sets this implementation is compiled against. + * + * @return a mask of all required `internal::instruction_set::` values + */ + virtual uint32_t required_instruction_sets() const { + return _required_instruction_sets; + } + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ + simdutf_warn_unused virtual bool validate_utf8(const char *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 + /** + * Validate the UTF-8 string and stop on errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf8_with_errors(const char *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_ascii(const char *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_ascii_with_errors(const char *buf, size_t len) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + /** + * Validate the ASCII string as a UTF-16BE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the ASCII string as a UTF-16LE sequence. + * An UTF-16 sequence is considered an ASCII sequence + * if it could be converted to an ASCII string losslessly. + * + * Overridden by each implementation. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ + simdutf_warn_unused virtual bool + validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII + +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-16LE string.This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16le_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ + simdutf_warn_unused virtual bool + validate_utf16le(const char16_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF16 + /** + * Validate the UTF-16BE string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf16be_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return true if and only if the string is valid UTF-16BE. + */ + simdutf_warn_unused virtual bool + validate_utf16be(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Validate the UTF-16LE string and stop on error. It might be faster than + * validate_utf16le when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16le_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + + /** + * Validate the UTF-16BE string and stop on error. It might be faster than + * validate_utf16be when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16BE string to validate. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf16be_with_errors(const char16_t *buf, + size_t len) const noexcept = 0; + /** + * Copies the UTF-16LE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16LE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; + /** + * Copies the UTF-16BE string while replacing mismatched surrogates with the + * Unicode replacement character U+FFFD. We allow the input and output to be + * the same buffer so that the correction is done in-place. + * + * Overridden by each implementation. + * + * @param input the UTF-16BE string to correct. + * @param len the length of the string in number of 2-byte code units + * (char16_t). + * @param output the output buffer. + */ + virtual void to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + /** + * Validate the UTF-32 string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return true if and only if the string is valid UTF-32. + */ + simdutf_warn_unused virtual bool + validate_utf32(const char32_t *buf, size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF32 + /** + * Validate the UTF-32 string and stop on error. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-32 string to validate. + * @param len the length of the string in number of 4-byte code units + * (char32_t). + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + validate_utf32_with_errors(const char32_t *buf, + size_t len) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-8 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf8_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf8(const char *input, size_t length, + char *utf8_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly Latin1 string into UTF-16LE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert Latin1 string into UTF-16BE string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert Latin1 string into UTF-32 string. + * + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the Latin1 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_latin1_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-8 string into latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into latin1 string with errors. + * If the string cannot be represented as Latin1, an error + * code is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_latin1_with_errors(const char *input, size_t length, + char *latin1_output) const noexcept = 0; + + /** + * Convert valid UTF-8 string into latin1 string. + * + * This function assumes that the input string is valid UTF-8 and that it can + * be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf8_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param latin1_output the pointer to buffer that can hold conversion result + * @return the number of written char; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_latin1(const char *input, size_t length, + char *latin1_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16le_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated + * if successful. + */ + simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors( + const char *input, size_t length, + char16_t *utf16_output) const noexcept = 0; + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format even when the UTF-16LE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format even when the UTF-16BE content contains mismatched + * surrogates that have to be replaced by the replacement character (0xFFFD). + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) where the count is the number of bytes required to + * encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS + * or SURROGATE. The count is correct regardless of the error field. + * When SURROGATE is returned, it does not indicate an error in the case of + * this function: it indicates that at least one surrogate has been + * encountered: the surrogates may be matched or not (thus this function does + * not validate). If the returned error code is SUCCESS, then the input + * contains no surrogate, is in the Basic Multilingual Plane, and is + * necessarily valid. + */ + virtual simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) const noexcept = 0; + +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-8 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf8_to_utf32_with_errors(const char *input, size_t length, + char32_t *utf32_output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-8 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert valid UTF-8 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char32_t + */ + simdutf_warn_unused virtual size_t + convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of 2-byte code units that this UTF-8 string would + * require in UTF-16LE format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t code units required to encode the UTF-8 + * string as UTF-16LE + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of 4-byte code units that this UTF-8 string would + * require in UTF-32 format. + * + * This function is equivalent to count_utf8. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * This function does not validate the input. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char32_t code units required to encode the UTF-8 + * string as UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string or if it cannot be represented as Latin1 + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into Latin1 string. + * If the string cannot be represented as Latin1, an error + * is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_latin1_with_errors(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into Latin1 string. + * + * This function assumes that the input string is valid UTF-L16LE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16le_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into Latin1 string. + * + * This function assumes that the input string is valid UTF16-BE and that it + * can be represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf16be_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_latin1(const char16_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16le_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16BE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf8(const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-16LE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16LE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-16BE + * string + */ + simdutf_warn_unused virtual size_t + convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char32_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors( + const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-32BE string. + * + * This function assumes that the input string is valid UTF-16BE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf32_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *utf32_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-8 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Convert possibly broken UTF-32 string into Latin1 string and stop on error. + * If the string cannot be represented as Latin1, an error is returned. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to buffer that can hold conversion + * result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_latin1_with_errors(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into Latin1 string. + * + * This function assumes that the input string is valid UTF-32 and can be + * represented as Latin1. If you violate this assumption, the result is + * implementation defined and may include system-dependent behavior such as + * crashes. + * + * This function is for expert users only and not part of our public API. Use + * convert_utf32_to_latin1 instead. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param latin1_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_latin1(const char32_t *input, size_t length, + char *latin1_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char written if + * successful. + */ + simdutf_warn_unused virtual result + convert_utf32_to_utf8_with_errors(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf8_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-16 string would require in Latin1 + * format. + * + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written code units; 0 if input is not a valid UTF-32 + * string + */ + simdutf_warn_unused virtual size_t + convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on + * error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of char16_t written if + * successful. + */ + simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors( + const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; + + /** + * Convert valid UTF-32 string into UTF-16BE string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @param utf16_buffer the pointer to a buffer that can hold the conversion + * result + * @return number of written code units; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t + convert_valid_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *utf16_buffer) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Change the endianness of the input. Can be used to go from UTF-16LE to + * UTF-16BE or from UTF-16BE to UTF-16LE. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param output the pointer to a buffer that can hold the conversion + * result + */ + virtual void change_endianness_utf16(const char16_t *input, size_t length, + char16_t *output) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this Latin1 string would require in UTF-8 + * format. + * + * @param input the Latin1 string to convert + * @param length the length of the string bytes + * @return the number of bytes required to encode the Latin1 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_latin1(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-32 string would require in UTF-8 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-8 + */ + simdutf_warn_unused virtual size_t + utf8_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf32(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-8 string would require in Latin1 + * format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-8 strings but in such cases the result is implementation defined. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in byte + * @return the number of bytes required to encode the UTF-8 string as Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + /** + * Compute the number of bytes that this UTF-16LE/BE string would require in + * Latin1 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * Latin1 + */ + simdutf_warn_unused virtual size_t + latin1_length_from_utf16(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of two-byte code units that this UTF-32 string would + * require in UTF-16 format. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-32 strings but in such cases the result is implementation defined. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as UTF-16 + */ + simdutf_warn_unused virtual size_t + utf16_length_from_utf32(const char32_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + /** + * Return the number of bytes that this UTF-32 string would require in Latin1 + * format. + * + * @param length the length of the string in 4-byte code units + * (char32_t) + * @return the number of bytes required to encode the UTF-32 string as Latin1 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_latin1(size_t length) const noexcept { + return length; + } +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + /** + * Compute the number of bytes that this UTF-16LE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16le. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16le(const char16_t *input, + size_t length) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16BE string would require in + * UTF-32 format. + * + * This function is equivalent to count_utf16be. + * + * This function does not validate the input. It is acceptable to pass invalid + * UTF-16 strings but in such cases the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return the number of bytes required to encode the UTF-16BE string as + * UTF-32 + */ + simdutf_warn_unused virtual size_t + utf32_length_from_utf16be(const char16_t *input, + size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + +#if SIMDUTF_FEATURE_UTF16 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16le(const char16_t *input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16BE. + * It is acceptable to pass invalid UTF-16 strings but in such cases + * the result is implementation defined. + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to process + * @param length the length of the string in 2-byte code units + * (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf16be(const char16_t *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_UTF8 + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * It is acceptable to pass invalid UTF-8 strings but in such cases + * the result is implementation defined. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ + simdutf_warn_unused virtual size_t + count_utf8(const char *input, size_t length) const noexcept = 0; +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_BASE64 + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char *input, size_t length) const noexcept; + + /** + * Provide the maximal binary length in bytes given the base64 input. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return maximal number of binary bytes + */ + simdutf_warn_unused size_t maximal_binary_length_from_base64( + const char16_t *input, size_t length) const noexcept; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in bytes) if any, or the number of bytes written if + * successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you + * fail to provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process, in ASCII stored as + * 16-bit units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the + * number of bytes written if successful. + */ + simdutf_warn_unused virtual result + base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that + * it will ignore any ASCII spaces in the input. You may provide a padded + * input (with one or two equal signs at the end) or an unpadded input + * (without any equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single + * remainder character (BASE64_INPUT_REMAINDER), or the input contains a + * character that is not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return a full_result pair struct (of type simdutf::result containing the + * three fields error, input_count and output_count). + */ + simdutf_warn_unused virtual full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept = 0; + + /** + * Provide the base64 length in bytes given the length of a binary input. + * + * @param length the length of the input in bytes + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of base64 bytes + */ + simdutf_warn_unused size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) const noexcept; + + /** + * Convert a binary input to a base64 output. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ + virtual size_t + binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) const noexcept = 0; + + /** + * Convert a binary input to a base64 output with lines of given length. + * Lines are separated by a single linefeed character. + * + * The default option (simdutf::base64_default) uses the characters `+` and + * `/` as part of its alphabet. Further, it adds padding (`=`) at the end of + * the output to ensure that the output length is a multiple of four. + * + * The URL option (simdutf::base64_url) uses the characters `-` and `_` as + * part of its alphabet. No padding is added at the end of the output. + * + * This function always succeeds. + * + * @param input the binary to process + * @param length the length of the input in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least base64_length_from_binary_with_lines(length, + * options, line_length) bytes long) + * @param line_length the length of each line, values smaller than 4 are + * interpreted as 4 + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary_with_lines(length, options, line_length) + */ + virtual size_t binary_to_base64_with_lines( + const char *input, size_t length, char *output, + size_t line_length = simdutf::default_line_length, + base64_options options = base64_default) const noexcept = 0; + + /** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ + virtual const char *find(const char *start, const char *end, + char character) const noexcept = 0; + virtual const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept = 0; +#endif // SIMDUTF_FEATURE_BASE64 + +#ifdef SIMDUTF_INTERNAL_TESTS + // This method is exported only in developer mode, its purpose + // is to expose some internal test procedures from the given + // implementation and then use them through our standard test + // framework. + // + // Regular users should not use it, the tests of the public + // API are enough. + + struct TestProcedure { + // display name + std::string name; + + // procedure should return whether given test pass or not + void (*procedure)(const implementation &); + }; + + virtual std::vector<TestProcedure> internal_tests() const; +#endif + +protected: + /** @private Construct an implementation with the given name and description. + * For subclasses. */ + simdutf_really_inline implementation(const char *name, + const char *description, + uint32_t required_instruction_sets) + : _name(name), _description(description), + _required_instruction_sets(required_instruction_sets) {} + +protected: + ~implementation() = default; + +private: + /** + * The name of this implementation. + */ + const char *_name; + + /** + * The description of this implementation. + */ + const char *_description; + + /** + * Instruction sets required for this implementation. + */ + const uint32_t _required_instruction_sets; +}; + +/** @private */ +namespace internal { + +/** + * The list of available implementations compiled into simdutf. + */ +class available_implementation_list { +public: + /** Get the list of available implementations compiled into simdutf */ + simdutf_really_inline available_implementation_list() {} + /** Number of implementations */ + size_t size() const noexcept; + /** STL const begin() iterator */ + const implementation *const *begin() const noexcept; + /** STL const end() iterator */ + const implementation *const *end() const noexcept; + + /** + * Get the implementation with the given name. + * + * Case sensitive. + * + * const implementation *impl = + * simdutf::available_implementations["westmere"]; if (!impl) { exit(1); } if + * (!imp->supported_by_runtime_system()) { exit(1); } + * simdutf::active_implementation = impl; + * + * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" + * @return the implementation, or nullptr if the parse failed. + */ + const implementation *operator[](const std::string &name) const noexcept { + for (const implementation *impl : *this) { + if (impl->name() == name) { + return impl; + } + } + return nullptr; + } + + /** + * Detect the most advanced implementation supported by the current host. + * + * This is used to initialize the implementation on startup. + * + * const implementation *impl = + * simdutf::available_implementation::detect_best_supported(); + * simdutf::active_implementation = impl; + * + * @return the most advanced supported implementation for the current host, or + * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no + * supported implementation. Will never return nullptr. + */ + const implementation *detect_best_supported() const noexcept; +}; + +template <typename T> class atomic_ptr { +public: + atomic_ptr(T *_ptr) : ptr{_ptr} {} + +#if defined(SIMDUTF_NO_THREADS) + operator const T *() const { return ptr; } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr; } + + operator T *() { return ptr; } + T &operator*() { return *ptr; } + T *operator->() { return ptr; } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#else + operator const T *() const { return ptr.load(); } + const T &operator*() const { return *ptr; } + const T *operator->() const { return ptr.load(); } + + operator T *() { return ptr.load(); } + T &operator*() { return *ptr; } + T *operator->() { return ptr.load(); } + atomic_ptr &operator=(T *_ptr) { + ptr = _ptr; + return *this; + } + +#endif + +private: +#if defined(SIMDUTF_NO_THREADS) + T *ptr; +#else + std::atomic<T *> ptr; +#endif +}; + +class detect_best_supported_implementation_on_first_use; + +} // namespace internal + +/** + * The list of available implementations compiled into simdutf. + */ +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & +get_available_implementations(); + +/** + * The active implementation. + * + * Automatically initialized on first use to the most advanced implementation + * supported by this hardware. + */ +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> & +get_active_implementation(); + +} // namespace simdutf + +#if SIMDUTF_FEATURE_BASE64 + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +namespace simdutf { + +template <typename chartype> +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template <typename chartype> +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same<chartype, char>::value || + std::is_same<chartype, char16_t>::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = (std::min)( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple<result, std::size_t> + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t<decltype(*input.data())>; + static_assert(std::is_same_v<CInput, char>, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t<decltype(*binary_output.data())>; + static_assert(std::is_same_v<COutput, char>, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl<char>( + reinterpret_cast<const char *>(input.data()), input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple<result, std::size_t> + base64_to_binary_safe( + std::span<const char16_t> input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast<char *>(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#endif // SIMDUTF_FEATURE_BASE64 + +#endif // SIMDUTF_IMPLEMENTATION_H +/* end file include/simdutf/implementation.h */ + +// Implementation-internal files (must be included before the implementations +// themselves, to keep amalgamation working--otherwise, the first time a file is +// included, it might be put inside the #ifdef +// SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other +// implementations can't compile unless that implementation is turned on). + +SIMDUTF_POP_DISABLE_WARNINGS + +#endif // SIMDUTF_H +/* end file include/simdutf.h */
+ simdutf/simdutf_c.h view
@@ -0,0 +1,339 @@+/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include <stddef.h> +#include <stdbool.h> +#include <stdint.h> + +#ifdef __has_include + #if __has_include(<uchar.h>) + #include <uchar.h> + #else // __has_include(<uchar.h>) + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include(<uchar.h>) +#else // __has_include(<uchar.h>) + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { +#endif + +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */
src/Data/Text.hs view
@@ -1,2143 +1,2276 @@-{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-}-{-# LANGUAGE TemplateHaskellQuotes #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE UnliftedFFITypes #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE PartialTypeSignatures #-}-{-# LANGUAGE PatternSynonyms #-}-{-# LANGUAGE ViewPatterns #-}--{-# OPTIONS_GHC -fno-warn-orphans #-}-{-# OPTIONS_GHC -Wno-partial-type-signatures #-}---- |--- Module : Data.Text--- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts,--- (c) 2008, 2009 Tom Harper--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text.--- Suitable for performance critical use, both in terms of large data--- quantities and high speed.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text as T------ To use an extended and very rich family of functions for working--- with Unicode text (including normalization, regular expressions,--- non-standard encodings, text breaking, and locales), see the--- <http://hackage.haskell.org/package/text-icu text-icu package >.-----module Data.Text- (- -- * Strict vs lazy types- -- $strict-- -- * Acceptable data- -- $replacement-- -- * Definition of character- -- $character_definition-- -- * Fusion- -- $fusion-- -- * Types- Text- , StrictText-- -- * Creation and elimination- , pack- , unpack- , singleton- , empty-- -- * Pattern matching- , pattern Empty- , pattern (:<)- , pattern (:>)-- -- * Basic interface- , cons- , snoc- , append- , uncons- , unsnoc- , head- , last- , tail- , init- , null- , length- , compareLength-- -- * Transformations- , map- , intercalate- , intersperse- , transpose- , reverse- , replace-- -- ** Case conversion- -- $case- , toCaseFold- , toLower- , toUpper- , toTitle-- -- ** Justification- , justifyLeft- , justifyRight- , center-- -- * Folds- , foldl- , foldl'- , foldl1- , foldl1'- , foldr- , foldr'- , foldr1- , foldlM'-- -- ** Special folds- , concat- , concatMap- , any- , all- , maximum- , minimum- , isAscii-- -- * Construction-- -- ** Scans- , scanl- , scanl1- , scanr- , scanr1-- -- ** Accumulating maps- , mapAccumL- , mapAccumR-- -- ** Generation and unfolding- , replicate- , unfoldr- , unfoldrN-- -- * Substrings-- -- ** Breaking strings- , take- , takeEnd- , drop- , dropEnd- , takeWhile- , takeWhileEnd- , dropWhile- , dropWhileEnd- , dropAround- , strip- , stripStart- , stripEnd- , splitAt- , breakOn- , breakOnEnd- , break- , span- , spanM- , spanEndM- , group- , groupBy- , inits- , initsNE- , tails- , tailsNE-- -- ** Breaking into many substrings- -- $split- , splitOn- , split- , chunksOf-- -- ** Breaking into lines and words- , lines- --, lines'- , words- , unlines- , unwords-- -- * Predicates- , isPrefixOf- , isSuffixOf- , isInfixOf-- -- ** View patterns- , stripPrefix- , stripSuffix- , commonPrefixes-- -- * Searching- , filter- , breakOnAll- , find- , elem- , partition-- -- , findSubstring-- -- * Indexing- -- $index- , index- , findIndex- , count-- -- * Zipping- , zip- , zipWith-- -- * Showing values- , show-- -- -* Ordered text- -- , sort-- -- * Low level operations- , copy- , unpackCString#- , unpackCStringAscii#-- , measureOff- ) where--import Prelude (Char, Bool(..), Int, Maybe(..), String,- Eq, (==), (/=), Ord(..), Ordering(..), (++),- Monad(..), pure, Read(..), Show,- (&&), (||), (+), (-), (.), ($), ($!), (>>),- not, return, otherwise, quot)-import Control.DeepSeq (NFData(rnf))-#if defined(ASSERTS)-import Control.Exception (assert)-#endif-import Data.Bits ((.&.))-import qualified Data.Char as Char-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,- Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Control.Monad (foldM)-import Control.Monad.ST (ST, runST)-import qualified Data.Text.Array as A-import qualified Data.List as L hiding (head, tail)-import qualified Data.List.NonEmpty as NonEmptyList-import Data.Binary (Binary(get, put))-import Data.Binary.Put (putBuilder)-import Data.Monoid (Monoid(..))-import Data.Semigroup (Semigroup(..))-import Data.String (IsString(..))-import Data.Text.Internal.ArrayUtils (memchr)-import Data.Text.Internal.IsAscii (isAscii)-import Data.Text.Internal.Reverse (reverse)-import Data.Text.Internal.Measure (measure_off)-import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4)-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder)-import Data.Text.Internal.Fusion (stream, reverseStream, unstream)-import Data.Text.Internal.Private (span_)-import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#)-import qualified Prelude as P-import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter,- reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray)-import Data.Text.Internal.Search (indices)-import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_)-#if defined(__HADDOCK__)-import Data.ByteString (ByteString)-import qualified Data.Text.Lazy as L-#endif-import Data.Word (Word8)-import Foreign.C.Types-import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)-import qualified GHC.Exts as Exts-import GHC.Int (Int8)-import GHC.Stack (HasCallStack)-import qualified Language.Haskell.TH.Lib as TH-import qualified Language.Haskell.TH.Syntax as TH-import Text.Printf (PrintfArg, formatArg, formatString)-import System.Posix.Types (CSsize(..))--#if MIN_VERSION_template_haskell(2,16,0)-import Data.Text.Foreign (asForeignPtr)-import System.IO.Unsafe (unsafePerformIO)-#endif---- $setup--- >>> :set -package transformers--- >>> import Control.Monad.Trans.State--- >>> import Data.Text--- >>> import qualified Data.Text as T--- >>> :seti -XOverloadedStrings---- $character_definition------ This package uses the term /character/ to denote Unicode /code points/.------ Note that this is not the same thing as a grapheme (e.g. a--- composition of code points that form one visual symbol). For--- instance, consider the grapheme \"ä\". This symbol has two--- Unicode representations: a single code-point representation--- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point),--- and a two code point representation @U+0061@ (the \"@A@\" code--- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point).---- $strict------ This package provides both strict and lazy 'Text' types. The--- strict type is provided by the "Data.Text" module, while the lazy--- type is provided by the "Data.Text.Lazy" module. Internally, the--- lazy @Text@ type consists of a list of strict chunks.------ The strict 'Text' type requires that an entire string fit into--- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of--- streaming strings that are larger than memory using a small memory--- footprint. In many cases, the overhead of chunked streaming makes--- the lazy 'Data.Text.Lazy.Text' type slower than its strict--- counterpart, but this is not always the case. Sometimes, the time--- complexity of a function in one module may be different from the--- other, due to their differing internal structures.------ Each module provides an almost identical API, with the main--- difference being that the strict module uses 'Int' values for--- lengths and counts, while the lazy module uses 'Data.Int.Int64'--- lengths.---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values--- (corresponding to 'Data.Char.Surrogate' category) that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"�\" (U+FFFD). Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\". The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)---- $fusion------ Starting from @text-1.3@ fusion is no longer implicit,--- and pipelines of transformations usually allocate intermediate 'Text' values.--- Users, who observe significant changes to performances,--- are encouraged to use fusion framework explicitly, employing--- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".--instance Eq Text where- Text arrA offA lenA == Text arrB offB lenB- | lenA == lenB = A.equal arrA offA arrB offB lenA- | otherwise = False- {-# INLINE (==) #-}--instance Ord Text where- compare = compareText--instance Read Text where- readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]---- | @since 1.2.2.0-instance Semigroup Text where- (<>) = append--instance Monoid Text where- mempty = empty- mappend = (<>)- mconcat = concat---- | Performs replacement on invalid scalar values:------ >>> :set -XOverloadedStrings--- >>> "\55555" :: Text--- "\65533"-instance IsString Text where- fromString = pack---- | Performs replacement on invalid scalar values:------ >>> :set -XOverloadedLists--- >>> ['\55555'] :: Text--- "\65533"------ @since 1.2.0.0-instance Exts.IsList Text where- type Item Text = Char- fromList = pack- toList = unpack--instance NFData Text where rnf !_ = ()---- | @since 1.2.1.0-instance Binary Text where- put t = do- -- This needs to be in sync with the Binary instance for ByteString- -- in the binary package.- put (lengthWord8 t)- putBuilder (encodeUtf8Builder t)- get = do- bs <- get- case decodeUtf8' bs of- P.Left exn -> P.fail (P.show exn)- P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you--- feel a mistake has been made, please feel free to submit--- improvements.------ The original discussion is archived here:--- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? >------ The followup discussion that changed the behavior of 'Data.Set.Set'--- and 'Data.Map.Map' is archived here:--- <https://mail.haskell.org/pipermail/libraries/2012-August/018366.html Proposal: Allow gunfold for Data.Map, ... >--instance Data Text where- gfoldl f z txt = z pack `f` (unpack txt)- toConstr _ = packConstr- gunfold k z c = case constrIndex c of- 1 -> k (z pack)- _ -> P.error "gunfold"- dataTypeOf _ = textDataType---- | @since 1.2.4.0-instance TH.Lift Text where-#if MIN_VERSION_template_haskell(2,16,0)- lift txt = do- let (ptr, len) = unsafePerformIO $ asForeignPtr txt- case len of- 0 -> TH.varE 'empty- _ ->- let- bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len)- lenQ = liftInt (P.fromIntegral len)- liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n)))- in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ-#else- lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack-#endif-#if MIN_VERSION_template_haskell(2,17,0)- liftTyped = TH.unsafeCodeCoerce . TH.lift-#elif MIN_VERSION_template_haskell(2,16,0)- liftTyped = TH.unsafeTExpCoerce . TH.lift-#endif--#if MIN_VERSION_template_haskell(2,16,0)-unpackCStringLen# :: Exts.Addr# -> Int -> Text-unpackCStringLen# addr# l = Text ba 0 l- where- ba = runST $ do- marr <- A.new l- A.copyFromPointer marr 0 (Exts.Ptr addr#) l- A.unsafeFreeze marr-{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat-#endif---- | @since 1.2.2.0-instance PrintfArg Text where- formatArg txt = formatString $ unpack txt--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Text" [packConstr]---- | /O(n)/ Compare two 'Text' values lexicographically.-compareText :: Text -> Text -> Ordering-compareText (Text arrA offA lenA) (Text arrB offB lenB) =- A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB--- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),--- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering--- of underlying bytearrays, no decoding is needed.---- -------------------------------------------------------------------------------- * Basic functions---- | /O(n)/ Adds a character to the front of a 'Text'. This function--- is more costly than its 'List' counterpart because it requires--- copying a new array. Performs replacement on--- invalid scalar values.-cons :: Char -> Text -> Text-cons c = unstream . S.cons (safe c) . stream-{-# INLINE [1] cons #-}--infixr 5 `cons`---- | /O(n)/ Adds a character to the end of a 'Text'. This copies the--- entire array in the process.--- Performs replacement on invalid scalar values.-snoc :: Text -> Char -> Text-snoc t c = unstream (S.snoc (stream t) (safe c))-{-# INLINE snoc #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'uncons' instead.-head :: HasCallStack => Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty.-uncons :: Text -> Maybe (Char, Text)-uncons t@(Text arr off len)- | len <= 0 = Nothing- | otherwise = Just $ let !(Iter c d) = iter t 0- in (c, text arr (off+d) (len-d))-{-# INLINE [1] uncons #-}---- | /O(1)/ Returns the last character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'unsnoc' instead.-last :: HasCallStack => Text -> Char-last t@(Text _ _ len)- | null t = emptyError "last"- | otherwise = let Iter c _ = reverseIter t (len - 1) in c-{-# INLINE [1] last #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty. This is a partial function, consider using 'uncons' instead.-tail :: HasCallStack => Text -> Text-tail t@(Text arr off len)- | null t = emptyError "tail"- | otherwise = text arr (off+d) (len-d)- where d = iter_ t 0-{-# INLINE [1] tail #-}---- | /O(1)/ Returns all but the last character of a 'Text', which must--- be non-empty. This is a partial function, consider using 'unsnoc' instead.-init :: HasCallStack => Text -> Text-init t@(Text arr off len)- | null t = emptyError "init"- | otherwise = text arr off (len + reverseIter_ t (len - 1))-{-# INLINE [1] init #-}---- | /O(1)/ Returns all but the last character and the last character of a--- 'Text', or 'Nothing' if empty.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc t@(Text arr off len)- | null t = Nothing- | otherwise = Just (text arr off (len + d), c)- where- Iter c d = reverseIter t (len - 1)-{-# INLINE [1] unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.-null :: Text -> Bool-null (Text _arr _off len) =-#if defined(ASSERTS)- assert (len >= 0) $-#endif- len <= 0-{-# INLINE [1] null #-}--{-# RULES- "TEXT null/empty -> True" null empty = True-#-}---- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/),--- to be used together with '(:<)' or '(:>)'.------ @since 2.1.2-pattern Empty :: Text-pattern Empty <- (null -> True) where- Empty = empty---- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/),--- to be used together with 'Empty'.------ @since 2.1.2-pattern (:<) :: Char -> Text -> Text-pattern x :< xs <- (uncons -> Just (x, xs)) where- (:<) = cons-infixr 5 :<-{-# COMPLETE Empty, (:<) #-}---- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/)--- to be used together with 'Empty'.------ @since 2.1.2-pattern (:>) :: Text -> Char -> Text-pattern xs :> x <- (unsnoc -> Just (xs, x)) where- (:>) = snoc-infixl 5 :>-{-# COMPLETE Empty, (:>) #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n)/ Returns the number of characters in a 'Text'.-length ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Int-length = P.negate . measureOff P.maxBound-{-# INLINE [1] length #-}--- length needs to be phased after the compareN/length rules otherwise--- it may inline before the rules have an opportunity to fire.--{-# RULES-"TEXT length/filter -> S.length/S.filter" forall p t.- length (filter p t) = S.length (S.filter p (stream t))-"TEXT length/unstream -> S.length" forall t.- length (unstream t) = S.length t-"TEXT length/pack -> P.length" forall t.- length (pack t) = P.length t-"TEXT length/map -> length" forall f t.- length (map f t) = length t-"TEXT length/zipWith -> length" forall f t1 t2.- length (zipWith f t1 t2) = min (length t1) (length t2)-"TEXT length/replicate -> n" forall n t.- length (replicate n t) = mul (max 0 n) (length t)-"TEXT length/cons -> length+1" forall c t.- length (cons c t) = 1 + length t-"TEXT length/intersperse -> 2*length-1" forall c t.- length (intersperse c t) = max 0 (mul 2 (length t) - 1)-"TEXT length/intercalate -> n*length" forall s ts.- length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)-"TEXT length/empty -> 0"- length empty = 0- #-}---- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number.------ @--- 'compareLength' t c = 'P.compare' ('length' t) c--- @------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int -> Ordering-compareLength t c = S.compareLengthI (stream t) c-{-# INLINE [1] compareLength #-}--{-# RULES-"TEXT compareN/length -> compareLength" [~1] forall t n.- compare (length t) n = compareLength t n- #-}--{-# RULES-"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.- eqInt (length t) n = compareLength t n == EQ- #-}--{-# RULES-"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.- neInt (length t) n = compareLength t n /= EQ- #-}--{-# RULES-"TEXT <N/length -> compareLength/==LT" [~1] forall t n.- ltInt (length t) n = compareLength t n == LT- #-}--{-# RULES-"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.- leInt (length t) n = compareLength t n /= GT- #-}--{-# RULES-"TEXT >N/length -> compareLength/==GT" [~1] forall t n.- gtInt (length t) n = compareLength t n == GT- #-}--{-# RULES-"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.- geInt (length t) n = compareLength t n /= LT- #-}---- -------------------------------------------------------------------------------- * Transformations--- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.------ Example:------ >>> let message = pack "I am not angry. Not at all."--- >>> T.map (\c -> if c == '.' then '!' else c) message--- "I am not angry! Not at all!"------ Performs replacement on invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f = \t -> if null t then empty else mapNonEmpty f t-{-# INLINE [1] map #-}--{-# RULES-"TEXT map/map -> map" forall f g t.- map f (map g t) = map (f . safe . g) t-#-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.------ Example:------ >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"]--- "WeNI!seekNI!theNI!HolyNI!Grail"-intercalate :: Text -> [Text] -> Text-intercalate t = concat . L.intersperse t-{-# INLINE [1] intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'.------ Example:------ >>> T.intersperse '.' "SHIELD"--- "S.H.I.E.L.D"------ Performs replacement on invalid scalar values.-intersperse :: Char -> Text -> Text-intersperse c t@(Text src o l) = if null t then empty else runST $ do- let !cLen = utf8Length c- dstLen = l + length t P.* cLen-- dst <- A.new dstLen-- let writeSep = case cLen of- 1 -> \dstOff ->- A.unsafeWrite dst dstOff (ord8 c)- 2 -> let (c0, c1) = ord2 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- A.unsafeWrite dst (dstOff + 2) c2- _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do- A.unsafeWrite dst dstOff c0- A.unsafeWrite dst (dstOff + 1) c1- A.unsafeWrite dst (dstOff + 2) c2- A.unsafeWrite dst (dstOff + 3) c3- let go !srcOff !dstOff = if srcOff >= o + l then return () else do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0- case d of- 1 -> do- A.unsafeWrite dst dstOff m0- writeSep (dstOff + 1)- go (srcOff + 1) (dstOff + 1 + cLen)- 2 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- writeSep (dstOff + 2)- go (srcOff + 2) (dstOff + 2 + cLen)- 3 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- writeSep (dstOff + 3)- go (srcOff + 3) (dstOff + 3 + cLen)- _ -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- writeSep (dstOff + 4)- go (srcOff + 4) (dstOff + 4 + cLen)-- go o 0- arr <- A.unsafeFreeze dst- return (Text arr 0 (dstLen - cLen))-{-# INLINE [1] intersperse #-}---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =--- 'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once. So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ >>> replace "oo" "foo" "oo"--- "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ >>> replace "ofo" "bar" "ofofo"--- "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: HasCallStack- => Text- -- ^ @needle@ to search for. If this string is empty, an- -- error will occur.- -> Text- -- ^ @replacement@ to replace @needle@ with.- -> Text- -- ^ @haystack@ in which to search.- -> Text-replace needle@(Text _ _ neeLen)- (Text repArr repOff repLen)- haystack@(Text hayArr hayOff hayLen)- | neeLen == 0 = emptyError "replace"- | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it- | L.null ixs = haystack- | otherwise = Text (A.run x) 0 len- where- ixs = indices needle haystack- len = hayLen - (neeLen - repLen) `mul` L.length ixs- x :: ST s (A.MArray s)- x = do- marr <- A.new len- let loop (i:is) o d = do- let d0 = d + i - o- d1 = d0 + repLen- A.copyI (i - o) marr d hayArr (hayOff+o)- A.copyI repLen marr d0 repArr repOff- loop is (i + neeLen) d1- loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o)- loop ixs 0 0- return marr---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ When case converting 'Text' values, do not use combinators like--- @map toUpper@ to case convert each character of a string--- individually, as this gives incorrect results according to the--- rules of some writing systems. The whole-string case conversion--- functions from this module, such as @toUpper@, obey the correct--- case conversion rules. As a result, these functions may map one--- input character to two or three output characters. For examples,--- see the documentation of each function.------ /Note/: In some languages, case conversion is a locale- and--- context-dependent operation. The case conversion functions in this--- module are /not/ locale sensitive. Programs that require locale--- sensitivity should use appropriate versions of the--- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >.---- | /O(n)/ Convert a string to folded case.------ This function is mainly useful for performing caseless (also known--- as case insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string. For instance,--- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case--- folded to the sequence \"մ\" (men, U+0574) followed by--- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign,--- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC)--- instead of itself.-toCaseFold :: Text -> Text-toCaseFold = \t ->- if null t then empty- else toCaseFoldNonEmpty t-{-# INLINE toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, \"İ\" (Latin capital letter I with dot above,--- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069)--- followed by \" ̇\" (combining dot above, U+0307).-toLower :: Text -> Text-toLower = \t ->- if null t then empty- else toLowerNonEmpty t-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the German \"ß\" (eszett, U+00DF) maps to the--- two-letter sequence \"SS\".-toUpper :: Text -> Text-toUpper = \t ->- if null t then empty- else toUpperNonEmpty t-{-# INLINE toUpper #-}---- | /O(n)/ Convert a string to title case, using simple case--- conversion.------ The first letter (as determined by 'Data.Char.isLetter')--- of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ This function is not idempotent.--- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).--- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input--- is converted to title case, becoming two letters.--- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter--- and as such is recognised as a letter by 'Data.Char.isLetter',--- so 'T.toTitle' @"ʼN"@ = @"'n"@.------ The result string may be longer than the input string. For example,--- the Latin small ligature fl (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased—but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle = \t ->- if null t then empty- else toTitleNonEmpty t-{-# INLINE toTitle #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right.--- Performs replacement on invalid scalar values.------ Examples:------ >>> justifyLeft 7 'x' "foo"--- "fooxxxx"------ >>> justifyLeft 3 'x' "foobar"--- "foobar"-justifyLeft :: Int -> Char -> Text -> Text-justifyLeft k c t- | len >= k = t- | otherwise = t `append` replicateChar (k-len) c- where len = length t-{-# INLINE [1] justifyLeft #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Performs replacement on--- invalid scalar values.------ Examples:------ >>> justifyRight 7 'x' "bar"--- "xxxxbar"------ >>> justifyRight 3 'x' "foobar"--- "foobar"-justifyRight :: Int -> Char -> Text -> Text-justifyRight k c t- | len >= k = t- | otherwise = replicateChar (k-len) c `append` t- where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side. Performs replacement on invalid--- scalar values.------ Examples:------ >>> center 8 'x' "HS"--- "xxxHSxxx"-center :: Int -> Char -> Text -> Text-center k c t- | len >= k = t- | otherwise = replicateChar l c `append` t `append` replicateChar r c- where len = length t- d = k - len- r = d `quot` 2- l = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument. Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.------ Examples:------ >>> transpose ["green","orange"]--- ["go","rr","ea","en","ng","e"]------ >>> transpose ["blue","red"]--- ["br","le","ud","e"]-transpose :: [Text] -> [Text]-transpose ts = P.map pack (L.transpose (P.map unpack ts))---- -------------------------------------------------------------------------------- * Reducing 'Text's (folds)---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.-foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.-foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ A monadic version of 'foldl''.------ @since 2.1.2-foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a-foldlM' f z t = S.foldlM' f z (stream t)-{-# INLINE foldlM' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.------ If the binary operator is strict in its second argument, use 'foldr''--- instead.------ 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually--- traverses the 'Text' from left to right, only as far as it needs to.------ For example, 'head' can be defined with /O(1)/ complexity using 'foldr':------ @--- head :: Text -> Char--- head = foldr const (error "head empty")--- @------ Searches from left to right with short-circuiting behavior can--- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem').-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- | /O(n)/ A strict version of 'foldr'.------ 'foldr'' evaluates as a right-to-left traversal using constant stack space.------ @since 2.0.1-foldr' :: (Char -> a -> a) -> a -> Text -> a-foldr' f z t = S.foldl' (P.flip f) z (reverseStream t)-{-# INLINE foldr' #-}---- -------------------------------------------------------------------------------- ** Special folds---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat ts = case ts of- [] -> empty- [t] -> t- _ | len == 0 -> empty- | otherwise -> Text (A.run go) 0 len- where- len = sumP "concat" $ L.map lengthWord8 ts- go :: ST s (A.MArray s)- go = do- arr <- A.new len- let step i (Text a o l) = A.copyI l arr i a o >> return (i + l)- foldM step 0 ts >> return arr---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty.-maximum :: HasCallStack => Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty.-minimum :: HasCallStack => Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- -------------------------------------------------------------------------------- * Building 'Text's--- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ __Properties__------ @'head' ('scanl' f z xs) = z@------ @'last' ('scanl' f z xs) = 'foldl' f z xs@-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))- where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument. Performs replacement on invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t | null t = empty- | otherwise = scanl f (unsafeHead t) (unsafeTail t)-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f z = S.reverse . S.reverseScanr g z . reverseStream- where g a b = safe (f a b)-{-# INLINE scanr #-}---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument. Performs replacement on invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t = empty- | otherwise = scanr f (last t) (init t)-{-# INLINE scanr1 #-}---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'. Performs--- replacement on invalid scalar values.-mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)-mapAccumL f z0 = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + 4) o 0 z0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text)- outer !dst !dstLen = inner- where- inner !srcOff !dstOff !z- | srcOff >= l + o = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (z, Text arr 0 dstOff)- | dstOff + 4 > dstLen = do- let !dstLen' = dstLen + (l + o) - srcOff + 4- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff z- | otherwise = do- let !(Iter c d) = iterArray src srcOff- (z', c') = f z c- d' <- unsafeWrite dst dstOff (safe c')- inner (srcOff + d) (dstOff + d') z'-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'.--- Performs replacement on invalid scalar values.-mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text)-mapAccumR f z0 = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + o - 1) (l + 4 - 1) z0- where- outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text)- outer !dst = inner- where- inner !srcOff !dstOff !z- | srcOff < o = do- dstLen <- A.getSizeofMArray dst- arr <- A.unsafeFreeze dst- return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1))- | dstOff < 3 = do- dstLen <- A.getSizeofMArray dst- let !dstLen' = dstLen + (srcOff - o) + 4- dst' <- A.new dstLen'- A.copyM dst' (dstLen' - dstLen) dst 0 dstLen- outer dst' srcOff (dstOff + dstLen' - dstLen) z- | otherwise = do- let !(Iter c d) = reverseIterArray src (srcOff)- (z', c') = f z c- c'' = safe c'- !d' = utf8Length c''- dstOff' = dstOff - d'- _ <- unsafeWrite dst (dstOff' + 1) c''- inner (srcOff + d) dstOff' z'-{-# INLINE mapAccumR #-}---- -------------------------------------------------------------------------------- ** Generating and unfolding 'Text's---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int -> Text -> Text-replicate n t@(Text a o l)- | n <= 0 || l <= 0 = empty- | n == 1 = t- | isSingleton t = replicateChar n (unsafeHead t)- | otherwise = runST $ do- let totalLen = n `mul` l- marr <- A.new totalLen- A.copyI l marr 0 a o- A.tile marr l- arr <- A.unsafeFreeze marr- return $ Text arr 0 totalLen-{-# INLINE [1] replicate #-}--{-# RULES-"TEXT replicate/singleton -> replicateChar" [~1] forall n c.- replicate n (singleton c) = replicateChar n c- #-}---- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the--- value of every element.-replicateChar :: Int -> Char -> Text-replicateChar !len !c'- | len <= 0 = empty- | Char.isAscii c = runST $ do- marr <- A.newFilled len (Char.ord c)- arr <- A.unsafeFreeze marr- return $ Text arr 0 len- | otherwise = runST $ do- let cLen = utf8Length c- totalLen = cLen P.* len- marr <- A.new totalLen- _ <- unsafeWrite marr 0 c- A.tile marr cLen- arr <- A.unsafeFreeze marr- return $ Text arr 0 totalLen- where- c = safe c'-{-# INLINE replicateChar #-}---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.--- Performs replacement on invalid scalar values.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'.--- Performs replacement on invalid scalar values.-unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- -------------------------------------------------------------------------------- * Substrings---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text.-take :: Int -> Text -> Text-take n t@(Text arr off len)- | n <= 0 = empty- | n >= len || m >= len || m < 0 = t- | otherwise = Text arr off m- where- m = measureOff n t-{-# INLINE [1] take #-}---- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@--- returns a non-negative number, measuring their size in 'Word8'. Otherwise,--- if @t@ is shorter, return a non-positive number, which is a negated total count--- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0.------ This function is used to implement 'take', 'drop', 'splitAt' and 'length'--- and is useful on its own in streaming and parsing libraries.------ @since 2.0-measureOff :: Int -> Text -> Int-measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else- cSsizeToInt $- measure_off arr (intToCSize off) (intToCSize len) (intToCSize n)---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ >>> takeEnd 3 "foobar"--- "bar"------ @since 1.1.1.0-takeEnd :: Int -> Text -> Text-takeEnd n t@(Text arr off len)- | n <= 0 = empty- | n >= len = t- | otherwise = text arr (off+i) (len-i)- where i = iterNEnd n t--iterNEnd :: Int -> Text -> Int-iterNEnd n t@(Text _arr _off len) = loop (len-1) n- where loop i !m- | m <= 0 = i+1- | i <= 0 = 0- | otherwise = loop (i+d) (m-1)- where d = reverseIter_ t i---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'.-drop :: Int -> Text -> Text-drop n t@(Text arr off len)- | n <= 0 = t- | n >= len || m >= len || m < 0 = empty- | otherwise = Text arr (off+m) (len-m)- where m = measureOff n t-{-# INLINE [1] drop #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ >>> dropEnd 3 "foobar"--- "foo"------ @since 1.1.1.0-dropEnd :: Int -> Text -> Text-dropEnd n t@(Text arr off len)- | n <= 0 = t- | n >= len = empty- | otherwise = text arr off (iterNEnd n t)---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t@(Text arr off len) = loop 0- where loop !i | i >= len = t- | p c = loop (i+d)- | otherwise = text arr off i- where Iter c d = iter t i-{-# INLINE [1] takeWhile #-}---- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.--- Examples:------ >>> takeWhileEnd (=='o') "foo"--- "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p t@(Text arr off len) = loop (len-1) len- where loop !i !l | l <= 0 = t- | p c = loop (i+d) (l+d)- | otherwise = text arr (off+l) (len-l)- where Iter c d = reverseIter t i-{-# INLINE [1] takeWhileEnd #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t@(Text arr off len) = loop 0 0- where loop !i !l | l >= len = empty- | p c = loop (i+d) (l+d)- | otherwise = Text arr (off+i) (len-l)- where Iter c d = iter t i-{-# INLINE [1] dropWhile #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.------ Examples:------ >>> dropWhileEnd (=='.') "foo..."--- "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p t@(Text arr off len) = loop (len-1) len- where loop !i !l | l <= 0 = empty- | p c = loop (i+d) (l+d)- | otherwise = Text arr off l- where Iter c d = reverseIter t i-{-# INLINE [1] dropWhileEnd #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string. Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile Char.isSpace-{-# INLINE stripStart #-}---- | /O(n)/ Remove trailing white space from a string. Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd Char.isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround Char.isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int -> Text -> (Text, Text)-splitAt n t@(Text arr off len)- | n <= 0 = (empty, t)- | n >= len || m >= len || m < 0 = (t, empty)- | otherwise = (Text arr off m, Text arr (off+m) (len-m))- where- m = measureOff n t---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the text.------ >>> T.span (=='0') "000AB"--- ("000","AB")-span :: (Char -> Bool) -> Text -> (Text, Text)-span p t = case span_ p t of- (# hd,tl #) -> (hd,tl)-{-# INLINE span #-}---- | /O(n)/ 'break' is like 'span', but the prefix returned is--- over elements that fail the predicate @p@.------ >>> T.break (=='c') "180cm"--- ("180","cm")-break :: (Char -> Bool) -> Text -> (Text, Text)-break p = span (not . p)-{-# INLINE break #-}---- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@,--- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of--- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text.------ >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97--- (("abc","efg"),101)------ 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity':------ @--- -- for all p :: Char -> Bool--- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p)--- @------ @since 2.0.1-spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)-spanM p t@(Text arr off len) = go 0- where- go !i | i < len = case iterArray arr (off+i) of- Iter c l -> do- continue <- p c- if continue then go (i+l)- else pure (text arr off i, text arr (off+i) (len-i))- go _ = pure (t, empty)-{-# INLINE spanM #-}---- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@,--- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of--- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text.------ >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122--- (("tuv","xyz"),118)------ @--- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p--- @------ @since 2.0.1-spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)-spanEndM p t@(Text arr off len) = go (len-1)- where- go !i | 0 <= i = case reverseIterArray arr (off+i) of- Iter c l -> do- continue <- p c- if continue then go (i+l)- else pure (text arr off (i+1), text arr (off+i+1) (len-i-1))- go _ = pure (empty, t)-{-# INLINE spanEndM #-}---- | /O(n)/ Group characters in a string according to a predicate.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy p = loop- where- loop t@(Text arr off len)- | null t = []- | otherwise = text arr off n : loop (text arr (off+n) (len-n))- where Iter c d = iter t 0- n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d))---- | Returns the /array/ index (in units of 'Word8') at which a--- character may be found. This is /not/ the same as the logical--- index returned by e.g. 'findIndex'.-findAIndexOrEnd :: (Char -> Bool) -> Text -> Int-findAIndexOrEnd q t@(Text _arr _off len) = go 0- where go !i | i >= len || q c = i- | otherwise = go (i+d)- where Iter c d = iter t i---- | /O(n)/ Group characters in a string by equality.-group :: Text -> [Text]-group = groupBy (==)---- | /O(n)/ Return all initial segments of the given 'Text', shortest--- first.-inits :: Text -> [Text]-inits = (NonEmptyList.toList $!) . initsNE---- | /O(n)/ Return all initial segments of the given 'Text', shortest--- first.------ @since 2.1.2-initsNE :: Text -> NonEmptyList.NonEmpty Text-initsNE t = empty NonEmptyList.:| case t of- Text arr off len ->- let loop i- | i >= len = []- | otherwise = let !j = i + iter_ t i in Text arr off j : loop j- in loop 0---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails = (NonEmptyList.toList $!) . tailsNE---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.------ @since 2.1.2-tailsNE :: Text -> NonEmptyList.NonEmpty Text-tailsNE t- | null t = empty NonEmptyList.:| []- | otherwise = t NonEmptyList.:| tails (unsafeTail t)---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be empty), consuming the delimiter. An empty--- delimiter is invalid, and will cause an error to be raised.------ Examples:------ >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne"--- ["a","b","d","e"]------ >>> splitOn "aaa" "aaaXaaaXaaaXaaa"--- ["","X","X","X",""]------ >>> splitOn "x" "x"--- ["",""]------ and------ > intercalate s . splitOn s == id--- > splitOn (singleton c) == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: HasCallStack- => Text- -- ^ String to split on. If this string is empty, an error- -- will occur.- -> Text- -- ^ Input text.- -> [Text]-splitOn pat@(Text _ _ l) src@(Text arr off len)- | l <= 0 = emptyError "splitOn"- | isSingleton pat = split (== unsafeHead pat) src- | otherwise = go 0 (indices pat src)- where- go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs- go s _ = [text arr (s+off) (len-s)]-{-# INLINE [1] splitOn #-}--{-# RULES-"TEXT splitOn/singleton -> split/==" [~1] forall c t.- splitOn (singleton c) t = split (==c) t- #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element. The--- resulting components do not contain the separators. Two adjacent--- separators result in an empty component in the output. eg.------ >>> split (=='a') "aabbaca"--- ["","","bb","c",""]------ >>> split (=='a') ""--- [""]-split :: (Char -> Bool) -> Text -> [Text]-split p t- | null t = [empty]- | otherwise = loop t- where loop s | null s' = [l]- | otherwise = l : loop (unsafeTail s')- where (# l, s' #) = span_ (not . p) s-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@. The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ >>> chunksOf 3 "foobarbaz"--- ["foo","bar","baz"]------ >>> chunksOf 4 "haskell.org"--- ["hask","ell.","org"]-chunksOf :: Int -> Text -> [Text]-chunksOf k = go- where- go t = case splitAt k t of- (a,b) | null a -> []- | otherwise -> a : go b-{-# INLINE chunksOf #-}---- ------------------------------------------------------------------------------- * Searching------------------------------------------------------------------------------------ ** Searching with a predicate---- | /O(n)/ The 'elem' function takes a character and a 'Text', and--- returns 'True' if the element is found in the given 'Text', or--- 'False' otherwise.-elem :: Char -> Text -> Bool-elem c t = S.any (== c) (stream t)-{-# INLINE elem #-}---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element matching the predicate, or 'Nothing' if--- there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p = filter_ text p-{-# INLINE [1] filter #-}--{-# RULES-"TEXT filter/filter -> filter" forall p q t.- filter p (filter q t) = filter (\c -> q c && p c) t-#-}---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@. The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched. The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ >>> breakOn "::" "a::b::c"--- ("a","::b::c")------ >>> breakOn "/" "foobar"--- ("foobar","")------ Laws:------ > append prefix match == haystack--- > where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: HasCallStack => Text -> Text -> (Text, Text)-breakOn pat src@(Text arr off len)- | null pat = emptyError "breakOn"- | otherwise = case indices pat src of- [] -> (src, empty)- (x:_) -> (text arr off x, text arr (off+x) (len-x))-{-# INLINE breakOn #-}---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the--- string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@. The second is the--- remainder of @haystack@, following the match.------ >>> breakOnEnd "::" "a::b::c"--- ("a::b::","c")-breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)-breakOnEnd pat src = (reverse b, reverse a)- where (a,b) = breakOn (reverse pat) (reverse src)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@. Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ >>> breakOnAll "::" ""--- []------ >>> breakOnAll "/" "a/b/c/"--- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")]------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: HasCallStack- => Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-breakOnAll pat src@(Text arr off slen)- | null pat = emptyError "breakOnAll"- | otherwise = L.map step (indices pat src)- where- step x = (chunk 0 x, chunk x (slen-x))- chunk !n !l = text arr (n+off) l-{-# INLINE breakOnAll #-}------------------------------------------------------------------------------------ ** Indexing 'Text's---- $index------ If you think of a 'Text' value as an array of 'Char' values (which--- it is not), you run the risk of writing inefficient code.------ An idiom that is common in some languages is to find the numeric--- offset of a character or substring, then use that number to split--- or trim the searched string. With a 'Text' value, this approach--- would require two /O(n)/ operations: one to perform the search, and--- one to operate from wherever the search ended.------ For example, suppose you have a string that you want to split on--- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of--- searching for the index of @\"::\"@ and taking the substrings--- before and after that index, you would instead use @breakOnAll \"::\"@.---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: HasCallStack => Text -> Int -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text'--- and returns the index of the first element in the 'Text' satisfying--- the predicate.-findIndex :: (Char -> Bool) -> Text -> Maybe Int-findIndex p t = S.findIndex p (stream t)-{-# INLINE findIndex #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: HasCallStack => Text -> Text -> Int-count pat- | null pat = emptyError "count"- | isSingleton pat = countChar (unsafeHead pat)- | otherwise = L.length . indices pat-{-# INLINE [1] count #-}--{-# RULES-"TEXT count/singleton -> countChar" [~1] forall c t.- count (singleton c) t = countChar c t- #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'.-countChar :: Char -> Text -> Int-countChar c t = S.countChar c (stream t)-{-# INLINE countChar #-}------------------------------------------------------------------------------------ * Zipping---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))- where g a b = safe (f a b)-{-# INLINE [1] zipWith #-}---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words (Text arr off len) = loop 0 0- where- loop !start !n- | n >= len = if start == n- then []- else [Text arr (start + off) (n - start)]- -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20- | isAsciiSpace w0 =- if start == n- then loop (n + 1) (n + 1)- else Text arr (start + off) (n - start) : loop (n + 1) (n + 1)- | w0 < 0x80 = loop start (n + 1)- -- or 2 bytes for 0xA0- | w0 == 0xC2, w1 == 0xA0 =- if start == n- then loop (n + 2) (n + 2)- else Text arr (start + off) (n - start) : loop (n + 2) (n + 2)- | w0 < 0xE0 = loop start (n + 2)- -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000- | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80- || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F)- || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 =- if start == n- then loop (n + 3) (n + 3)- else Text arr (start + off) (n - start) : loop (n + 3) (n + 3)- | otherwise = loop start (n + utf8LengthByLeader w0)- where- w0 = A.unsafeIndex arr (off + n)- w1 = A.unsafeIndex arr (off + n + 1)- w2 = A.unsafeIndex arr (off + n + 2)-{-# INLINE words #-}---- Adapted from Data.ByteString.Internal.isSpaceWord8-isAsciiSpace :: Word8 -> Bool-isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5)-{-# INLINE isAsciiSpace #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters--- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.------ 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.-lines :: Text -> [Text]-lines (Text arr@(A.ByteArray arr#) off len) = go off- where- go !n- | n >= len + off = []- | delta < 0 = [Text arr n (len + off - n)]- | otherwise = Text arr n delta : go (n + delta + 1)- where- delta = memchr arr# n (len + off - n) 0x0A-{-# INLINE lines #-}---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' if and only if the first is a prefix of the second.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) =- alen <= blen && S.isPrefixOf (stream a) (stream b)-{-# INLINE [1] isPrefixOf #-}---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' if and only if the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf a@(Text _aarr _aoff alen) b@(Text barr boff blen) =- d >= 0 && a == b'- where d = blen - alen- b' | d == 0 = b- | otherwise = Text barr (boff+d) alen-{-# INLINE isSuffixOf #-}---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' if and only if the first is contained, wholly and intact, anywhere--- within the second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Text -> Bool-isInfixOf needle haystack- | null needle = True- | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack- | otherwise = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ >>> stripPrefix "foo" "foobar"--- Just "bar"------ >>> stripPrefix "" "baz"--- Just "baz"------ >>> stripPrefix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _ = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p@(Text _arr _off plen) t@(Text arr off len)- | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen)- | otherwise = Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ >>> commonPrefixes "foobar" "fooquux"--- Just ("foo","bar","quux")------ >>> commonPrefixes "veeble" "fetzer"--- Nothing------ >>> commonPrefixes "" "baz"--- Nothing-commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text)-commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1)- | len0 == 0 = Nothing- | len1 == 0 = Nothing- | otherwise = go 0 0- where- go !i !j- | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i))- | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty)- | a == b = go (i + 1) k- | k > 0 = Just (Text arr0 off0 k,- Text arr0 (off0 + k) (len0 - k),- Text arr1 (off1 + k) (len1 - k))- | otherwise = Nothing- where- a = A.unsafeIndex arr0 (off0 + i)- b = A.unsafeIndex arr1 (off1 + i)- isLeader = word8ToInt8 a >= -64- k = if isLeader then i else j-{-# INLINE commonPrefixes #-}---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ >>> stripSuffix "bar" "foobar"--- Just "foo"------ >>> stripSuffix "" "baz"--- Just "baz"------ >>> stripSuffix "foo" "quux"--- Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _ = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p@(Text _arr _off plen) t@(Text arr off len)- | p `isSuffixOf` t = Just $! text arr off (len-plen)- | otherwise = Nothing---- | Add a list of non-negative numbers. Errors out on overflow.-sumP :: String -> [Int] -> Int-sumP fun = L.foldl' add 0- where add a x- | ax >= 0 = ax- | otherwise = overflowError fun- where ax = a + x-{-# INLINE sumP #-} -- Use foldl' and inline for fusion.--emptyError :: HasCallStack => String -> a-emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input"--overflowError :: HasCallStack => String -> a-overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow"---- | Convert a value to 'Text'.------ @since 2.1.2-show :: Show a => a -> Text-show = pack . P.show---- | /O(n)/ Make a distinct copy of the given string, sharing no--- storage with the original string.------ As an example, suppose you read a large string, of which you need--- only a small portion. If you do not use 'copy', the entire original--- array will be kept alive in memory by the smaller string. Making a--- copy \"breaks the link\" to the original array, allowing it to be--- garbage collected if there are no other live references to it.-copy :: Text -> Text-copy t@(Text arr off len)- | null t = empty- | otherwise = Text (A.run go) 0 len- where- go :: ST s (A.MArray s)- go = do- marr <- A.new len- A.copyI len marr 0 arr off- return marr--ord8 :: Char -> Word8-ord8 = P.fromIntegral . Char.ord--intToCSize :: Int -> CSize-intToCSize = P.fromIntegral--cSsizeToInt :: CSsize -> Int-cSsizeToInt = P.fromIntegral--word8ToInt8 :: Word8 -> Int8-word8ToInt8 = P.fromIntegral------------------------------------------------------ NOTE: the named chunk below used by doctest;--- verify the doctests via `doctest -fobject-code Data/Text.hs`---- $setup--- >>> :set -XOverloadedStrings--- >>> import qualified Data.Text as T+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples, TypeFamilies #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text +-- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text. +-- Suitable for performance critical use, both in terms of large data +-- quantities and high speed. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text as T +-- +-- To use an extended and very rich family of functions for working +-- with Unicode text (including normalization, regular expressions, +-- non-standard encodings, text breaking, and locales), see the +-- <http://hackage.haskell.org/package/text-icu text-icu package >. +-- + +module Data.Text + ( + -- * Strict vs lazy types + -- $strict + + -- * Acceptable data + -- $replacement + + -- * Definition of character + -- $character_definition + + -- * Fusion + -- $fusion + + -- * Types + Text + , StrictText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr' + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , replicate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , breakOn + , breakOnEnd + , break + , span + , spanM + , spanEndM + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + + -- ** Breaking into lines and words + , lines + --, lines' + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , breakOnAll + , find + , elem + , partition + + -- , findSubstring + + -- * Indexing + -- $index + , index + , findIndex + , count + + -- * Zipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + + -- * Low level operations + , copy + , unpackCString# + , unpackCStringAscii# + + , measureOff + ) where + +import Prelude (Char, Bool(..), Int, Maybe(..), String, + Eq, (==), (/=), Ord(..), Ordering(..), (++), + Monad(..), pure, Read(..), Show, + (&&), (||), (+), (-), (.), ($), ($!), (>>), + not, return, otherwise, quot) +import Control.DeepSeq (NFData(rnf)) +#if defined(ASSERTS) +import Control.Exception (assert) +#endif +import Data.Bits ((.&.)) +import qualified Data.Char as Char +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Control.Monad (foldM) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import qualified Data.List as L hiding (head, tail) +import qualified Data.List.NonEmpty as NonEmptyList +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import Data.Text.Internal.ArrayUtils (memchr) +import Data.Text.Internal.IsAscii (isAscii) +import Data.Text.Internal.Reverse (reverse) +import Data.Text.Internal.Measure (measure_off) +import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr3, ord2, ord3, ord4) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import Data.Text.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, unstream) +import Data.Text.Internal.Private (span_) +import Data.Text.Internal (Text(..), StrictText, empty, firstf, mul, safe, text, append, pack) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, + reverseIter_, unsafeHead, unsafeTail, iterArray, reverseIterArray) +import Data.Text.Internal.Search (indices) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, toTitleNonEmpty, filter_) +#if defined(__HADDOCK__) +import Data.ByteString (ByteString) +import qualified Data.Text.Lazy as L +#endif +import Data.Word (Word8) +import Foreign.C.Types +import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) +import qualified GHC.Exts as Exts +import GHC.Int (Int8) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Lib as TH +import qualified Language.Haskell.TH.Syntax as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) +import System.Posix.Types (CSsize(..)) + +#if __GLASGOW_HASKELL__ >= 810 +import Data.Text.Foreign (asForeignPtr) +import System.IO.Unsafe (unsafePerformIO) +#endif + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +-- $character_definition +-- +-- This package uses the term /character/ to denote Unicode /code points/. +-- +-- Note that this is not the same thing as a grapheme (e.g. a +-- composition of code points that form one visual symbol). For +-- instance, consider the grapheme \"ä\". This symbol has two +-- Unicode representations: a single code-point representation +-- @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code point), +-- and a two code point representation @U+0061@ (the \"@A@\" code +-- point) and @U+0308@ (the @COMBINING DIAERESIS@ code point). + +-- $strict +-- +-- This package provides both strict and lazy 'Text' types. The +-- strict type is provided by the "Data.Text" module, while the lazy +-- type is provided by the "Data.Text.Lazy" module. Internally, the +-- lazy @Text@ type consists of a list of strict chunks. +-- +-- The strict 'Text' type requires that an entire string fit into +-- memory at once. The lazy 'Data.Text.Lazy.Text' type is capable of +-- streaming strings that are larger than memory using a small memory +-- footprint. In many cases, the overhead of chunked streaming makes +-- the lazy 'Data.Text.Lazy.Text' type slower than its strict +-- counterpart, but this is not always the case. Sometimes, the time +-- complexity of a function in one module may be different from the +-- other, due to their differing internal structures. +-- +-- Each module provides an almost identical API, with the main +-- difference being that the strict module uses 'Int' values for +-- lengths and counts, while the lazy module uses 'Data.Int.Int64' +-- lengths. + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >. +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >) +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +instance Eq Text where + Text arrA offA lenA == Text arrB offB lenB + | lenA == lenB = A.equal arrA offA arrB offB lenA + | otherwise = False + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +-- +-- Beware: @stimes@ will crash if the given number does not fit into +-- an @Int@. +instance Semigroup Text where + (<>) = append + + stimes howManyTimes + | howManyTimes < 0 = P.error "Data.Text.stimes: given number is negative!" + | otherwise = + let howManyTimesInt = P.fromIntegral howManyTimes :: Int + in if P.fromIntegral howManyTimesInt == howManyTimes && howManyTimesInt >= 0 + then replicate howManyTimesInt + else P.error "Data.Text.stimes: given number does not fit into an Int!" + + sconcat = concat . NonEmptyList.toList + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where rnf !_ = () + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (lengthWord8 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Set".@'Data.Set.Set' and @"Data.Map".@'Data.Map.Map'. If you +-- feel a mistake has been made, please feel free to submit +-- improvements. +-- +-- The original discussion is archived here: +-- <https://mail.haskell.org/pipermail/haskell-cafe/2010-January/072379.html could we get a Data instance for Data.Text.Text? > +-- +-- The followup discussion that changed the behavior of 'Data.Set.Set' +-- and 'Data.Map.Map' is archived here: +-- <https://mail.haskell.org/pipermail/libraries/2012-August/018366.html Proposal: Allow gunfold for Data.Map, ... > + +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> P.error "gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 914 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> [| empty |] + _ -> + let + bytesQ = TH.liftAddrCompat ptr 0 (P.fromIntegral len) + lenQ = TH.liftIntCompat (P.fromIntegral len) + in [| unpackCStringLen# $bytesQ $lenQ |] +#elif __GLASGOW_HASKELL__ >= 810 + lift txt = do + let (ptr, len) = unsafePerformIO $ asForeignPtr txt + case len of + 0 -> TH.varE 'empty + _ -> + let + bytesQ = TH.litE . TH.bytesPrimL $ TH.mkBytes ptr 0 (P.fromIntegral len) + lenQ = liftInt (P.fromIntegral len) + liftInt n = (TH.appE (TH.conE 'Exts.I#) (TH.litE (TH.IntPrimL n))) + in TH.varE 'unpackCStringLen# `TH.appE` bytesQ `TH.appE` lenQ +#else + lift = TH.appE (TH.varE 'pack) . TH.stringE . unpack +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +#if __GLASGOW_HASKELL__ >= 810 +unpackCStringLen# :: Exts.Addr# -> Int -> Text +unpackCStringLen# addr# l = Text ba 0 l + where + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Exts.Ptr addr#) l + A.unsafeFreeze marr +{-# NOINLINE unpackCStringLen# #-} -- set as NOINLINE to avoid generated code bloat +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Text" [packConstr] + +-- | /O(n)/ Compare two 'Text' values lexicographically. +compareText :: Text -> Text -> Ordering +compareText (Text arrA offA lenA) (Text arrB offB lenB) = + A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(n)/ Adds a character to the front of a 'Text'. This function +-- is more costly than its 'List' counterpart because it requires +-- copying a new array. Performs replacement on +-- invalid scalar values. +cons :: Char -> Text -> Text +cons c (Text srcArr srcOff srcLen) = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = chLen + srcLen + marr <- A.new totalLen + _ <- unsafeWrite marr 0 ch + A.copyI srcLen marr chLen srcArr srcOff + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +-- Performs replacement on invalid scalar values. +snoc :: Text -> Char -> Text +snoc (Text srcArr srcOff srcLen) c = runST $ do + let ch = safe c + chLen = utf8Length ch + totalLen = srcLen + chLen + marr <- A.new totalLen + A.copyI srcLen marr 0 srcArr srcOff + _ <- unsafeWrite marr srcLen ch + arr <- A.unsafeFreeze marr + pure $ Text arr 0 totalLen +{-# INLINE snoc #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t + | null t = emptyError "head" + | otherwise = let Iter c _ = iter t 0 in c +{-# INLINE head #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons t@(Text arr off len) + | len <= 0 = Nothing + | otherwise = Just $ let !(Iter c d) = iter t 0 + in (c, text arr (off+d) (len-d)) +{-# INLINE [1] uncons #-} + +-- | /O(1)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last t@(Text _ _ len) + | null t = emptyError "last" + | otherwise = let Iter c _ = reverseIter t (len - 1) in c +{-# INLINE [1] last #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail t@(Text arr off len) + | null t = emptyError "tail" + | otherwise = text arr (off+d) (len-d) + where d = iter_ t 0 +{-# INLINE [1] tail #-} + +-- | /O(1)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init t@(Text arr off len) + | null t = emptyError "init" + | otherwise = text arr off (len + reverseIter_ t (len - 1)) +{-# INLINE [1] init #-} + +-- | /O(1)/ Returns all but the last character and the last character of a +-- 'Text', or 'Nothing' if empty. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc t@(Text arr off len) + | null t = Nothing + | otherwise = Just (text arr off (len + d), c) + where + Iter c d = reverseIter t (len - 1) +{-# INLINE [1] unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null (Text _arr _off len) = +#if defined(ASSERTS) + assert (len >= 0) $ +#endif + len <= 0 +{-# INLINE [1] null #-} + +{-# RULES + "TEXT null/empty -> True" null empty = True +#-} + +-- | Bidirectional pattern synonym for 'empty' and 'null' (both /O(1)/), +-- to be used together with '(:<)' or '(:>)'. +-- +-- @since 2.1.2 +pattern Empty :: Text +pattern Empty <- (null -> True) where + Empty = empty + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton (Text arr off len) = + len /= 0 && len == utf8LengthByLeader (A.unsafeIndex arr off) +{-# INLINE isSingleton #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Int +length = P.negate . measureOff P.maxBound +{-# INLINE [1] length #-} +-- length needs to be phased after the compareN/length rules otherwise +-- it may inline before the rules have an opportunity to fire. + +{-# RULES +"TEXT length/filter -> S.length/S.filter" forall p t. + length (filter p t) = S.length (S.filter p (stream t)) +"TEXT length/unstream -> S.length" forall t. + length (unstream t) = S.length t +"TEXT length/pack -> P.length" forall t. + length (pack t) = P.length t +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = mul (max 0 n) (length t) +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (mul 2 (length t) - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) +"TEXT length/empty -> 0" + length empty = 0 + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +{-# RULES +"TEXT compareN/length -> compareLength" [~1] forall t n. + compare (length t) n = compareLength t n + #-} + +{-# RULES +"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n. + eqInt (length t) n = compareLength t n == EQ + #-} + +{-# RULES +"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n. + neInt (length t) n = compareLength t n /= EQ + #-} + +{-# RULES +"TEXT <N/length -> compareLength/==LT" [~1] forall t n. + ltInt (length t) n = compareLength t n == LT + #-} + +{-# RULES +"TEXT <=N/length -> compareLength//=GT" [~1] forall t n. + leInt (length t) n = compareLength t n /= GT + #-} + +{-# RULES +"TEXT >N/length -> compareLength/==GT" [~1] forall t n. + gtInt (length t) n = compareLength t n == GT + #-} + +{-# RULES +"TEXT >=N/length -> compareLength//=LT" [~1] forall t n. + geInt (length t) n = compareLength t n /= LT + #-} + +-- ----------------------------------------------------------------------------- +-- * Transformations +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- +-- Example: +-- +-- >>> let message = pack "I am not angry. Not at all." +-- >>> T.map (\c -> if c == '.' then '!' else c) message +-- "I am not angry! Not at all!" +-- +-- Performs replacement on invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = \t -> if null t then empty else mapNonEmpty f t +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +-- +-- Example: +-- +-- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] +-- "WeNI!seekNI!theNI!HolyNI!Grail" +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. +-- +-- Example: +-- +-- >>> T.intersperse '.' "SHIELD" +-- "S.H.I.E.L.D" +-- +-- Performs replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t@(Text src o l) = if null t then empty else runST $ do + let !cLen = utf8Length c + dstLen = l + length t P.* cLen + + dst <- A.new dstLen + + let writeSep = case cLen of + 1 -> \dstOff -> + A.unsafeWrite dst dstOff (ord8 c) + 2 -> let (c0, c1) = ord2 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + A.unsafeWrite dst (dstOff + 3) c3 + let go !srcOff !dstOff = if srcOff >= o + l then return () else do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff m0 + writeSep (dstOff + 1) + go (srcOff + 1) (dstOff + 1 + cLen) + 2 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + writeSep (dstOff + 2) + go (srcOff + 2) (dstOff + 2 + cLen) + 3 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + writeSep (dstOff + 3) + go (srcOff + 3) (dstOff + 3 + cLen) + _ -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + writeSep (dstOff + 4) + go (srcOff + 4) (dstOff + 4 + cLen) + + go o 0 + arr <- A.unsafeFreeze dst + return (Text arr 0 (dstLen - cLen)) +{-# INLINE [1] intersperse #-} + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- >>> replace "oo" "foo" "oo" +-- "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- >>> replace "ofo" "bar" "ofofo" +-- "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace needle@(Text _ _ neeLen) + (Text repArr repOff repLen) + haystack@(Text hayArr hayOff hayLen) + | neeLen == 0 = emptyError "replace" + | len == 0 = empty -- if also haystack is empty, we can't just return 'haystack' as worker/wrapper might duplicate it + | L.null ixs = haystack + | otherwise = Text (A.run x) 0 len + where + ixs = indices needle haystack + len = hayLen - (neeLen - repLen) `mul` L.length ixs + x :: ST s (A.MArray s) + x = do + marr <- A.new len + let loop (i:is) o d = do + let d0 = d + i - o + d1 = d0 + repLen + A.copyI (i - o) marr d hayArr (hayOff+o) + A.copyI repLen marr d0 repArr repOff + loop is (i + neeLen) d1 + loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) + loop ixs 0 0 + return marr + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- When case converting 'Text' values, do not use combinators like +-- @map toUpper@ to case convert each character of a string +-- individually, as this gives incorrect results according to the +-- rules of some writing systems. The whole-string case conversion +-- functions from this module, such as @toUpper@, obey the correct +-- case conversion rules. As a result, these functions may map one +-- input character to two or three output characters. For examples, +-- see the documentation of each function. +-- +-- /Note/: In some languages, case conversion is a locale- and +-- context-dependent operation. The case conversion functions in this +-- module are /not/ locale sensitive. Programs that require locale +-- sensitivity should use appropriate versions of the +-- <http://hackage.haskell.org/package/text-icu-0.6.3.7/docs/Data-Text-ICU.html#g:4 case mapping functions from the text-icu package >. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (also known +-- as case insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature \"ﬓ\" (men now, U+FB13) is case +-- folded to the sequence \"մ\" (men, U+0574) followed by +-- \"ն\" (now, U+0576), while the Greek \"µ\" (micro sign, +-- U+00B5) is case folded to \"μ\" (small letter mu, U+03BC) +-- instead of itself. +toCaseFold :: Text -> Text +toCaseFold = \t -> + if null t then empty + else toCaseFoldNonEmpty t +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, \"İ\" (Latin capital letter I with dot above, +-- U+0130) maps to the sequence \"i\" (Latin small letter i, U+0069) +-- followed by \" ̇\" (combining dot above, U+0307). +toLower :: Text -> Text +toLower = \t -> + if null t then empty + else toLowerNonEmpty t +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German \"ß\" (eszett, U+00DF) maps to the +-- two-letter sequence \"SS\". +toUpper :: Text -> Text +toUpper = \t -> + if null t then empty + else toUpperNonEmpty t +{-# INLINE toUpper #-} + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = \t -> + if null t then empty + else toTitleNonEmpty t +{-# INLINE toTitle #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. +-- Performs replacement on invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyLeft 7 'x' "foo" +-- "fooxxxx" +-- +-- >>> justifyLeft 3 'x' "foobar" +-- "foobar" +justifyLeft :: Int -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChar (k-len) c + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- >>> justifyRight 7 'x' "bar" +-- "xxxxbar" +-- +-- >>> justifyRight 3 'x' "foobar" +-- "foobar" +justifyRight :: Int -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChar (k-len) c `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- >>> center 8 'x' "HS" +-- "xxxHSxxx" +center :: Int -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChar l c `append` t `append` replicateChar r c + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +-- +-- Examples: +-- +-- >>> transpose ["green","orange"] +-- ["go","rr","ea","en","ng","e"] +-- +-- >>> transpose ["blue","red"] +-- ["br","le","ud","e"] +transpose :: [Text] -> [Text] +transpose ts = P.map pack (L.transpose (P.map unpack ts)) + +-- ----------------------------------------------------------------------------- +-- * Reducing 'Text's (folds) + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z (Text arr off len) = go (off + len - 1) + where + go !i + | i < off = z + | otherwise = let !(Iter c l) = reverseIterArray arr i in f (go (i + l)) c +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) (f acc c) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = case uncons t of + Nothing -> emptyError "foldl" + Just (c, t') -> foldl f c t' +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = case uncons t of + Nothing -> emptyError "foldl'" + Just (c, t') -> foldl' f c t' +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z (Text arr off len) = go off z + where + go !i !acc + | i >= off + len = pure acc + | otherwise = let !(Iter c l) = iterArray arr i in go (i + l) P.=<< f acc c +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- If the binary operator is strict in its second argument, use 'foldr'' +-- instead. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +-- +-- Searches from left to right with short-circuiting behavior can +-- also be defined using 'foldr' (/e.g./, 'any', 'all', 'find', 'elem'). +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z (Text arr off len) = go off + where + go !i + | i >= off + len = z + | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l)) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = case unsnoc t of + Nothing -> emptyError "foldr1" + Just (t', c) -> foldr f c t' +{-# INLINE foldr1 #-} + +-- | /O(n)/ A strict version of 'foldr'. +-- +-- 'foldr'' evaluates as a right-to-left traversal using constant stack space. +-- +-- @since 2.0.1 +foldr' :: (Char -> a -> a) -> a -> Text -> a +foldr' f z (Text arr off len) = go (off + len - 1) z + where + go !i !acc + | i < off = acc + | otherwise = let !(Iter c l) = reverseIterArray arr i in go (i + l) (f c acc) +{-# INLINE foldr' #-} + +-- ----------------------------------------------------------------------------- +-- ** Special folds + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat ts = case ts of + [] -> empty + [t] -> t + _ | len == 0 -> empty + | otherwise -> Text (A.run go) 0 len + where + len = sumP "concat" $ L.map lengthWord8 ts + go :: ST s (A.MArray s) + go = do + arr <- A.new len + let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) + foldM step 0 ts >> return arr + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p = foldr (\c acc -> p c || acc) False +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p = foldr (\c acc -> p c && acc) True +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum = foldl1' max +-- This could be implemented faster: look for the longest +-- and largest UTF-8 sequence, then decode it to Char only once, +-- instead of decoding all characters, but I doubt anyone cares +-- about the performance of 'maximum' much. +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum = foldl1' min +-- This could be implemented faster, see the comment for 'maximum' above. +{-# INLINE minimum #-} + +-- ----------------------------------------------------------------------------- +-- * Building 'Text's +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + marr <- A.new l' + d' <- unsafeWrite marr 0 c0' + outer marr l' o d' c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Char -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !c + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + pure $ Text arr 0 dstOff + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff c + | otherwise = do + let !(Iter c' d) = iterArray src srcOff + c'' = safe $ f c c' + d' <- unsafeWrite dst dstOff c'' + inner (srcOff + d) (dstOff + d') c'' + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t | null t = empty + | otherwise = scanl f (unsafeHead t) (unsafeTail t) +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f c0 (Text src o l) = runST $ do + let l' = l + 4 + c0' = safe c0 + !d' = utf8Length c0' + marr <- A.new l' + _ <- unsafeWrite marr (l' - d') c0' + outer marr (l + o - 1) (l' - d' - 1) c0' + where + outer :: forall s. A.MArray s -> Int -> Int -> Char -> ST s Text + outer !dst = inner + where + inner !srcOff !dstOff !c + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + pure $ Text arr (dstOff + 1) (dstLen - dstOff - 1) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) c + | otherwise = do + let !(Iter c' d) = reverseIterArray src srcOff + c'' = safe $ f c' c + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' c'' + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) +{-# INLINE scanr1 #-} + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumL f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !z + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (z, Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff z + | otherwise = do + let !(Iter c d) = iterArray src srcOff + (z', c') = f z c + d' <- unsafeWrite dst dstOff (safe c') + inner (srcOff + d) (dstOff + d') z' + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. +-- Performs replacement on invalid scalar values. +mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumR f z0 (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + o - 1) (l + 4 - 1) z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) + outer !dst = inner + where + inner !srcOff !dstOff !z + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) z + | otherwise = do + let !(Iter c d) = reverseIterArray src srcOff + (z', c') = f z c + c'' = safe c' + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' z' + +-- ----------------------------------------------------------------------------- +-- ** Generating and unfolding 'Text's + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int -> Text -> Text +replicate n t@(Text a o l) + | n <= 0 || l <= 0 = empty + | n == 1 = t + | isSingleton t = replicateChar n (unsafeHead t) + | otherwise = runST $ do + let totalLen = n `mul` l + marr <- A.new totalLen + A.copyI l marr 0 a o + A.tile marr l + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen +{-# INLINE [1] replicate #-} + +{-# RULES +"TEXT replicate/singleton -> replicateChar" [~1] forall n c. + replicate n (singleton c) = replicateChar n c + #-} + +-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the +-- value of every element. +replicateChar :: Int -> Char -> Text +replicateChar !len !c' + | len <= 0 = empty + | Char.isAscii c = runST $ do + marr <- A.newFilled len (Char.ord c) + arr <- A.unsafeFreeze marr + return $ Text arr 0 len + | otherwise = runST $ do + let cLen = utf8Length c + totalLen = cLen P.* len + marr <- A.new totalLen + _ <- unsafeWrite marr 0 c + A.tile marr cLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen + where + c = safe c' +{-# INLINE replicateChar #-} + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- ----------------------------------------------------------------------------- +-- * Substrings + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int -> Text -> Text +take n t@(Text arr off len) + | n <= 0 = empty + | n >= len || m >= len || m < 0 = t + | otherwise = Text arr off m + where + m = measureOff n t +{-# INLINE [1] take #-} + +-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ +-- returns a non-negative number, measuring their size in 'Word8'. Otherwise, +-- if @t@ is shorter, return a non-positive number, which is a negated total count +-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. +-- +-- This function is used to implement 'take', 'drop', 'splitAt' and 'length' +-- and is useful on its own in streaming and parsing libraries. +-- +-- @since 2.0 +measureOff :: Int -> Text -> Int +measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else + cSsizeToInt $ + measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> takeEnd 3 "foobar" +-- "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int -> Text -> Text +takeEnd n t@(Text arr off len) + | n <= 0 = empty + | n >= len = t + | otherwise = text arr (off+i) (len-i) + where i = iterNEnd n t + +iterNEnd :: Int -> Text -> Int +iterNEnd n t@(Text _arr _off len) = loop (len-1) n + where loop i !m + | m <= 0 = i+1 + | i <= 0 = 0 + | otherwise = loop (i+d) (m-1) + where d = reverseIter_ t i + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int -> Text -> Text +drop n t@(Text arr off len) + | n <= 0 = t + | n >= len || m >= len || m < 0 = empty + | otherwise = Text arr (off+m) (len-m) + where m = measureOff n t +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- >>> dropEnd 3 "foobar" +-- "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int -> Text -> Text +dropEnd n t@(Text arr off len) + | n <= 0 = t + | n >= len = empty + | otherwise = text arr off (iterNEnd n t) + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t@(Text arr off len) = loop 0 + where loop !i | i >= len = t + | p c = loop (i+d) + | otherwise = text arr off i + where Iter c d = iter t i +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- >>> takeWhileEnd (=='o') "foo" +-- "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = t + | p c = loop (i+d) (l+d) + | otherwise = text arr (off+l) (len-l) + where Iter c d = reverseIter t i +{-# INLINE [1] takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t@(Text arr off len) = loop 0 0 + where loop !i !l | l >= len = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr (off+i) (len-l) + where Iter c d = iter t i +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- >>> dropWhileEnd (=='.') "foo..." +-- "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p t@(Text arr off len) = loop (len-1) len + where loop !i !l | l <= 0 = empty + | p c = loop (i+d) (l+d) + | otherwise = Text arr off l + where Iter c d = reverseIter t i +{-# INLINE [1] dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile Char.isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd Char.isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround Char.isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int -> Text -> (Text, Text) +splitAt n t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len || m < 0 = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m = measureOff n t + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p t = case span_ p t of + (# hd,tl #) -> (hd,tl) +{-# INLINE span #-} + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is +-- over elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p = span (not . p) +{-# INLINE break #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t@(Text arr off len) = go 0 + where + go !i | i < len = case iterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off i, text arr (off+i) (len-i)) + go _ = pure (t, empty) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t@(Text arr off len) = go (len-1) + where + go !i | 0 <= i = case reverseIterArray arr (off+i) of + Iter c l -> do + continue <- p c + if continue then go (i+l) + else pure (text arr off (i+1), text arr (off+i+1) (len-i-1)) + go _ = pure (empty, t) +{-# INLINE spanEndM #-} + +-- | /O(n)/ Group characters in a string according to a predicate. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy p = loop + where + loop t@(Text arr off len) + | null t = [] + | otherwise = text arr off n : loop (text arr (off+n) (len-n)) + where Iter c d = iter t 0 + n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) + +-- | Returns the /array/ index (in units of 'Word8') at which a +-- character may be found. This is /not/ the same as the logical +-- index returned by e.g. 'findIndex'. +findAIndexOrEnd :: (Char -> Bool) -> Text -> Int +findAIndexOrEnd q t@(Text _arr _off len) = go 0 + where go !i | i >= len || q c = i + | otherwise = go (i+d) + where Iter c d = iter t i + +-- | /O(n)/ Group characters in a string by equality. +group :: Text -> [Text] +group = groupBy (==) + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +inits :: Text -> [Text] +inits = (NonEmptyList.toList $!) . initsNE + +-- | /O(n)/ Return all initial segments of the given 'Text', shortest +-- first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmptyList.NonEmpty Text +initsNE t = empty NonEmptyList.:| case t of + Text arr off len -> + let loop i + | i >= len = [] + | otherwise = let !j = i + iter_ t i in Text arr off j : loop j + in loop 0 + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NonEmptyList.toList $!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmptyList.NonEmpty Text +tailsNE t + | null t = empty NonEmptyList.:| [] + | otherwise = t NonEmptyList.:| tails (unsafeTail t) + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be empty), consuming the delimiter. An empty +-- delimiter is invalid, and will cause an error to be raised. +-- +-- Examples: +-- +-- >>> splitOn "\r\n" "a\r\nb\r\nd\r\ne" +-- ["a","b","d","e"] +-- +-- >>> splitOn "aaa" "aaaXaaaXaaaXaaa" +-- ["","X","X","X",""] +-- +-- >>> splitOn "x" "x" +-- ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat@(Text _ _ l) src@(Text arr off len) + | l <= 0 = emptyError "splitOn" + | isSingleton pat = split (== unsafeHead pat) src + | otherwise = go 0 (indices pat src) + where + go !s (x:xs) = text arr (s+off) (x-s) : go (x+l) xs + go s _ = [text arr (s+off) (len-s)] +{-# INLINE [1] splitOn #-} + +{-# RULES +"TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- >>> split (=='a') "aabbaca" +-- ["","","bb","c",""] +-- +-- >>> split (=='a') "" +-- [""] +split :: (Char -> Bool) -> Text -> [Text] +split p t + | null t = [empty] + | otherwise = loop t + where loop s | null s' = [l] + | otherwise = l : loop (unsafeTail s') + where (# l, s' #) = span_ (not . p) s +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- >>> chunksOf 3 "foobarbaz" +-- ["foo","bar","baz"] +-- +-- >>> chunksOf 4 "haskell.org" +-- ["hask","ell.","org"] +chunksOf :: Int -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- ---------------------------------------------------------------------------- +-- * Searching + +------------------------------------------------------------------------------- +-- ** Searching with a predicate + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem = any . (==) +-- TODO This can be implemented much faster: there is no need to decode +-- any UTF-8 sequences at all. +{-# INLINE elem #-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element matching the predicate, or 'Nothing' if +-- there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p = foldr (\c acc -> if p c then Just c else acc) Nothing +{-# INLINE find #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = filter_ text p +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- >>> breakOn "::" "a::b::c" +-- ("a","::b::c") +-- +-- >>> breakOn "/" "foobar" +-- ("foobar","") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src@(Text arr off len) + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> (text arr off x, text arr (off+x) (len-x)) +{-# INLINE breakOn #-} + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the +-- string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- >>> breakOnEnd "::" "a::b::c" +-- ("a::b::","c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = (reverse b, reverse a) + where (a,b) = breakOn (reverse pat) (reverse src) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- >>> breakOnAll "::" "" +-- [] +-- +-- >>> breakOnAll "/" "a/b/c/" +-- [("a","/b/c/"),("a/b","/c/"),("a/b/c","/")] +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src@(Text arr off slen) + | null pat = emptyError "breakOnAll" + | otherwise = L.map step (indices pat src) + where + step x = (chunk 0 x, chunk x (slen-x)) + chunk !n !l = text arr (n+off) l +{-# INLINE breakOnAll #-} + +------------------------------------------------------------------------------- +-- ** Indexing 'Text's + +-- $index +-- +-- If you think of a 'Text' value as an array of 'Char' values (which +-- it is not), you run the risk of writing inefficient code. +-- +-- An idiom that is common in some languages is to find the numeric +-- offset of a character or substring, then use that number to split +-- or trim the searched string. With a 'Text' value, this approach +-- would require two /O(n)/ operations: one to perform the search, and +-- one to operate from wherever the search ended. +-- +-- For example, suppose you have a string that you want to split on +-- the substring @\"::\"@, such as @\"foo::bar::quux\"@. Instead of +-- searching for the index of @\"::\"@ and taking the substrings +-- before and after that index, you would instead use @breakOnAll \"::\"@. + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int -> Char +index t@(Text _ _ lenInBytes) ix + | ix < 0 + = P.error $ "Data.Text.index: negative index " ++ P.show ix + | off < 0 || off == lenInBytes + = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + | otherwise = ch + where + off = measureOff ix t + Iter ch _ = iter t off +{-# INLINE index #-} + +-- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' +-- and returns the index of the first element in the 'Text' satisfying +-- the predicate. +findIndex :: (Char -> Bool) -> Text -> Maybe Int +findIndex p t = S.findIndex p (stream t) +{-# INLINE findIndex #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int +count pat + | null pat = emptyError "count" + | isSingleton pat = countChar (unsafeHead pat) + | otherwise = L.length . indices pat +{-# INLINE [1] count #-} + +{-# RULES +"TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int +countChar c = foldl' (\acc c' -> if c == c' then acc + 1 else acc) 0 +{-# INLINE countChar #-} + +------------------------------------------------------------------------------- +-- * Zipping + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [1] zipWith #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words (Text arr off len) = loop 0 0 + where + loop !start !n + | n >= len = if start == n + then [] + else [Text arr (start + off) (n - start)] + -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 + | isAsciiSpace w0 = + if start == n + then loop (n + 1) (n + 1) + else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) + | w0 < 0x80 = loop start (n + 1) + -- or 2 bytes for 0xA0 + | w0 == 0xC2, w1 == 0xA0 = + if start == n + then loop (n + 2) (n + 2) + else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) + | w0 < 0xE0 = loop start (n + 2) + -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 + | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 + || w0 == 0xE2 && (w1 == 0x80 && Char.isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) + || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = + if start == n + then loop (n + 3) (n + 3) + else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) + | otherwise = loop start (n + utf8LengthByLeader w0) + where + w0 = A.unsafeIndex arr (off + n) + w1 = A.unsafeIndex arr (off + n + 1) + w2 = A.unsafeIndex arr (off + n + 2) +{-# INLINE words #-} + +-- Adapted from Data.ByteString.Internal.isSpaceWord8 +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) +{-# INLINE isAsciiSpace #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines (Text arr@(A.ByteArray arr#) off len) = go off + where + go !n + | n >= len + off = [] + | delta < 0 = [Text arr n (len + off - n)] + | otherwise = Text arr n delta : go (n + delta + 1) + where + delta = memchr arr# n (len + off - n) 0x0A +{-# INLINE lines #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr bOff aLen +{-# INLINE [1] isPrefixOf #-} + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf a@(Text _aArr _aOff aLen) b@(Text bArr bOff bLen) = + d >= 0 && a == b' + where d = bLen - aLen + b' | d == 0 = b + | otherwise = Text bArr (bOff+d) aLen +{-# INLINE isSuffixOf #-} + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (unsafeHead needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripPrefix "foo" "foobar" +-- Just "bar" +-- +-- >>> stripPrefix "" "baz" +-- Just "baz" +-- +-- >>> stripPrefix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p@(Text _arr _off plen) t@(Text arr off len) + | p `isPrefixOf` t = Just $! text arr (off+plen) (len-plen) + | otherwise = Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- >>> commonPrefixes "foobar" "fooquux" +-- Just ("foo","bar","quux") +-- +-- >>> commonPrefixes "veeble" "fetzer" +-- Nothing +-- +-- >>> commonPrefixes "" "baz" +-- Nothing +commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) +commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) + | len0 == 0 = Nothing + | len1 == 0 = Nothing + | otherwise = go 0 0 + where + go !i !j + | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) + | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) + | a == b = go (i + 1) k + | k > 0 = Just (Text arr0 off0 k, + Text arr0 (off0 + k) (len0 - k), + Text arr1 (off1 + k) (len1 - k)) + | otherwise = Nothing + where + a = A.unsafeIndex arr0 (off0 + i) + b = A.unsafeIndex arr1 (off1 + i) + isLeader = word8ToInt8 a >= -64 + k = if isLeader then i else j +{-# INLINE commonPrefixes #-} + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- >>> stripSuffix "bar" "foobar" +-- Just "foo" +-- +-- >>> stripSuffix "" "baz" +-- Just "baz" +-- +-- >>> stripSuffix "foo" "quux" +-- Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p@(Text _arr _off plen) t@(Text arr off len) + | p `isSuffixOf` t = Just $! text arr off (len-plen) + | otherwise = Nothing + +-- | Add a list of non-negative numbers. Errors out on overflow. +sumP :: String -> [Int] -> Int +sumP fun = L.foldl' add 0 + where add a x + | ax >= 0 = ax + | otherwise = overflowError fun + where ax = a + x +{-# INLINE sumP #-} -- Use foldl' and inline for fusion. + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" + +overflowError :: HasCallStack => String -> a +overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" + +-- | Convert a value to 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +-- | /O(n)/ Make a distinct copy of the given string, sharing no +-- storage with the original string. +-- +-- As an example, suppose you read a large string, of which you need +-- only a small portion. If you do not use 'copy', the entire original +-- array will be kept alive in memory by the smaller string. Making a +-- copy \"breaks the link\" to the original array, allowing it to be +-- garbage collected if there are no other live references to it. +copy :: Text -> Text +copy t@(Text arr off len) + | null t = empty + | otherwise = Text (A.run go) 0 len + where + go :: ST s (A.MArray s) + go = do + marr <- A.new len + A.copyI len marr 0 arr off + return marr + +ord8 :: Char -> Word8 +ord8 = P.fromIntegral . Char.ord + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = P.fromIntegral + +word8ToInt8 :: Word8 -> Int8 +word8ToInt8 = P.fromIntegral + +------------------------------------------------- +-- NOTE: the named chunk below used by doctest; +-- verify the doctests via `doctest -fobject-code Data/Text.hs` + +-- $setup +-- >>> :set -XOverloadedStrings +-- >>> import qualified Data.Text as T
src/Data/Text/Array.hs view
@@ -1,339 +1,361 @@-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE PatternSynonyms #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE UnboxedTuples #-}-{-# LANGUAGE UnliftedFFITypes #-}--{-# OPTIONS_GHC -fno-warn-unused-matches #-}--- |--- Module : Data.Text.Array--- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Packed, unboxed, heap-resident arrays. Suitable for performance--- critical use, both in terms of large data quantities and high--- speed.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions, e.g.------ > import qualified Data.Text.Array as A------ The names in this module resemble those in the 'Data.Array' family--- of modules, but are shorter due to the assumption of qualified--- naming.-module Data.Text.Array- (- -- * Types- Array- , pattern ByteArray- , MArray- , pattern MutableByteArray- -- * Functions- , resizeM- , shrinkM- , copyM- , copyI- , copyFromPointer- , copyToPointer- , empty- , equal- , compare- , run- , run2- , toList- , unsafeFreeze- , unsafeIndex- , new- , newPinned- , newFilled- , unsafeWrite- , tile- , getSizeofMArray- ) where--#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif-#if !MIN_VERSION_base(4,11,0)-import Foreign.C.Types (CInt(..))-#endif-import GHC.Exts hiding (toList)-import GHC.ST (ST(..), runST)-import GHC.Word (Word8(..))-import qualified Prelude-import Prelude hiding (length, read, compare)-import Data.Array.Byte (ByteArray(..), MutableByteArray(..))---- | Immutable array type.-type Array = ByteArray---- | Mutable array type, for use in the ST monad.-type MArray = MutableByteArray---- | Create an uninitialized mutable array.-new :: forall s. Int -> ST s (MArray s)-new (I# len#)-#if defined(ASSERTS)- | I# len# < 0 = error "Data.Text.Array.new: size overflow"-#endif- | otherwise = ST $ \s1# ->- case newByteArray# len# s1# of- (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)-{-# INLINE new #-}---- | Create an uninitialized mutable pinned array.------ @since 2.0-newPinned :: forall s. Int -> ST s (MArray s)-newPinned (I# len#)-#if defined(ASSERTS)- | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow"-#endif- | otherwise = ST $ \s1# ->- case newPinnedByteArray# len# s1# of- (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #)-{-# INLINE newPinned #-}---- | @since 2.0-newFilled :: Int -> Int -> ST s (MArray s)-newFilled (I# len#) (I# c#) = ST $ \s1# ->- case newByteArray# len# s1# of- (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of- s3# -> (# s3#, MutableByteArray marr# #)-{-# INLINE newFilled #-}---- | @since 2.0-tile :: MArray s -> Int -> ST s ()-tile marr tileLen = do- totalLen <- getSizeofMArray marr- let go l- | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l)- | otherwise = copyM marr l marr 0 l >> go (2 * l)- go tileLen-{-# INLINE tile #-}---- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!-unsafeFreeze :: MArray s -> ST s Array-unsafeFreeze (MutableByteArray marr) = ST $ \s1# ->- case unsafeFreezeByteArray# marr s1# of- (# s2#, ba# #) -> (# s2#, ByteArray ba# #)-{-# INLINE unsafeFreeze #-}---- | Unchecked read of an immutable array. May return garbage or--- crash on an out-of-bounds access.-unsafeIndex ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Array -> Int -> Word8-unsafeIndex (ByteArray arr) i@(I# i#) =-#if defined(ASSERTS)- let word8len = I# (sizeofByteArray# arr) in- if i < 0 || i >= word8len- then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len)- else-#endif- case indexWord8Array# arr i# of r# -> (W8# r#)-{-# INLINE unsafeIndex #-}---- | @since 2.0-getSizeofMArray :: MArray s -> ST s Int-getSizeofMArray (MutableByteArray marr) = ST $ \s0# ->- -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is- -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'.- case getSizeofMutableByteArray# marr s0# of- (# s1#, word8len# #) -> (# s1#, I# word8len# #)--#if defined(ASSERTS)-checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s ()-checkBoundsM ma i elSize = do- len <- getSizeofMArray ma- if i < 0 || i + elSize > len- then error ("bounds error, offset " ++ show i ++ ", length " ++ show len)- else return ()-#endif---- | Unchecked write of a mutable array. May return garbage or crash--- on an out-of-bounds access.-unsafeWrite ::-#if defined(ASSERTS)- HasCallStack =>-#endif- MArray s -> Int -> Word8 -> ST s ()-unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) =-#if defined(ASSERTS)- checkBoundsM ma i 1 >>-#endif- (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of- s2# -> (# s2#, () #))-{-# INLINE unsafeWrite #-}---- | Convert an immutable array to a list.-toList :: Array -> Int -> Int -> [Word8]-toList ary off len = loop 0- where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1)- | otherwise = []---- | An empty immutable array.-empty :: Array-empty = runST (new 0 >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result.-run :: (forall s. ST s (MArray s)) -> Array-run k = runST (k >>= unsafeFreeze)---- | Run an action in the ST monad and return an immutable array of--- its result paired with whatever else the action returns.-run2 :: (forall s. ST s (MArray s, a)) -> (Array, a)-run2 k = runST (do- (marr,b) <- k- arr <- unsafeFreeze marr- return (arr,b))-{-# INLINE run2 #-}---- | @since 2.0-resizeM :: MArray s -> Int -> ST s (MArray s)-resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# ->- case resizeMutableByteArray# ma i# s1# of- (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #)-{-# INLINE resizeM #-}---- | @since 2.0-shrinkM ::-#if defined(ASSERTS)- HasCallStack =>-#endif- MArray s -> Int -> ST s ()-shrinkM (MutableByteArray marr) i@(I# newSize) = do-#if defined(ASSERTS)- oldSize <- getSizeofMArray (MutableByteArray marr)- if I# newSize > oldSize- then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize)- else return ()-#endif- ST $ \s1# ->- case shrinkMutableByteArray# marr newSize s1# of- s2# -> (# s2#, () #)-{-# INLINE shrinkM #-}---- | Copy some elements of a mutable array.-copyM :: MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> MArray s -- ^ Source- -> Int -- ^ Source offset- -> Int -- ^ Count- -> ST s ()-copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyM: count must be >= 0, but got " ++ show count-#endif- | otherwise = do-#if defined(ASSERTS)- srcLen <- getSizeofMArray src- dstLen <- getSizeofMArray dst- if srcOff + count > srcLen- then error "copyM: source is too short"- else return ()- if dstOff + count > dstLen- then error "copyM: destination is too short"- else return ()-#endif- ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyM #-}---- | Copy some elements of an immutable array.-copyI :: Int -- ^ Count- -> MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> Array -- ^ Source- -> Int -- ^ Source offset- -> ST s ()-copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#)-#if defined(ASSERTS)- | count < 0 = error $- "copyI: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyByteArray# src# srcOff# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyI #-}---- | Copy from pointer.------ @since 2.0-copyFromPointer- :: MArray s -- ^ Destination- -> Int -- ^ Destination offset- -> Ptr Word8 -- ^ Source- -> Int -- ^ Count- -> ST s ()-copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyFromPointer: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyAddrToByteArray# src# dst# dstOff# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyFromPointer #-}---- | Copy to pointer.------ @since 2.0-copyToPointer- :: Array -- ^ Source- -> Int -- ^ Source offset- -> Ptr Word8 -- ^ Destination- -> Int -- ^ Count- -> ST s ()-copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#)-#if defined(ASSERTS)- | count < 0 = error $- "copyToPointer: count must be >= 0, but got " ++ show count-#endif- | otherwise = ST $ \s1# ->- case copyByteArrayToAddr# src# srcOff# dst# count# s1# of- s2# -> (# s2#, () #)-{-# INLINE copyToPointer #-}---- | Compare portions of two arrays for equality. No bounds checking--- is performed.-equal :: Array -> Int -> Array -> Int -> Int -> Bool-equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0-{-# INLINE equal #-}---- | Compare portions of two arrays. No bounds checking is performed.------ @since 2.0-compare :: Array -> Int -> Array -> Int -> Int -> Ordering-compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0-{-# INLINE compare #-}--compareInternal- :: Array -- ^ First- -> Int -- ^ Offset into first- -> Array -- ^ Second- -> Int -- ^ Offset into second- -> Int -- ^ Count- -> Int-compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i- where-#if MIN_VERSION_base(4,11,0)- i = I# (compareByteArrays# src1# off1# src2# off2# count#)-#else- i = fromIntegral (memcmp src1# off1# src2# off2# count#)--foreign import ccall unsafe "_hs_text_memcmp2" memcmp- :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt-#endif-{-# INLINE compareInternal #-}+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-unused-matches #-} +-- | +-- Module : Data.Text.Array +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Packed, unboxed, heap-resident arrays. Suitable for performance +-- critical use, both in terms of large data quantities and high +-- speed. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions, e.g. +-- +-- > import qualified Data.Text.Array as A +-- +-- The names in this module resemble those in the 'Data.Array' family +-- of modules, but are shorter due to the assumption of qualified +-- naming. +module Data.Text.Array + ( + -- * Types + Array + , pattern ByteArray + , MArray + , pattern MutableByteArray + -- * Functions + , resizeM + , shrinkM + , copyM + , copyI + , copyFromPointer + , copyToPointer + , empty + , equal + , compare + , run + , run2 + , toList + , unsafeFreeze + , unsafeIndex + , new + , newPinned + , newFilled + , unsafeWrite + , tile + , getSizeofMArray + ) where + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif +#if !MIN_VERSION_base(4,11,0) +import Foreign.C.Types (CInt(..)) +#endif +import GHC.Exts hiding (toList) +import GHC.ST (ST(..), runST) +import GHC.Word (Word8(..)) +import qualified Prelude +import Prelude hiding (length, read, compare) +import Data.Array.Byte (ByteArray(..), MutableByteArray(..)) + +-- | Immutable array type. +type Array = ByteArray + +-- | Mutable array type, for use in the ST monad. +type MArray = MutableByteArray + +-- | Create an uninitialized mutable array. +new :: forall s. Int -> ST s (MArray s) +new (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.new: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE new #-} + +-- | Create an uninitialized mutable pinned array. +-- +-- @since 2.0 +newPinned :: forall s. Int -> ST s (MArray s) +newPinned (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newPinnedByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE newPinned #-} + +-- | @since 2.0 +newFilled :: Int -> Int -> ST s (MArray s) +newFilled (I# len#) (I# c#) = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of + s3# -> (# s3#, MutableByteArray marr# #) +{-# INLINE newFilled #-} + +-- | @since 2.0 +tile :: MArray s -> Int -> ST s () +tile marr tileLen = do + totalLen <- getSizeofMArray marr + let go l + | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) + | otherwise = copyM marr l marr 0 l >> go (2 * l) + go tileLen +{-# INLINE tile #-} + +-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! +unsafeFreeze :: MArray s -> ST s Array +unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> + case unsafeFreezeByteArray# marr s1# of + (# s2#, ba# #) -> (# s2#, ByteArray ba# #) +{-# INLINE unsafeFreeze #-} + +-- | Unchecked read of an immutable array. May return garbage or +-- crash on an out-of-bounds access. +unsafeIndex :: +#if defined(ASSERTS) + HasCallStack => +#endif + Array -> Int -> Word8 +unsafeIndex (ByteArray arr) i@(I# i#) = +#if defined(ASSERTS) + let word8len = I# (sizeofByteArray# arr) in + if i < 0 || i >= word8len + then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) + else +#endif + case indexWord8Array# arr i# of r# -> (W8# r#) +{-# INLINE unsafeIndex #-} + +-- | @since 2.0 +getSizeofMArray :: MArray s -> ST s Int +getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> + -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is + -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. + case getSizeofMutableByteArray# marr s0# of + (# s1#, word8len# #) -> (# s1#, I# word8len# #) + +#if defined(ASSERTS) +checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () +checkBoundsM ma i elSize = do + len <- getSizeofMArray ma + if i < 0 || i + elSize > len + then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) + else return () +#endif + +-- | Unchecked write of a mutable array. May return garbage or crash +-- on an out-of-bounds access. +unsafeWrite :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> Word8 -> ST s () +unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = +#if defined(ASSERTS) + checkBoundsM ma i 1 >> +#endif + (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of + s2# -> (# s2#, () #)) +{-# INLINE unsafeWrite #-} + +-- | Convert an immutable array to a list. +toList :: Array -> Int -> Int -> [Word8] +toList ary off len = loop 0 + where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) + | otherwise = [] + +-- | An empty immutable array. +empty :: Array +empty = runST (new 0 >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result. +run :: (forall s. ST s (MArray s)) -> Array +run k = runST (k >>= unsafeFreeze) + +-- | Run an action in the ST monad and return an immutable array of +-- its result paired with whatever else the action returns. +run2 :: (forall s. ST s (MArray s, a)) -> (Array, a) +run2 k = runST (do + (marr,b) <- k + arr <- unsafeFreeze marr + return (arr,b)) +{-# INLINE run2 #-} + +-- | @since 2.0 +resizeM :: MArray s -> Int -> ST s (MArray s) +resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> + case resizeMutableByteArray# ma i# s1# of + (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) +{-# INLINE resizeM #-} + +-- | @since 2.0 +shrinkM :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> ST s () +shrinkM (MutableByteArray marr) i@(I# newSize) = do +#if defined(ASSERTS) + oldSize <- getSizeofMArray (MutableByteArray marr) + if I# newSize > oldSize + then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) + else return () +#endif + ST $ \s1# -> + case shrinkMutableByteArray# marr newSize s1# of + s2# -> (# s2#, () #) +{-# INLINE shrinkM #-} + +-- | Copy some elements of a mutable array. +copyM :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> MArray s -- ^ Source + -> Int -- ^ Source offset + -> Int -- ^ Count + -> ST s () +copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyM: count must be >= 0, but got " ++ show count +#endif + | otherwise = do +#if defined(ASSERTS) + srcLen <- getSizeofMArray src + dstLen <- getSizeofMArray dst + if srcOff + count > srcLen + then error "copyM: source is too short" + else return () + if dstOff + count > dstLen + then error "copyM: destination is too short" + else return () +#endif + ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyM #-} + +-- | Copy some elements of an immutable array. +copyI :: Int -- ^ Count + -> MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Array -- ^ Source + -> Int -- ^ Source offset + -> ST s () +copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyI: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyI #-} + +-- | Copy from pointer. +-- +-- @since 2.0 +copyFromPointer + :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Ptr Word8 -- ^ Source + -> Int -- ^ Count + -> ST s () +copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyFromPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyAddrToByteArray# src# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyFromPointer #-} + +-- | Copy to pointer. +-- +-- @since 2.0 +copyToPointer + :: Array -- ^ Source + -> Int -- ^ Source offset + -> Ptr Word8 -- ^ Destination + -> Int -- ^ Count + -> ST s () +copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyToPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArrayToAddr# src# srcOff# dst# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyToPointer #-} + +-- | Compare portions of two arrays for equality. No bounds checking +-- is performed. +equal + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Bool +equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 +{-# INLINE equal #-} + +-- | Compare portions of two arrays. No bounds checking is performed. +-- +-- @since 2.0 +compare + :: Array + -- ^ First array + -> Int + -- ^ Offset in the first array + -> Array + -- ^ Second array + -> Int + -- ^ Offset in the second array + -> Int + -- ^ How many bytes to compare? + -> Ordering +compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 +{-# INLINE compare #-} + +compareInternal + :: Array -- ^ First + -> Int -- ^ Offset into first + -> Array -- ^ Second + -> Int -- ^ Offset into second + -> Int -- ^ Count + -> Int +compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i + where +#if MIN_VERSION_base(4,11,0) + i = I# (compareByteArrays# src1# off1# src2# off2# count#) +#else + i = fromIntegral (memcmp src1# off1# src2# off2# count#) + +foreign import ccall unsafe "_hs_text_memcmp2" memcmp + :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> CInt +#endif +{-# INLINE compareInternal #-}
src/Data/Text/Encoding.hs view
@@ -1,581 +1,588 @@-{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash,- UnliftedFFITypes #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE ScopedTypeVariables #-}--- |--- Module : Data.Text.Encoding--- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts,--- (c) 2008, 2009 Tom Harper--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Functions for converting 'Text' values to and from 'ByteString',--- using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Encoding- (- -- * Decoding ByteStrings to Text- -- $strict-- -- ** Total Functions #total#- -- $total- decodeLatin1- , decodeASCIIPrefix- , decodeUtf8Lenient- , decodeUtf8'- , decodeASCII'-- -- *** Controllable error handling- , decodeUtf8With- , decodeUtf16LEWith- , decodeUtf16BEWith- , decodeUtf32LEWith- , decodeUtf32BEWith-- -- *** Stream oriented decoding- -- $stream- , streamDecodeUtf8With- , Decoding(..)-- -- *** Incremental UTF-8 decoding- -- $incremental- , decodeUtf8Chunk- , decodeUtf8More- , Utf8State- , startUtf8State- , StrictBuilder- , StrictTextBuilder- , strictBuilderToText- , textToStrictBuilder-- -- ** Partial Functions- -- $partial- , decodeASCII- , decodeUtf8- , decodeUtf16LE- , decodeUtf16BE- , decodeUtf32LE- , decodeUtf32BE-- -- *** Stream oriented decoding- , streamDecodeUtf8-- -- * Encoding Text to ByteStrings- , encodeUtf8- , encodeUtf16LE- , encodeUtf16BE- , encodeUtf32LE- , encodeUtf32BE-- -- * Encoding Text using ByteString Builders- , encodeUtf8Builder- , encodeUtf8BuilderEscaped-- -- * ByteString validation- -- $validation- , validateUtf8Chunk- , validateUtf8More- ) where--import Control.Exception (evaluate, try)-import Data.Word (Word8)-import GHC.Exts (byteArrayContents#, unsafeCoerce#)-import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr))-import Data.ByteString (ByteString)-#if defined(PURE_HASKELL)-import Control.Monad.ST.Unsafe (unsafeSTToIO)-import Data.ByteString.Char8 (unpack)-import Data.Text.Internal (pack)-import Foreign.Ptr (minusPtr, plusPtr)-import Foreign.Storable (poke)-#else-import Control.Monad.ST (runST)-import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)-import Data.Bits (shiftR, (.&.))-import Data.Text.Internal.ByteStringCompat (withBS)-import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)-import Foreign.C.Types (CSize(..))-import Foreign.Ptr (Ptr, minusPtr, plusPtr)-import Foreign.Storable (poke, peekByteOff)-#endif-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode)-import Data.Text.Internal (Text(..), empty)-import Data.Text.Internal.Encoding-import Data.Text.Internal.IsAscii (asciiPrefixLength)-import Data.Text.Unsafe (unsafeDupablePerformIO)-import Data.Text.Show ()-import qualified Data.ByteString as B-import qualified Data.ByteString.Internal as B-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Builder.Prim.Internal as BP-import qualified Data.ByteString.Short.Internal as SBS-import qualified Data.Text.Array as A-import qualified Data.Text.Internal.Encoding.Fusion as E-import qualified Data.Text.Internal.Fusion as F-#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif---- $validation--- These functions are for validating 'ByteString's as encoded text.---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- $total------ These functions facilitate total decoding and should be preferred--- over their partial counterparts.---- $partial------ These functions are partial and should only be used with great caution--- (preferably not at all). See "Data.Text.Encoding#g:total" for better--- solutions.---- | Decode a 'ByteString' containing ASCII text.------ This is a total function which returns a pair of the longest ASCII prefix--- as 'Text', and the remaining suffix as 'ByteString'.------ Important note: the pair is lazy. This lets you check for errors by testing--- whether the second component is empty, without forcing the first component--- (which does a copy).--- To drop references to the input bytestring, force the prefix--- (using 'seq' or @BangPatterns@) and drop references to the suffix.------ === Properties------ - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@.--- - Either @suffix@ is empty, or @'B.head' suffix > 127@.------ @since 2.0.2-decodeASCIIPrefix :: ByteString -> (Text, ByteString)-decodeASCIIPrefix bs = if B.null bs- then (empty, B.empty)- else- let len = asciiPrefixLength bs- prefix =- let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in- Text (A.ByteArray arr) 0 len- suffix = B.drop len bs in- (prefix, suffix)-{-# INLINE decodeASCIIPrefix #-}---- | Decode a 'ByteString' containing 7-bit ASCII encoded text.------ This is a total function which returns either the 'ByteString' converted to a--- 'Text' containing ASCII text, or 'Nothing'.------ Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid--- input instead of discarding it.------ @since 2.0.2-decodeASCII' :: ByteString -> Maybe Text-decodeASCII' bs =- let (prefix, suffix) = decodeASCIIPrefix bs in- if B.null suffix then Just prefix else Nothing-{-# INLINE decodeASCII' #-}---- | Decode a 'ByteString' containing 7-bit ASCII encoded text.------ This is a partial function: it checks that input does not contain--- anything except ASCII and copies buffer or throws an error otherwise.-decodeASCII :: ByteString -> Text-decodeASCII bs =- let (prefix, suffix) = decodeASCIIPrefix bs in- case B.uncons suffix of- Nothing -> prefix- Just (word, _) ->- let !errPos = B.length bs - B.length suffix in- error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.------ 'decodeLatin1' is semantically equivalent to--- @Data.Text.pack . Data.ByteString.Char8.unpack@------ This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII)--- characters to UTf-8 requires actual work and is not just buffer copying.----decodeLatin1 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Text-#if defined(PURE_HASKELL)-decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs)-#else-decodeLatin1 bs = withBS bs $ \fp len -> runST $ do- dst <- A.new (2 * len)- let inner srcOff dstOff = if srcOff >= len then return dstOff else do- asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len)- if asciiPrefixLen == 0- then do- byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff- A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6))- A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F))- inner (srcOff + 1) (dstOff + 2)- else do- unsafeIOToST $ unsafeWithForeignPtr fp $ \src ->- unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen- inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen)- actualLen <- inner 0 0- dst' <- A.resizeM dst actualLen- arr <- A.unsafeFreeze dst'- return $ Text arr 0 actualLen-#endif--#if !defined(PURE_HASKELL)-foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii- :: Ptr Word8 -> Ptr Word8 -> IO CSize-#endif---- $stream------ The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept--- a 'ByteString' that represents a possibly incomplete input (e.g. a--- packet from a network stream) that may not end on a UTF-8 boundary.------ 1. The maximal prefix of 'Text' that could be decoded from the--- given input.------ 2. The suffix of the 'ByteString' that could not be decoded due to--- insufficient input.------ 3. A function that accepts another 'ByteString'. That string will--- be assumed to directly follow the string that was passed as--- input to the original function, and it will in turn be decoded.------ To help understand the use of these functions, consider the Unicode--- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi--- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes.------ Now suppose that we receive this encoded string as 3 packets that--- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\",--- \"\\x83\"]@. We cannot decode the entire Unicode string until we--- have received all three packets, but we would like to make progress--- as we receive each one.------ @--- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\"--- ghci> s0--- 'Some' \"hi \" \"\\xe2\" _--- @------ We use the continuation @f0@ to decode our second packet.------ @--- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\"--- ghci> s1--- 'Some' \"\" \"\\xe2\\x98\"--- @------ We could not give @f0@ enough input to decode anything, so it--- returned an empty string. Once we feed our second continuation @f1@--- the last byte of input, it will make progress.------ @--- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\"--- ghci> s2--- 'Some' \"\\x2603\" \"\" _--- @------ If given invalid input, an exception will be thrown by the function--- or continuation where it is encountered.---- | A stream oriented decoding result.------ @since 1.0.0.0-data Decoding = Some !Text !ByteString (ByteString -> Decoding)--instance Show Decoding where- showsPrec d (Some t bs _) = showParen (d > prec) $- showString "Some " . showsPrec prec' t .- showChar ' ' . showsPrec prec' bs .- showString " _"- where prec = 10; prec' = prec + 1---- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8--- encoded text that is known to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown (either by this function or a continuation) that cannot be--- caught in pure code. For more control over the handling of invalid--- data, use 'streamDecodeUtf8With'.------ @since 1.0.0.0-streamDecodeUtf8 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Decoding-streamDecodeUtf8 = streamDecodeUtf8With strictDecode---- | Decode, in a stream oriented way, a lazy 'ByteString' containing UTF-8--- encoded text.------ @since 1.0.0.0-streamDecodeUtf8With ::-#if defined(ASSERTS)- HasCallStack =>-#endif- OnDecodeError -> ByteString -> Decoding-streamDecodeUtf8With onErr = loop startUtf8State- where- loop s chunk =- let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk- in Some (strictBuilderToText builder) undecoded (loop s')---- | Decode a 'ByteString' containing UTF-8 encoded text.------ Surrogate code points in replacement character returned by 'OnDecodeError'--- will be automatically remapped to the replacement char @U+FFFD@.-decodeUtf8With ::-#if defined(ASSERTS)- HasCallStack =>-#endif- OnDecodeError -> ByteString -> Text-decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg--invalidUtf8Msg :: String-invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream"---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code. For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.------ This is a partial function: it checks that input is a well-formed--- UTF-8 sequence and copies buffer or throws an error otherwise.----decodeUtf8 :: ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}---- | Decode a 'ByteString' containing UTF-8 encoded text.------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.-decodeUtf8' ::-#if defined(ASSERTS)- HasCallStack =>-#endif- ByteString -> Either UnicodeException Text-decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode-{-# INLINE decodeUtf8' #-}---- | Decode a 'ByteString' containing UTF-8 encoded text.------ Any invalid input bytes will be replaced with the Unicode replacement--- character U+FFFD.------ @since 2.0-decodeUtf8Lenient :: ByteString -> Text-decodeUtf8Lenient = decodeUtf8With lenientDecode---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder =- -- manual eta-expansion to ensure inlining works as expected- \txt -> B.builder (step txt)- where- step txt@(Text arr off len) !k br@(B.BufferRange op ope)- -- Ensure that the common case is not recursive and therefore yields- -- better code.- | op' <= ope = do- unsafeSTToIO $ A.copyToPointer arr off op len- k (B.BufferRange op' ope)- | otherwise = textCopyStep txt k br- where- op' = op `plusPtr` len-{-# INLINE encodeUtf8Builder #-}--textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a-textCopyStep (Text arr off len) k =- go off (off + len)- where- go !ip !ipe (B.BufferRange op ope)- | inpRemaining <= outRemaining = do- unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining- let !br = B.BufferRange (op `plusPtr` inpRemaining) ope- k br- | otherwise = do- unsafeSTToIO $ A.copyToPointer arr ip op outRemaining- let !ip' = ip + outRemaining- return $ B.bufferFull 1 ope (go ip' ipe)- where- outRemaining = ope `minusPtr` op- inpRemaining = ipe - ip---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}--- TODO: Extend documentation with references to source code in @blaze-html@--- or @aeson@ that uses this function.-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped be =- -- manual eta-expansion to ensure inlining works as expected- \txt -> B.builder (mkBuildstep txt)- where- bound = max 4 $ BP.sizeBound be-- mkBuildstep (Text arr off len) !k =- outerLoop off- where- iend = off + len-- outerLoop !i0 !br@(B.BufferRange op0 ope)- | i0 >= iend = k br- | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining)- -- TODO: Use a loop with an integrated bound's check if outRemaining- -- is smaller than 8, as this will save on divisions.- | otherwise = return $ B.bufferFull bound op0 (outerLoop i0)- where- outRemaining = (ope `minusPtr` op0) `quot` bound- inpRemaining = iend - i0-- goPartial !iendTmp = go i0 op0- where- go !i !op- | i < iendTmp = do- let w = A.unsafeIndex arr i- if w < 0x80- then BP.runB be w op >>= go (i + 1)- else poke op w >> go (i + 1) (op `plusPtr` 1)- | otherwise = outerLoop i (B.BufferRange op ope)---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> ByteString-encodeUtf8 (Text arr off len)- | len == 0 = B.empty- -- It would be easier to use Data.ByteString.Short.fromShort and slice later,- -- but this is undesirable when len is significantly smaller than length arr.- | otherwise = unsafeDupablePerformIO $ do- marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len- unsafeSTToIO $ A.copyI len marr 0 arr off- let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba))- (PlainPtr mba)- pure $ B.fromForeignPtr fp 0 len---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> ByteString-encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt))-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> ByteString-encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt))-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> ByteString-encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt))-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> ByteString-encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt))-{-# INLINE encodeUtf32BE #-}---- $incremental--- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more--- control for error-handling and streaming.------ - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error.--- For example this lets you insert or delete arbitrary text, or do some--- stateful operations before resuming, such as keeping track of error locations.--- In contrast, the older stream-oriented interface only lets you substitute--- a single fixed 'Char' for each invalid byte in 'OnDecodeError'.--- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks--- before doing the copying work to construct a 'Text', or you can--- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'.------ For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'.+{-# LANGUAGE BangPatterns, CPP, GeneralizedNewtypeDeriving, MagicHash, + UnliftedFFITypes #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +-- | +-- Module : Data.Text.Encoding +-- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts, +-- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting 'Text' values to and from 'ByteString', +-- using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- <http://hackage.haskell.org/package/text-icu text-icu package>. + +module Data.Text.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeASCIIPrefix + , decodeUtf8Lenient + , decodeUtf8' + , decodeASCII' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- *** Incremental UTF-8 decoding + -- $incremental + , decodeUtf8Chunk + , decodeUtf8More + , Utf8State + , startUtf8State + , StrictBuilder + , StrictTextBuilder + , strictBuilderToText + , textToStrictBuilder + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- ** Stream oriented decoding + -- $stream + , streamDecodeUtf8 + , streamDecodeUtf8With + , Decoding(..) + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + + -- * ByteString validation + -- $validation + , validateUtf8Chunk + , validateUtf8More + ) where + +import Control.Exception (evaluate, try) +import Data.Word (Word8) +import GHC.Exts (byteArrayContents#, unsafeCoerce#) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) +import Data.ByteString (ByteString) +#if defined(PURE_HASKELL) +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Char8 (unpack) +import Data.Text.Internal (pack) +import Foreign.Ptr (minusPtr, plusPtr) +import Foreign.Storable (poke) +#else +import Control.Monad.ST (runST) +import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) +import Data.Bits (shiftR, (.&.)) +import Data.Text.Internal.ByteStringCompat (withBS) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Foreign.C.Types (CSize(..)) +import Foreign.Ptr (Ptr, minusPtr, plusPtr) +import Foreign.Storable (poke, peekByteOff) +#endif +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Encoding +import Data.Text.Internal.IsAscii (asciiPrefixLength) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Data.Text.Show () +import qualified Data.ByteString as B +import qualified Data.ByteString.Internal as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Builder.Prim.Internal as BP +import qualified Data.ByteString.Short.Internal as SBS +import qualified Data.Text.Array as A +import qualified Data.Text.Internal.Encoding.Fusion as E +import qualified Data.Text.Internal.Fusion as F +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +-- $validation +-- These functions are for validating 'ByteString's as encoded text. + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing ASCII text. +-- +-- This is a total function which returns a pair of the longest ASCII prefix +-- as 'Text', and the remaining suffix as 'ByteString'. +-- +-- Important note: the pair is lazy. This lets you check for errors by testing +-- whether the second component is empty, without forcing the first component +-- (which does a copy). +-- To drop references to the input bytestring, force the prefix +-- (using 'seq' or @BangPatterns@) and drop references to the suffix. +-- +-- === Properties +-- +-- - If @(prefix, suffix) = decodeAsciiPrefix s@, then @'encodeUtf8' prefix <> suffix = s@. +-- - Either @suffix@ is empty, or @'B.head' suffix > 127@. +-- +-- @since 2.0.2 +decodeASCIIPrefix :: ByteString -> (Text, ByteString) +decodeASCIIPrefix bs = if B.null bs + then (empty, B.empty) + else + let len = asciiPrefixLength bs + prefix = + let !(SBS.SBS arr) = SBS.toShort (B.take len bs) in + Text (A.ByteArray arr) 0 len + suffix = B.drop len bs in + (prefix, suffix) +{-# INLINE decodeASCIIPrefix #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a total function which returns either the 'ByteString' converted to a +-- 'Text' containing ASCII text, or 'Nothing'. +-- +-- Use 'decodeASCIIPrefix' to retain the longest ASCII prefix for an invalid +-- input instead of discarding it. +-- +-- @since 2.0.2 +decodeASCII' :: ByteString -> Maybe Text +decodeASCII' bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + if B.null suffix then Just prefix else Nothing +{-# INLINE decodeASCII' #-} + +-- | Decode a 'ByteString' containing 7-bit ASCII encoded text. +-- +-- This is a partial function: it checks that input does not contain +-- anything except ASCII and copies buffer or throws an error otherwise. +decodeASCII :: ByteString -> Text +decodeASCII bs = + let (prefix, suffix) = decodeASCIIPrefix bs in + case B.uncons suffix of + Nothing -> prefix + Just (word, _) -> + let !errPos = B.length bs - B.length suffix in + error $ "decodeASCII: detected non-ASCII codepoint " ++ show word ++ " at position " ++ show errPos + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +-- +-- 'decodeLatin1' is semantically equivalent to +-- @'Data.Text.pack' . 'Data.ByteString.Char8.unpack'@ +-- +-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) +-- characters to UTf-8 requires actual work and is not just buffer copying. +-- +decodeLatin1 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Text +#if defined(PURE_HASKELL) +decodeLatin1 bs = pack (Data.ByteString.Char8.unpack bs) +#else +decodeLatin1 bs = withBS bs $ \fp len -> runST $ do + dst <- A.new (2 * len) + let inner srcOff dstOff = if srcOff >= len then return dstOff else do + asciiPrefixLen <- fmap fromIntegral $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) + if asciiPrefixLen == 0 + then do + byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff + A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) + A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) + inner (srcOff + 1) (dstOff + 2) + else do + unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen + inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) + actualLen <- inner 0 0 + dst' <- A.resizeM dst actualLen + arr <- A.unsafeFreeze dst' + return $ Text arr 0 actualLen +#endif + +#if !defined(PURE_HASKELL) +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize +#endif + +-- $stream +-- +-- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept +-- a strict 'ByteString' that represents a possibly incomplete input (e.g. a +-- packet from a network stream) that may not end on a UTF-8 boundary +-- and return 'Decoding', which consists of: +-- +-- * The maximal prefix of 'Text' that could be decoded from the +-- given input. +-- +-- * The suffix of the 'ByteString' that could not be decoded due to +-- insufficient input. +-- +-- * A function that accepts another 'ByteString'. That string will +-- be assumed to directly follow the string that was passed as +-- input to the original function, and it will in turn be decoded. +-- +-- To help understand the use of these functions, consider the Unicode +-- string @\"hi ☃\"@. If encoded as UTF-8, this becomes @\"hi +-- \\xe2\\x98\\x83\"@; the final @\'☃\'@ is encoded as 3 bytes. +-- +-- Now suppose that we receive this encoded string as 3 packets that +-- are split up on untidy boundaries: @[\"hi \\xe2\", \"\\x98\", +-- \"\\x83\"]@. We cannot decode the entire Unicode string until we +-- have received all three packets, but we would like to make progress +-- as we receive each one. +-- +-- @ +-- ghci> let s0\@('Some' _ _ f0) = 'streamDecodeUtf8' \"hi \\xe2\" +-- ghci> s0 +-- 'Some' \"hi \" \"\\xe2\" _ +-- @ +-- +-- We use the continuation @f0@ to decode our second packet. +-- +-- @ +-- ghci> let s1\@('Some' _ _ f1) = f0 \"\\x98\" +-- ghci> s1 +-- 'Some' \"\" \"\\xe2\\x98\" +-- @ +-- +-- We could not give @f0@ enough input to decode anything, so it +-- returned an empty string. Once we feed our second continuation @f1@ +-- the last byte of input, it will make progress. +-- +-- @ +-- ghci> let s2\@('Some' _ _ f2) = f1 \"\\x83\" +-- ghci> s2 +-- 'Some' \"\\x2603\" \"\" _ +-- @ +-- +-- If given invalid input, an exception will be thrown by the function +-- or continuation where it is encountered. + +-- | A stream-oriented decoding result (see 'streamDecodeUtf8' and 'streamDecodeUtf8With'). +-- +-- @since 1.0.0.0 +data Decoding = Some + !Text + -- ^ The maximal prefix that could be decoded from the given input. + !ByteString + -- ^ The remaining suffix of the input that could not be decoded + -- (usually because the input breaks in the middle of UTF-8 character) + (ByteString -> Decoding) + -- ^ The continuation call which should be fed with the next + -- chunk of the input. + +instance Show Decoding where + showsPrec d (Some t bs _) = showParen (d > prec) $ + showString "Some " . showsPrec prec' t . + showChar ' ' . showsPrec prec' bs . + showString " _" + where prec = 10; prec' = prec + 1 + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data that is known to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown (either by this function or a continuation) that cannot be +-- caught in pure code. For more control over the handling of invalid +-- data, use 'streamDecodeUtf8With'. +-- +-- @since 1.0.0.0 +streamDecodeUtf8 :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Decoding +streamDecodeUtf8 = streamDecodeUtf8With strictDecode + +-- | Initiate a stream-oriented decoding +-- with a strict 'ByteString' containing UTF-8 data. +-- +-- @since 1.0.0.0 +streamDecodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Decoding +streamDecodeUtf8With onErr = loop startUtf8State + where + loop s chunk = + let (builder, undecoded, s') = decodeUtf8With2 onErr invalidUtf8Msg s chunk + in Some (strictBuilderToText builder) undecoded (loop s') + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Surrogate code points in replacement character returned by 'OnDecodeError' +-- will be automatically remapped to the replacement char @U+FFFD@. +decodeUtf8With :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> Text +decodeUtf8With onErr = decodeUtf8With1 onErr invalidUtf8Msg + +invalidUtf8Msg :: String +invalidUtf8Msg = "Data.Text.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +-- +-- This is a partial function: it checks that input is a well-formed +-- UTF-8 sequence and copies buffer or throws an error otherwise. +-- +decodeUtf8 :: ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +decodeUtf8' :: +#if defined(ASSERTS) + HasCallStack => +#endif + ByteString -> Either UnicodeException Text +decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode +{-# INLINE decodeUtf8' #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.0 +decodeUtf8Lenient :: ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (step txt) + where + step txt@(Text arr off len) !k br@(B.BufferRange op ope) + -- Ensure that the common case is not recursive and therefore yields + -- better code. + | op' <= ope = do + unsafeSTToIO $ A.copyToPointer arr off op len + k (B.BufferRange op' ope) + | otherwise = textCopyStep txt k br + where + op' = op `plusPtr` len +{-# INLINE encodeUtf8Builder #-} + +textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a +textCopyStep (Text arr off len) k = + go off (off + len) + where + go !ip !ipe (B.BufferRange op ope) + | inpRemaining <= outRemaining = do + unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining + let !br = B.BufferRange (op `plusPtr` inpRemaining) ope + k br + | otherwise = do + unsafeSTToIO $ A.copyToPointer arr ip op outRemaining + let !ip' = ip + outRemaining + return $ B.bufferFull 1 ope (go ip' ipe) + where + outRemaining = ope `minusPtr` op + inpRemaining = ipe - ip + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +-- TODO: Extend documentation with references to source code in @blaze-html@ +-- or @aeson@ that uses this function. +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped be = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (mkBuildstep txt) + where + bound = max 4 $ BP.sizeBound be + + mkBuildstep (Text arr off len) !k = + outerLoop off + where + iend = off + len + + outerLoop !i0 !br@(B.BufferRange op0 ope) + | i0 >= iend = k br + | outRemaining > 0 = goPartial (i0 + min outRemaining inpRemaining) + -- TODO: Use a loop with an integrated bound's check if outRemaining + -- is smaller than 8, as this will save on divisions. + | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) + where + outRemaining = (ope `minusPtr` op0) `quot` bound + inpRemaining = iend - i0 + + goPartial !iendTmp = go i0 op0 + where + go !i !op + | i < iendTmp = do + let w = A.unsafeIndex arr i + if w < 0x80 + then BP.runB be w op >>= go (i + 1) + else poke op w >> go (i + 1) (op `plusPtr` 1) + | otherwise = outerLoop i (B.BufferRange op ope) + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> ByteString +encodeUtf8 (Text arr off len) + | len == 0 = B.empty + -- It would be easier to use Data.ByteString.Short.fromShort and slice later, + -- but this is undesirable when len is significantly smaller than length arr. + | otherwise = unsafeDupablePerformIO $ do + marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len + unsafeSTToIO $ A.copyI len marr 0 arr off + let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) + (PlainPtr mba) + pure $ B.fromForeignPtr fp 0 len + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> ByteString +encodeUtf16LE txt = E.unstream (E.restreamUtf16LE (F.stream txt)) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> ByteString +encodeUtf16BE txt = E.unstream (E.restreamUtf16BE (F.stream txt)) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> ByteString +encodeUtf32LE txt = E.unstream (E.restreamUtf32LE (F.stream txt)) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> ByteString +encodeUtf32BE txt = E.unstream (E.restreamUtf32BE (F.stream txt)) +{-# INLINE encodeUtf32BE #-} + +-- $incremental +-- The functions 'decodeUtf8Chunk' and 'decodeUtf8More' provide more +-- control for error-handling and streaming. +-- +-- - Those functions return an UTF-8 prefix of the given 'ByteString' up to the next error. +-- For example this lets you insert or delete arbitrary text, or do some +-- stateful operations before resuming, such as keeping track of error locations. +-- In contrast, the older stream-oriented interface only lets you substitute +-- a single fixed 'Char' for each invalid byte in 'OnDecodeError'. +-- - That prefix is encoded as a 'StrictBuilder', so you can accumulate chunks +-- before doing the copying work to construct a 'Text', or you can +-- output decoded fragments immediately as a lazy 'Data.Text.Lazy.Text'. +-- +-- For even lower-level primitives, see 'validateUtf8Chunk' and 'validateUtf8More'.
src/Data/Text/Encoding/Error.hs view
@@ -1,120 +1,119 @@-{-# LANGUAGE CPP, DeriveDataTypeable #-}-{-# LANGUAGE Safe #-}--- |--- Module : Data.Text.Encoding.Error--- Copyright : (c) Bryan O'Sullivan 2009------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Types and functions for dealing with encoding and decoding errors--- in Unicode text.------ The standard functions for encoding and decoding text are strict,--- which is to say that they throw exceptions on invalid input. This--- is often unhelpful on real world input, so alternative functions--- exist that accept custom handlers for dealing with invalid inputs.--- These 'OnError' handlers are normal Haskell functions. You can use--- one of the presupplied functions in this module, or you can write a--- custom handler of your own.--module Data.Text.Encoding.Error- (- -- * Error handling types- UnicodeException(..)- , OnError- , OnDecodeError- , OnEncodeError- -- * Useful error handling functions- , lenientDecode- , strictDecode- , strictEncode- , ignore- , replace- ) where--import Control.DeepSeq (NFData (..))-import Control.Exception (Exception, throw)-import Data.Typeable (Typeable)-import Data.Word (Word8)-import Numeric (showHex)---- | Function type for handling a coding error. It is supplied with--- two inputs:------ * A 'String' that describes the error.------ * The input value that caused the error. If the error arose--- because the end of input was reached or could not be identified--- precisely, this value will be 'Nothing'.------ If the handler returns a value wrapped with 'Just', that value will--- be used in the output as the replacement for the invalid input. If--- it returns 'Nothing', no value will be used in the output.------ Should the handler need to abort processing, it should use 'error'--- or 'throw' an exception (preferably a 'UnicodeException'). It may--- use the description provided to construct a more helpful error--- report.-type OnError a b = String -> Maybe a -> Maybe b---- | A handler for a decoding error.-type OnDecodeError = OnError Word8 Char---- | A handler for an encoding error.-{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-}-type OnEncodeError = OnError Char Word8---- | An exception type for representing Unicode encoding errors.-data UnicodeException =- DecodeError String (Maybe Word8)- -- ^ Could not decode a byte sequence because it was invalid under- -- the given encoding, or ran out of input in mid-decode.- | EncodeError String (Maybe Char)- -- ^ Tried to encode a character that could not be represented- -- under the given encoding, or ran out of input in mid-encode.- deriving (Eq, Typeable)--{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-}--showUnicodeException :: UnicodeException -> String-showUnicodeException (DecodeError desc (Just w))- = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)-showUnicodeException (DecodeError desc Nothing)- = "Cannot decode input: " ++ desc-showUnicodeException (EncodeError desc (Just c))- = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc)-showUnicodeException (EncodeError desc Nothing)- = "Cannot encode input: " ++ desc--instance Show UnicodeException where- show = showUnicodeException--instance Exception UnicodeException--instance NFData UnicodeException where- rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` ()- rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` ()---- | Throw a 'UnicodeException' if decoding fails.-strictDecode :: OnDecodeError-strictDecode desc c = throw (DecodeError desc c)---- | Replace an invalid input byte with the Unicode replacement--- character U+FFFD.-lenientDecode :: OnDecodeError-lenientDecode _ _ = Just '\xfffd'---- | Throw a 'UnicodeException' if encoding fails.-{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-}-strictEncode :: OnEncodeError-strictEncode desc c = throw (EncodeError desc c)---- | Ignore an invalid input, substituting nothing in the output.-ignore :: OnError a b-ignore _ _ = Nothing---- | Replace an invalid input with a valid output.-replace :: b -> OnError a b-replace c _ _ = Just c+{-# LANGUAGE CPP #-} +{-# LANGUAGE Safe #-} +-- | +-- Module : Data.Text.Encoding.Error +-- Copyright : (c) Bryan O'Sullivan 2009 +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Types and functions for dealing with encoding and decoding errors +-- in Unicode text. +-- +-- The standard functions for encoding and decoding text are strict, +-- which is to say that they throw exceptions on invalid input. This +-- is often unhelpful on real world input, so alternative functions +-- exist that accept custom handlers for dealing with invalid inputs. +-- These 'OnError' handlers are normal Haskell functions. You can use +-- one of the presupplied functions in this module, or you can write a +-- custom handler of your own. + +module Data.Text.Encoding.Error + ( + -- * Error handling types + UnicodeException(..) + , OnError + , OnDecodeError + , OnEncodeError + -- * Useful error handling functions + , lenientDecode + , strictDecode + , strictEncode + , ignore + , replace + ) where + +import Control.DeepSeq (NFData (..)) +import Control.Exception (Exception, throw) +import Data.Word (Word8) +import Numeric (showHex) + +-- | Function type for handling a coding error. It is supplied with +-- two inputs: +-- +-- * A 'String' that describes the error. +-- +-- * The input value that caused the error. If the error arose +-- because the end of input was reached or could not be identified +-- precisely, this value will be 'Nothing'. +-- +-- If the handler returns a value wrapped with 'Just', that value will +-- be used in the output as the replacement for the invalid input. If +-- it returns 'Nothing', no value will be used in the output. +-- +-- Should the handler need to abort processing, it should use 'error' +-- or 'throw' an exception (preferably a 'UnicodeException'). It may +-- use the description provided to construct a more helpful error +-- report. +type OnError a b = String -> Maybe a -> Maybe b + +-- | A handler for a decoding error. +type OnDecodeError = OnError Word8 Char + +-- | A handler for an encoding error. +{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-} +type OnEncodeError = OnError Char Word8 + +-- | An exception type for representing Unicode encoding errors. +data UnicodeException = + DecodeError String (Maybe Word8) + -- ^ Could not decode a byte sequence because it was invalid under + -- the given encoding, or ran out of input in mid-decode. + | EncodeError String (Maybe Char) + -- ^ Tried to encode a character that could not be represented + -- under the given encoding, or ran out of input in mid-encode. + deriving (Eq) + +{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-} + +showUnicodeException :: UnicodeException -> String +showUnicodeException (DecodeError desc (Just w)) + = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc) +showUnicodeException (DecodeError desc Nothing) + = "Cannot decode input: " ++ desc +showUnicodeException (EncodeError desc (Just c)) + = "Cannot encode character '\\x" ++ showHex (fromEnum c) ("': " ++ desc) +showUnicodeException (EncodeError desc Nothing) + = "Cannot encode input: " ++ desc + +instance Show UnicodeException where + show = showUnicodeException + +instance Exception UnicodeException + +instance NFData UnicodeException where + rnf (DecodeError desc w) = rnf desc `seq` rnf w `seq` () + rnf (EncodeError desc c) = rnf desc `seq` rnf c `seq` () + +-- | Throw a 'UnicodeException' if decoding fails. +strictDecode :: OnDecodeError +strictDecode desc c = throw (DecodeError desc c) + +-- | Replace an invalid input byte with the Unicode replacement +-- character U+FFFD. +lenientDecode :: OnDecodeError +lenientDecode _ _ = Just '\xfffd' + +-- | Throw a 'UnicodeException' if encoding fails. +{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-} +strictEncode :: OnEncodeError +strictEncode desc c = throw (EncodeError desc c) + +-- | Ignore an invalid input, substituting nothing in the output. +ignore :: OnError a b +ignore _ _ = Nothing + +-- | Replace an invalid input with a valid output. +replace :: b -> OnError a b +replace c _ _ = Just c
src/Data/Text/Foreign.hs view
@@ -1,206 +1,212 @@-{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-}--- |--- Module : Data.Text.Foreign--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Support for using 'Text' data with native code via the Haskell--- foreign function interface.--module Data.Text.Foreign- (- -- * Interoperability with native code- -- $interop- I8- -- * Safe conversion functions- , fromPtr- , fromPtr0- , useAsPtr- , asForeignPtr- -- ** Encoding as UTF-8- , peekCString- , withCString- , peekCStringLen- , withCStringLen- -- * Unsafe conversion code- , lengthWord8- , unsafeCopyToPtr- -- * Low-level manipulation- -- $lowlevel- , dropWord8- , takeWord8- ) where--import Control.Monad.ST.Unsafe (unsafeSTToIO)-import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen)-import Data.Text.Encoding (decodeUtf8, encodeUtf8)-import Data.Text.Internal (Text(..), empty)-import Data.Text.Internal.Unsafe (unsafeWithForeignPtr)-import Data.Text.Show (addrLen)-import Data.Text.Unsafe (lengthWord8)-import Data.Word (Word8)-import Foreign.C.String (CString, CStringLen)-import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray)-import Foreign.Marshal.Alloc (allocaBytes)-import Foreign.Ptr (Ptr, castPtr)-import Foreign.Storable (pokeByteOff)-import GHC.Exts (Ptr(..))-import qualified Data.Text.Array as A---- $interop------ The 'Text' type is implemented using arrays that are not guaranteed--- to have a fixed address in the Haskell heap. All communication with--- native code must thus occur by copying data back and forth.------ The 'Text' type's internal representation is UTF-8.--- To interoperate with native libraries that use different--- internal representations, such as UTF-16 or UTF-32, consider using--- the functions in the 'Data.Text.Encoding' module.---- | A type representing a number of UTF-8 code units.------ @since 2.0-newtype I8 = I8 Int- deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show)---- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the--- contents of the array.-fromPtr :: Ptr Word8 -- ^ source array- -> I8 -- ^ length of source array (in 'Word8' units)- -> IO Text-fromPtr _ (I8 0) = pure empty-fromPtr ptr (I8 len) = unsafeSTToIO $ do- dst <- A.new len- A.copyFromPointer dst 0 ptr len- arr <- A.unsafeFreeze dst- return $! Text arr 0 len---- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the--- contents of the NUL-terminated array.------ @since 2.0.1-fromPtr0 :: Ptr Word8 -- ^ source array- -> IO Text-fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#))---- $lowlevel------ Foreign functions that use UTF-8 internally may return indices in--- units of 'Word8' instead of characters. These functions may--- safely be used with such indices, as they will adjust offsets if--- necessary to preserve the validity of a Unicode string.---- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in--- length.------ If @n@ would cause the 'Text' to end inside a code point, the--- end of the prefix will be advanced by several additional 'Word8' units--- to maintain its validity.------ @since 2.0-takeWord8 :: I8 -> Text -> Text-takeWord8 = (fst .) . splitAtWord8---- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units--- dropped from its beginning.------ If @n@ would cause the 'Text' to begin inside a code point, the--- beginning of the suffix will be advanced by several additional 'Word8'--- unit to maintain its validity.------ @since 2.0-dropWord8 :: I8 -> Text -> Text-dropWord8 = (snd .) . splitAtWord8--splitAtWord8 :: I8 -> Text -> (Text, Text)-splitAtWord8 (I8 n) t@(Text arr off len)- | n <= 0 = (empty, t)- | n >= len || m >= len = (t, empty)- | otherwise = (Text arr off m, Text arr (off+m) (len-m))- where- m | w0 < 0x80 = n -- last char is ASCII- | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence- | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence- | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence- | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence- | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence- | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence- | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence- | otherwise = n -- pre-pre-last char starts 3-byte sequence- w0 = A.unsafeIndex arr (off+n-1)- w1 = A.unsafeIndex arr (off+n-2)- w2 = A.unsafeIndex arr (off+n-3)---- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big--- enough to hold the contents of the entire 'Text'.-unsafeCopyToPtr :: Text -> Ptr Word8 -> IO ()-unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len---- | /O(n)/ Perform an action on a temporary, mutable copy of a--- 'Text'. The copy is freed as soon as the action returns.-useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a-useAsPtr t@(Text _arr _off len) action =- allocaBytes len $ \buf -> do- unsafeCopyToPtr t buf- action (castPtr buf) (I8 len)---- | /O(n)/ Make a mutable copy of a 'Text'.-asForeignPtr :: Text -> IO (ForeignPtr Word8, I8)-asForeignPtr t@(Text _arr _off len) = do- fp <- mallocForeignPtrArray len- unsafeWithForeignPtr fp $ unsafeCopyToPtr t- return (fp, I8 len)---- | Marshal a 'Text' into a C string with a trailing NUL byte,--- encoded as UTF-8 in temporary storage.------ The 'Text' itself must not contain any NUL bytes, this precondition--- is not checked. Cf. 'withCStringLen'.------ The temporary storage is freed when the subcomputation terminates--- (either normally or via an exception), so the pointer to the--- temporary storage must /not/ be used after this function returns.------ @since 2.0.1-withCString :: Text -> (CString -> IO a) -> IO a-withCString t@(Text _arr _off len) action =- allocaBytes (len + 1) $ \buf -> do- unsafeCopyToPtr t buf- pokeByteOff buf len (0 :: Word8)- action (castPtr buf)---- | /O(n)/ Decode a C string with explicit length, which is assumed--- to have been encoded as UTF-8. If decoding fails, a--- 'UnicodeException' is thrown.------ @since 1.0.0.0-peekCStringLen :: CStringLen -> IO Text-peekCStringLen cs = do- bs <- unsafePackCStringLen cs- return $! decodeUtf8 bs---- | /O(n)/ Decode a null-terminated C string, which is assumed--- to have been encoded as UTF-8. If decoding fails, a--- 'UnicodeException' is thrown.------ @since 2.1.2-peekCString :: CString -> IO Text-peekCString cs = do- bs <- unsafePackCString cs- return $! decodeUtf8 bs---- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary--- storage, with explicit length information. The encoded string may--- contain NUL bytes, and is not followed by a trailing NUL byte.------ The temporary storage is freed when the subcomputation terminates--- (either normally or via an exception), so the pointer to the--- temporary storage must /not/ be used after this function returns.------ @since 1.0.0.0-withCStringLen :: Text -> (CStringLen -> IO a) -> IO a-withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act+{-# LANGUAGE CPP, GeneralizedNewtypeDeriving, MagicHash #-} +-- | +-- Module : Data.Text.Foreign +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Support for using 'Text' data with native code via the Haskell +-- foreign function interface. + +module Data.Text.Foreign + ( + -- * Interoperability with native code + -- $interop + I8 + -- * Pointer conversion functions + , fromPtr + , fromPtr0 + , useAsPtr + , asForeignPtr + -- ** Encoding as UTF-8 + , peekCString + , withCString + , peekCStringLen + , withCStringLen + -- * Low-level manipulation + -- $lowlevel + , dropWord8 + , takeWord8 + , lengthWord8 + , unsafeCopyToPtr + ) where + +import Control.Monad.ST.Unsafe (unsafeSTToIO) +import Data.ByteString.Unsafe (unsafePackCStringLen, unsafePackCString, unsafeUseAsCStringLen) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Data.Text.Internal (Text(..), empty) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Show (addrLen) +import Data.Text.Unsafe (lengthWord8) +import Data.Word (Word8) +import Foreign.C.String (CString, CStringLen) +import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) +import Foreign.Marshal.Alloc (allocaBytes) +import Foreign.Ptr (Ptr, castPtr) +import Foreign.Storable (pokeByteOff) +import GHC.Exts (Ptr(..)) +import qualified Data.Text.Array as A + +-- $interop +-- +-- The 'Text' type is implemented using arrays that are not guaranteed +-- to have a fixed address in the Haskell heap. All communication with +-- native code must thus occur by copying data back and forth. +-- +-- The 'Text' type's internal representation is UTF-8. +-- To interoperate with native libraries that use different +-- internal representations, such as UTF-16 or UTF-32, consider using +-- the functions in the 'Data.Text.Encoding' module. + +-- | A type representing a number of UTF-8 code units. +-- +-- @since 2.0 +newtype I8 = I8 Int + deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the array. +-- +-- __This function is unsafe.__ The source array must contain a valid +-- UTF-8 string of the given length. There are no guarantees about what +-- happens otherwise. +fromPtr :: Ptr Word8 -- ^ source array + -> I8 -- ^ length of source array (in 'Word8' units) + -> IO Text +fromPtr _ (I8 0) = pure empty +fromPtr ptr (I8 len) = unsafeSTToIO $ do + dst <- A.new len + A.copyFromPointer dst 0 ptr len + arr <- A.unsafeFreeze dst + return $! Text arr 0 len + +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the +-- contents of the NUL-terminated array. +-- +-- __This function is unsafe.__ The source array must contain a NULL-terminated +-- valid UTF-8 string. There are no guarantees about what happens otherwise. +-- +-- @since 2.0.1 +fromPtr0 :: Ptr Word8 -- ^ source array + -> IO Text +fromPtr0 ptr@(Ptr addr#) = fromPtr ptr (fromIntegral (addrLen addr#)) + +-- $lowlevel +-- +-- Foreign functions that use UTF-8 internally may return indices in +-- units of 'Word8' instead of characters. These functions may +-- safely be used with such indices, as they will adjust offsets if +-- necessary to preserve the validity of a Unicode string. + +-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in +-- length. +-- +-- If @n@ would cause the 'Text' to end inside a code point, the +-- end of the prefix will be advanced by several additional 'Word8' units +-- to maintain its validity. +-- +-- @since 2.0 +takeWord8 :: I8 -> Text -> Text +takeWord8 = (fst .) . splitAtWord8 + +-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units +-- dropped from its beginning. +-- +-- If @n@ would cause the 'Text' to begin inside a code point, the +-- beginning of the suffix will be advanced by several additional 'Word8' +-- unit to maintain its validity. +-- +-- @since 2.0 +dropWord8 :: I8 -> Text -> Text +dropWord8 = (snd .) . splitAtWord8 + +splitAtWord8 :: I8 -> Text -> (Text, Text) +splitAtWord8 (I8 n) t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) + where + m | w0 < 0x80 = n -- last char is ASCII + | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence + | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence + | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence + | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence + | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence + | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence + | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence + | otherwise = n -- pre-pre-last char starts 3-byte sequence + w0 = A.unsafeIndex arr (off+n-1) + w1 = A.unsafeIndex arr (off+n-2) + w2 = A.unsafeIndex arr (off+n-3) + +-- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big +-- enough to hold the contents of the entire 'Text'. +unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () +unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len + +-- | /O(n)/ Perform an action on a temporary, mutable copy of a +-- 'Text'. The copy is freed as soon as the action returns. +useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a +useAsPtr t@(Text _arr _off len) action = + allocaBytes len $ \buf -> do + unsafeCopyToPtr t buf + action (castPtr buf) (I8 len) + +-- | /O(n)/ Make a mutable copy of a 'Text'. +asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) +asForeignPtr t@(Text _arr _off len) = do + fp <- mallocForeignPtrArray len + unsafeWithForeignPtr fp $ unsafeCopyToPtr t + return (fp, I8 len) + +-- | Marshal a 'Text' into a C string with a trailing NUL byte, +-- encoded as UTF-8 in temporary storage. +-- +-- The 'Text' itself must not contain any NUL bytes, this precondition +-- is not checked. Cf. 'withCStringLen'. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 2.0.1 +withCString :: Text -> (CString -> IO a) -> IO a +withCString t@(Text _arr _off len) action = + allocaBytes (len + 1) $ \buf -> do + unsafeCopyToPtr t buf + pokeByteOff buf len (0 :: Word8) + action (castPtr buf) + +-- | /O(n)/ Decode a C string with explicit length, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 1.0.0.0 +peekCStringLen :: CStringLen -> IO Text +peekCStringLen cs = do + bs <- unsafePackCStringLen cs + return $! decodeUtf8 bs + +-- | /O(n)/ Decode a null-terminated C string, which is assumed +-- to have been encoded as UTF-8. If decoding fails, a +-- 'UnicodeException' is thrown. +-- +-- @since 2.1.2 +peekCString :: CString -> IO Text +peekCString cs = do + bs <- unsafePackCString cs + return $! decodeUtf8 bs + +-- | Marshal a 'Text' into a C string encoded as UTF-8 in temporary +-- storage, with explicit length information. The encoded string may +-- contain NUL bytes, and is not followed by a trailing NUL byte. +-- +-- The temporary storage is freed when the subcomputation terminates +-- (either normally or via an exception), so the pointer to the +-- temporary storage must /not/ be used after this function returns. +-- +-- @since 1.0.0.0 +withCStringLen :: Text -> (CStringLen -> IO a) -> IO a +withCStringLen t act = unsafeUseAsCStringLen (encodeUtf8 t) act
src/Data/Text/IO.hs view
@@ -1,189 +1,211 @@-{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-}-{-# LANGUAGE Trustworthy #-}--- |--- Module : Data.Text.IO--- Copyright : (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Simon Marlow--- License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ Efficient locale-sensitive support for text I\/O.------ The functions in this module obey the runtime system's locale,--- character set encoding, and line ending conversion settings.------ If you want to do I\/O using the UTF-8 encoding, use @Data.Text.IO.Utf8@,--- which is faster than this module.------ If you know in advance that you will be working with data that has--- a specific encoding, and your application is highly--- performance sensitive, you may find that it is faster to perform--- I\/O with bytestrings and to encode and decode yourself than to use--- the functions in this module.--module Data.Text.IO- (- -- * File-at-a-time operations- readFile- , writeFile- , appendFile- -- * Operations on handles- , hGetContents- , hGetChunk- , hGetLine- , hPutStr- , hPutStrLn- -- * Special cases for standard input and output- , interact- , getContents- , getLine- , putStr- , putStrLn- ) where--import Data.Text (Text)-import Prelude hiding (appendFile, getContents, getLine, interact,- putStr, putStrLn, readFile, writeFile)-import System.IO (Handle, IOMode(..), openFile, stdin, stdout,- withFile)-import qualified Control.Exception as E-import Control.Monad (liftM2, when)-import Data.IORef (readIORef)-import qualified Data.Text as T-import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn)-import GHC.IO.Buffer (CharBuffer, isEmptyBuffer)-import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType))-import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle)-import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..))-import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)-import System.IO.Error (isEOFError)---- | The 'readFile' function reads a file and returns the contents of--- the file as a string. The entire file is read strictly, as with--- 'getContents'.------ Beware that this function (similarly to 'Prelude.readFile') is locale-dependent.--- Unexpected system locale may cause your application to read corrupted data or--- throw runtime exceptions about "invalid argument (invalid byte sequence)"--- or "invalid argument (invalid character)". This is also slow, because GHC--- first converts an entire input to UTF-32, which is afterwards converted to UTF-8.------ If your data is UTF-8,--- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile'--- is a much faster and safer alternative.-readFile :: FilePath -> IO Text-readFile name = openFile name ReadMode >>= hGetContents---- | Write a string to a file. The file is truncated to zero length--- before writing begins.-writeFile :: FilePath -> Text -> IO ()-writeFile p = withFile p WriteMode . flip hPutStr---- | Write a string to the end of a file.-appendFile :: FilePath -> Text -> IO ()-appendFile p = withFile p AppendMode . flip hPutStr--catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool)-catchError caller h Handle__{..} err- | isEOFError err = do- buf <- readIORef haCharBuffer- return $ if isEmptyBuffer buf- then (T.empty, True)- else (T.singleton '\r', True)- | otherwise = E.throwIO (augmentIOError err caller h)---- | Wrap readChunk and return a value indicating if we're reached the EOF.--- This is needed because unpack_nl is unable to discern the difference--- between a buffer with just \r due to EOF or because not enough data was left--- for decoding. e.g. the final character decoded from the byte buffer was \r.-readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool)-readChunkEof hh buf = do t <- readChunk hh buf- return (t, False)---- | /Experimental./ Read a single chunk of strict text from a--- 'Handle'. The size of the chunk depends on the amount of input--- currently buffered.------ This function blocks only if there is no data available, and EOF--- has not yet been reached. Once EOF is reached, this function--- returns an empty string instead of throwing an exception.-hGetChunk :: Handle -> IO Text-hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk- where- readSingleChunk hh@Handle__{..} = do- buf <- readIORef haCharBuffer- (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh- return (hh, t)---- | Read the remaining contents of a 'Handle' as a string. The--- 'Handle' is closed once the contents have been read, or if an--- exception is thrown.------ Internally, this function reads a chunk at a time from the--- lower-level buffering abstraction, and concatenates the chunks into--- a single string once the entire file has been read.------ As a result, it requires approximately twice as much memory as its--- result to construct its result. For files more than a half of--- available RAM in size, this may result in memory exhaustion.-hGetContents :: Handle -> IO Text-hGetContents h = do- chooseGoodBuffering h- wantReadableHandle "hGetContents" h readAll- where- readAll hh@Handle__{..} = do- let readChunks = do- buf <- readIORef haCharBuffer- (t, eof) <- readChunkEof hh buf- `E.catch` catchError "hGetContents" h hh- if eof- then return [t]- else (t:) `fmap` readChunks- ts <- readChunks- (hh', _) <- hClose_help hh- return (hh'{haType=ClosedHandle}, T.concat ts)---- | Use a more efficient buffer size if we're reading in--- block-buffered mode with the default buffer size. When we can--- determine the size of the handle we're reading, set the buffer size--- to that, so that we can read the entire file in one chunk.--- Otherwise, use a buffer size of at least 16KB.-chooseGoodBuffering :: Handle -> IO ()-chooseGoodBuffering h = do- bufMode <- hGetBuffering h- case bufMode of- BlockBuffering Nothing -> do- d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) ->- if ioe_type e == InappropriateType- then return 16384 -- faster than the 2KB default- else E.throwIO e- when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d- _ -> return ()---- | Read a single line from a handle.-hGetLine :: Handle -> IO Text-hGetLine = hGetLineWith T.concat---- | The 'interact' function takes a function of type @Text -> Text@--- as its argument. The entire input from the standard input device is--- passed to this function as its argument, and the resulting string--- is output on the standard output device.-interact :: (Text -> Text) -> IO ()-interact f = putStr . f =<< getContents---- | Read all user input on 'stdin' as a single string.-getContents :: IO Text-getContents = hGetContents stdin---- | Read a single line of user input from 'stdin'.-getLine :: IO Text-getLine = hGetLine stdin---- | Write a string to 'stdout'.-putStr :: Text -> IO ()-putStr = hPutStr stdout---- | Write a string to 'stdout', followed by a newline.-putStrLn :: Text -> IO ()-putStrLn = hPutStrLn stdout+{-# LANGUAGE BangPatterns, CPP, RecordWildCards, ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +-- | +-- Module : Data.Text.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- Efficient locale-sensitive support for text I\/O. +-- +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you want to do I\/O using the UTF-8 encoding, use "Data.Text.IO.Utf8", +-- which is faster than this module. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding, and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. + +module Data.Text.IO + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetChunk + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Data.Text (Text) +import Prelude hiding (appendFile, getContents, getLine, interact, + putStr, putStrLn, readFile, writeFile) +import System.IO (Handle, IOMode(..), openFile, stdin, stdout, + withFile) +import qualified Control.Exception as E +import Control.Monad (liftM2, when) +import Data.IORef (readIORef) +import qualified Data.Text as T +import Data.Text.Internal.IO (hGetLineWith, readChunk, hPutStr, hPutStrLn) +import GHC.IO.Buffer (CharBuffer, isEmptyBuffer) +import GHC.IO.Exception (IOException(ioe_type), IOErrorType(InappropriateType)) +import GHC.IO.Handle.Internals (augmentIOError, hClose_help, wantReadableHandle) +import GHC.IO.Handle.Types (BufferMode(..), Handle__(..), HandleType(..)) +import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) +import System.IO.Error (isEOFError) + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' +-- is a much faster and safer alternative. +readFile :: FilePath -> IO Text +readFile name = openFile name ReadMode >>= hGetContents + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile p = withFile p WriteMode . flip hPutStr + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile p = withFile p AppendMode . flip hPutStr + +catchError :: String -> Handle -> Handle__ -> IOError -> IO (Text, Bool) +catchError caller h Handle__{..} err + | isEOFError err = do + buf <- readIORef haCharBuffer + return $ if isEmptyBuffer buf + then (T.empty, True) + else (T.singleton '\r', True) + | otherwise = E.throwIO (augmentIOError err caller h) + +-- | Wrap readChunk and return a value indicating if we're reached the EOF. +-- This is needed because unpack_nl is unable to discern the difference +-- between a buffer with just \r due to EOF or because not enough data was left +-- for decoding. e.g. the final character decoded from the byte buffer was \r. +readChunkEof :: Handle__ -> CharBuffer -> IO (Text, Bool) +readChunkEof hh buf = do t <- readChunk hh buf + return (t, False) + +-- | Read a single chunk of strict text from a +-- 'Handle'. The size of the chunk depends on the amount of input +-- currently buffered. +-- +-- This function blocks only if there is no data available, and EOF +-- has not yet been reached. Once EOF is reached, this function +-- returns an empty string instead of throwing an exception. +-- +-- === Behavior +-- +-- Unlike byte-oriented functions, 'hGetChunk' operates on complete UTF-8 +-- characters. Since UTF-8 characters can occupy 1 to 4 bytes, this function +-- cannot guarantee reading an exact number of bytes. Instead, it reads +-- complete characters up to the handle's internal buffer limit. +-- +-- === Buffer Size +-- +-- The maximum chunk size is determined by the handle's internal character +-- buffer, which is set to 8192 bytes (2048 characters) by the GHC runtime +-- constant @dEFAULT_CHAR_BUFFER_SIZE@. This buffer size cannot be modified +-- through any public API. +-- +-- === UTF-8 Considerations +-- +-- When working with UTF-8 encoded text: +-- +-- * The function will never return a partial character +-- * The actual number of bytes read may vary depending on the character +-- encoding (ASCII characters = 1 byte, other Unicode characters = 2-4 bytes) +hGetChunk :: Handle -> IO Text +hGetChunk h = wantReadableHandle "hGetChunk" h readSingleChunk + where + readSingleChunk hh@Handle__{..} = do + buf <- readIORef haCharBuffer + (t, _) <- readChunkEof hh buf `E.catch` catchError "hGetChunk" h hh + return (hh, t) + +-- | Read the remaining contents of a 'Handle' as a string. The +-- 'Handle' is closed once the contents have been read, or if an +-- exception is thrown. +-- +-- Internally, this function reads a chunk at a time from the +-- lower-level buffering abstraction, and concatenates the chunks into +-- a single string once the entire file has been read. +-- +-- As a result, it requires approximately twice as much memory as its +-- result to construct its result. For files more than a half of +-- available RAM in size, this may result in memory exhaustion. +hGetContents :: Handle -> IO Text +hGetContents h = do + chooseGoodBuffering h + wantReadableHandle "hGetContents" h readAll + where + readAll hh@Handle__{..} = do + let readChunks = do + buf <- readIORef haCharBuffer + (t, eof) <- readChunkEof hh buf + `E.catch` catchError "hGetContents" h hh + if eof + then return [t] + else (t:) `fmap` readChunks + ts <- readChunks + (hh', _) <- hClose_help hh + return (hh'{haType=ClosedHandle}, T.concat ts) + +-- | Use a more efficient buffer size if we're reading in +-- block-buffered mode with the default buffer size. When we can +-- determine the size of the handle we're reading, set the buffer size +-- to that, so that we can read the entire file in one chunk. +-- Otherwise, use a buffer size of at least 16KB. +chooseGoodBuffering :: Handle -> IO () +chooseGoodBuffering h = do + bufMode <- hGetBuffering h + case bufMode of + BlockBuffering Nothing -> do + d <- E.catch (liftM2 (-) (hFileSize h) (hTell h)) $ \(e::IOException) -> + if ioe_type e == InappropriateType + then return 16384 -- faster than the 2KB default + else E.throwIO e + when (d > 0) . hSetBuffering h . BlockBuffering . Just . fromInteger $ d + _ -> return () + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = hGetLineWith T.concat + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = hGetContents stdin + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = hGetLine stdin + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = hPutStr stdout + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = hPutStrLn stdout
src/Data/Text/IO/Utf8.hs view
@@ -1,94 +1,94 @@--- |--- Module : Data.Text.IO.Utf8--- License : BSD-style--- Portability : GHC------ Efficient UTF-8 support for text I\/O.--- Unlike @Data.Text.IO@, these functions do not depend on the locale--- and do not do line ending conversion.-module Data.Text.IO.Utf8- (- -- * File-at-a-time operations- readFile- , writeFile- , appendFile- -- * Operations on handles- , hGetContents- , hGetLine- , hPutStr- , hPutStrLn- -- * Special cases for standard input and output- , interact- , getContents- , getLine- , putStr- , putStrLn- ) where--import Prelude ()-import Control.Exception (evaluate)-import Control.Monad ((<=<), (=<<))-import Data.ByteString (ByteString)-import qualified Data.ByteString.Char8 as B-import Data.Function ((.))-import Data.Text (Text)-import Data.Text.Encoding (decodeUtf8, encodeUtf8)-import GHC.IO.Handle (Handle)-import System.IO (IO, FilePath)--decodeUtf8IO :: ByteString -> IO Text-decodeUtf8IO = evaluate . decodeUtf8---- | The 'readFile' function reads a file and returns the contents of--- the file as a string. The entire file is read strictly, as with--- 'getContents'.-readFile :: FilePath -> IO Text-readFile = decodeUtf8IO <=< B.readFile---- | Write a string to a file. The file is truncated to zero length--- before writing begins.-writeFile :: FilePath -> Text -> IO ()-writeFile fp = B.writeFile fp . encodeUtf8---- | Write a string to the end of a file.-appendFile :: FilePath -> Text -> IO ()-appendFile fp = B.appendFile fp . encodeUtf8---- | Read the remaining contents of a 'Handle' as a string.-hGetContents :: Handle -> IO Text-hGetContents = decodeUtf8IO <=< B.hGetContents---- | Read a single line from a handle.-hGetLine :: Handle -> IO Text-hGetLine = decodeUtf8IO <=< B.hGetLine---- | Write a string to a handle.-hPutStr :: Handle -> Text -> IO ()-hPutStr h = B.hPutStr h . encodeUtf8---- | Write a string to a handle, followed by a newline.-hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h = B.hPutStrLn h . encodeUtf8---- | The 'interact' function takes a function of type @Text -> Text@--- as its argument. The entire input from the standard input device is--- passed to this function as its argument, and the resulting string--- is output on the standard output device.-interact :: (Text -> Text) -> IO ()-interact f = putStr . f =<< getContents---- | Read all user input on 'stdin' as a single string.-getContents :: IO Text-getContents = decodeUtf8IO =<< B.getContents---- | Read a single line of user input from 'stdin'.-getLine :: IO Text-getLine = decodeUtf8IO =<< B.getLine---- | Write a string to 'stdout'.-putStr :: Text -> IO ()-putStr = B.putStr . encodeUtf8---- | Write a string to 'stdout', followed by a newline.-putStrLn :: Text -> IO ()-putStrLn = B.putStrLn . encodeUtf8+-- | +-- Module : Data.Text.IO.Utf8 +-- License : BSD-style +-- Portability : GHC +-- +-- Efficient UTF-8 support for text I\/O. +-- Unlike "Data.Text.IO", these functions do not depend on the locale +-- and do not do line ending conversion. +module Data.Text.IO.Utf8 + ( + -- * File-at-a-time operations + readFile + , writeFile + , appendFile + -- * Operations on handles + , hGetContents + , hGetLine + , hPutStr + , hPutStrLn + -- * Special cases for standard input and output + , interact + , getContents + , getLine + , putStr + , putStrLn + ) where + +import Prelude () +import Control.Exception (evaluate) +import Control.Monad ((<=<), (=<<)) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B +import Data.Function ((.)) +import Data.Text (Text) +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import GHC.IO.Handle (Handle) +import System.IO (IO, FilePath) + +decodeUtf8IO :: ByteString -> IO Text +decodeUtf8IO = evaluate . decodeUtf8 + +-- | The 'readFile' function reads a file and returns the contents of +-- the file as a string. The entire file is read strictly, as with +-- 'getContents'. +readFile :: FilePath -> IO Text +readFile = decodeUtf8IO <=< B.readFile + +-- | Write a string to a file. The file is truncated to zero length +-- before writing begins. +writeFile :: FilePath -> Text -> IO () +writeFile fp = B.writeFile fp . encodeUtf8 + +-- | Write a string to the end of a file. +appendFile :: FilePath -> Text -> IO () +appendFile fp = B.appendFile fp . encodeUtf8 + +-- | Read the remaining contents of a 'Handle' as a string. +hGetContents :: Handle -> IO Text +hGetContents = decodeUtf8IO <=< B.hGetContents + +-- | Read a single line from a handle. +hGetLine :: Handle -> IO Text +hGetLine = decodeUtf8IO <=< B.hGetLine + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h = B.hPutStr h . encodeUtf8 + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h = B.hPutStrLn h . encodeUtf8 + +-- | The 'interact' function takes a function of type @Text -> Text@ +-- as its argument. The entire input from the standard input device is +-- passed to this function as its argument, and the resulting string +-- is output on the standard output device. +interact :: (Text -> Text) -> IO () +interact f = putStr . f =<< getContents + +-- | Read all user input on 'stdin' as a single string. +getContents :: IO Text +getContents = decodeUtf8IO =<< B.getContents + +-- | Read a single line of user input from 'stdin'. +getLine :: IO Text +getLine = decodeUtf8IO =<< B.getLine + +-- | Write a string to 'stdout'. +putStr :: Text -> IO () +putStr = B.putStr . encodeUtf8 + +-- | Write a string to 'stdout', followed by a newline. +putStrLn :: Text -> IO () +putStrLn = B.putStrLn . encodeUtf8
src/Data/Text/Internal.hs view
@@ -1,275 +1,274 @@-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE UnboxedTuples #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module : Data.Text.Internal--- Copyright : (c) 2008, 2009 Tom Harper,--- (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.------ You should not use this module unless you are determined to monkey--- with the internals, as the functions here do just about nothing to--- preserve data invariants. You have been warned!--module Data.Text.Internal- (- -- * Types- -- $internals- Text(..)- , StrictText- -- * Construction- , text- , textP- -- * Safety- , safe- -- * Code that must be here for accessibility- , empty- , append- -- * Utilities- , firstf- -- * Checked multiplication- , mul- , mul32- , mul64- -- * Debugging- , showText- -- * Conversions- , pack- ) where--#if defined(ASSERTS)-import Control.Exception (assert)-import GHC.Stack (HasCallStack)-#endif-import Control.Monad.ST (ST, runST)-import Data.Bits-import Data.Int (Int32, Int64)-import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)-import Data.Typeable (Typeable)-import qualified Data.Text.Array as A---- | A space efficient, packed, unboxed Unicode text type.-data Text = Text- {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8- {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence- deriving (Typeable)---- | Type synonym for the strict flavour of 'Text'.-type StrictText = Text---- | Smart constructor.-text_ ::-#if defined(ASSERTS)- HasCallStack =>-#endif- A.Array -- ^ bytearray encoded as UTF-8- -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence- -> Text-text_ arr off len =-#if defined(ASSERTS)- let c = A.unsafeIndex arr off- in assert (len >= 0) .- assert (off >= 0) .- assert (len == 0 || c < 0x80 || c >= 0xC0) $-#endif- Text arr off len-{-# INLINE text_ #-}---- | /O(1)/ The empty 'Text'.-empty :: Text-empty = Text A.empty 0 0-{-# NOINLINE empty #-}---- | /O(n)/ Appends one 'Text' to the other by copying both of them--- into a new 'Text'.-append :: Text -> Text -> Text-append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2)- | len1 == 0 = b- | len2 == 0 = a- | len > 0 = Text (A.run x) 0 len- | otherwise = error $ "Data.Text.append: size overflow"- where- len = len1+len2- x :: ST s (A.MArray s)- x = do- arr <- A.new len- A.copyI len1 arr 0 arr1 off1- A.copyI len2 arr len1 arr2 off2- return arr-{-# NOINLINE append #-}---- | Construct a 'Text' without invisibly pinning its byte array in--- memory if its length has dwindled to zero.--- It ensures that empty 'Text' values are shared.-text ::-#if defined(ASSERTS)- HasCallStack =>-#endif- A.Array -- ^ bytearray encoded as UTF-8- -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence- -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence- -> Text-text arr off len | len == 0 = empty- | otherwise = text_ arr off len-{-# INLINE [0] text #-}--textP :: A.Array -> Int -> Int -> Text-{-# DEPRECATED textP "Use text instead" #-}-textP = text---- | A useful 'show'-like function for debugging purposes.-showText :: Text -> String-showText (Text arr off len) =- "Text " ++ show (A.toList arr off len) ++ ' ' :- show off ++ ' ' : show len---- | Map a 'Char' to a 'Text'-safe value.------ Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode--- scalar values, but are unfortunately admitted as valid 'Char'--- values by Haskell. They cannot be represented in a 'Text'. This--- function remaps those code points to the Unicode replacement--- character (U+FFFD, \'�\'), and leaves other code points--- unchanged.-safe :: Char -> Char-safe c- | ord c .&. 0x1ff800 /= 0xd800 = c- | otherwise = '\xfffd'-{-# INLINE [0] safe #-}---- | Apply a function to the first element of an optional pair.-firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)-firstf f (Just (a, b)) = Just (f a, b)-firstf _ Nothing = Nothing---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul :: Int -> Int -> Int-mul a b- | finiteBitSize (0 :: Word) == 64- = int64ToInt $ intToInt64 a `mul64` intToInt64 b- | otherwise- = int32ToInt $ intToInt32 a `mul32` intToInt32 b-{-# INLINE mul #-}-infixl 7 `mul`---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul64 :: Int64 -> Int64 -> Int64-mul64 a b- | a >= 0 && b >= 0 = mul64_ a b- | a >= 0 = -mul64_ a (-b)- | b >= 0 = -mul64_ (-a) b- | otherwise = mul64_ (-a) (-b)-{-# INLINE mul64 #-}-infixl 7 `mul64`--mul64_ :: Int64 -> Int64 -> Int64-mul64_ a b- | ahi > 0 && bhi > 0 = error "overflow"- | top > 0x7fffffff = error "overflow"- | total < 0 = error "overflow"- | otherwise = total- where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #)- (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #)- top = ahi * blo + alo * bhi- total = (top `shiftL` 32) + alo * blo-{-# INLINE mul64_ #-}---- | Checked multiplication. Calls 'error' if the result would--- overflow.-mul32 :: Int32 -> Int32 -> Int32-mul32 a b = case int32ToInt64 a * int32ToInt64 b of- ab | ab < min32 || ab > max32 -> error "overflow"- | otherwise -> int64ToInt32 ab- where min32 = -0x80000000 :: Int64- max32 = 0x7fffffff-{-# INLINE mul32 #-}-infixl 7 `mul32`--intToInt64 :: Int -> Int64-intToInt64 = fromIntegral--int64ToInt :: Int64 -> Int-int64ToInt = fromIntegral--intToInt32 :: Int -> Int32-intToInt32 = fromIntegral--int32ToInt :: Int32 -> Int-int32ToInt = fromIntegral--int32ToInt64 :: Int32 -> Int64-int32ToInt64 = fromIntegral--int64ToInt32 :: Int64 -> Int32-int64ToInt32 = fromIntegral---- $internals------ Internally, the 'Text' type is represented as an array of 'Word8'--- UTF-8 code units. The offset and length fields in the constructor--- are in these units, /not/ units of 'Char'.------ Invariants that all functions must maintain:------ * Since the 'Text' type uses UTF-8 internally, it cannot represent--- characters in the reserved surrogate code point range U+D800 to--- U+DFFF. To maintain this invariant, the 'safe' function maps--- 'Char' values in this range to the replacement character (U+FFFD,--- \'�\').------ * Offset and length must point to a valid UTF-8 sequence of bytes.--- Violation of this may cause memory access violation and divergence.---- -------------------------------------------------------------------------------- * Conversion to/from 'Text'---- | /O(n)/ Convert a 'String' into a 'Text'.--- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id':------ >>> Data.Text.unpack (pack "\55555")--- "\65533"-pack :: String -> Text-pack [] = empty-pack xs = runST $ do- -- It's tempting to allocate a buffer of 4 * length xs bytes,- -- but not only it's wasteful for predominantly ASCII arguments,- -- the computation of length xs would force allocation of the entire xs at once.- let dstLen = 64- dst <- A.new dstLen- outer dst dstLen 0 xs- where- outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text- outer !dst !dstLen = inner- where- inner !dstOff [] = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- inner !dstOff ccs@(c : cs)- -- Each 'Char' takes up to 4 bytes- | dstOff + 4 > dstLen = do- -- Double size of the buffer- let !dstLen' = dstLen * 2- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' dstOff ccs- | otherwise = do- d <- unsafeWrite dst dstOff (safe c)- inner (dstOff + d) cs-{-# NOINLINE [0] pack #-}+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE UnboxedTuples #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. +-- +-- You should not use this module unless you are determined to monkey +-- with the internals, as the functions here do just about nothing to +-- preserve data invariants. You have been warned! + +module Data.Text.Internal + ( + -- * Types + -- $internals + Text(..) + , StrictText + -- * Construction + , text + , textP + -- * Safety + , safe + -- * Code that must be here for accessibility + , empty + , append + -- * Utilities + , firstf + -- * Checked multiplication + , mul + , mul32 + , mul64 + -- * Debugging + , showText + -- * Conversions + , pack + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Control.Monad.ST (ST, runST) +import Data.Bits +import Data.Int (Int32, Int64) +import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) +import qualified Data.Text.Array as A + +-- | A space efficient, packed, unboxed Unicode text type. +data Text = Text + {-# UNPACK #-} !A.Array -- ^ bytearray encoded as UTF-8 + {-# UNPACK #-} !Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + {-# UNPACK #-} !Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + +-- | Type synonym for the strict flavour of 'Text'. +-- +-- @since 2.1.1 +type StrictText = Text + +-- | Smart constructor. +text_ :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text_ arr off len = +#if defined(ASSERTS) + let c = A.unsafeIndex arr off + in assert (len >= 0) . + assert (off >= 0) . + assert (len == 0 || c < 0x80 || c >= 0xC0) $ +#endif + Text arr off len +{-# INLINE text_ #-} + +-- | /O(1)/ The empty 'Text'. +empty :: Text +empty = Text A.empty 0 0 +{-# NOINLINE empty #-} + +-- | /O(n)/ Appends one 'Text' to the other by copying both of them +-- into a new 'Text'. +append :: Text -> Text -> Text +append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) + | len1 == 0 = b + | len2 == 0 = a + | len > 0 = Text (A.run x) 0 len + | otherwise = error $ "Data.Text.append: size overflow" + where + len = len1+len2 + x :: ST s (A.MArray s) + x = do + arr <- A.new len + A.copyI len1 arr 0 arr1 off1 + A.copyI len2 arr len1 arr2 off2 + return arr +{-# NOINLINE append #-} + +-- | Construct a 'Text' without invisibly pinning its byte array in +-- memory if its length has dwindled to zero. +-- It ensures that empty 'Text' values are shared. +text :: +#if defined(ASSERTS) + HasCallStack => +#endif + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text +text arr off len | len == 0 = empty + | otherwise = text_ arr off len +{-# INLINE [0] text #-} + +textP :: A.Array -> Int -> Int -> Text +{-# DEPRECATED textP "Use text instead" #-} +textP = text + +-- | A useful 'show'-like function for debugging purposes. +showText :: Text -> String +showText (Text arr off len) = + "Text " ++ show (A.toList arr off len) ++ ' ' : + show off ++ ' ' : show len + +-- | Map a 'Char' to a 'Text'-safe value. +-- +-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode +-- scalar values, but are unfortunately admitted as valid 'Char' +-- values by Haskell. They cannot be represented in a 'Text'. This +-- function remaps those code points to the Unicode replacement +-- character (U+FFFD, \'�\'), and leaves other code points +-- unchanged. +safe :: Char -> Char +safe c + | ord c .&. 0x1ff800 /= 0xd800 = c + | otherwise = '\xfffd' +{-# INLINE [0] safe #-} + +-- | Apply a function to the first element of an optional pair. +firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b) +firstf f (Just (a, b)) = Just (f a, b) +firstf _ Nothing = Nothing + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul :: Int -> Int -> Int +mul a b + | finiteBitSize (0 :: Word) == 64 + = int64ToInt $ intToInt64 a `mul64` intToInt64 b + | otherwise + = int32ToInt $ intToInt32 a `mul32` intToInt32 b +{-# INLINE mul #-} +infixl 7 `mul` + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul64 :: Int64 -> Int64 -> Int64 +mul64 a b + | a >= 0 && b >= 0 = mul64_ a b + | a >= 0 = -mul64_ a (-b) + | b >= 0 = -mul64_ (-a) b + | otherwise = mul64_ (-a) (-b) +{-# INLINE mul64 #-} +infixl 7 `mul64` + +mul64_ :: Int64 -> Int64 -> Int64 +mul64_ a b + | ahi > 0 && bhi > 0 = error "overflow" + | top > 0x7fffffff = error "overflow" + | total < 0 = error "overflow" + | otherwise = total + where (# ahi, alo #) = (# a `shiftR` 32, a .&. 0xffffffff #) + (# bhi, blo #) = (# b `shiftR` 32, b .&. 0xffffffff #) + top = ahi * blo + alo * bhi + total = (top `shiftL` 32) + alo * blo +{-# INLINE mul64_ #-} + +-- | Checked multiplication. Calls 'error' if the result would +-- overflow. +mul32 :: Int32 -> Int32 -> Int32 +mul32 a b = case int32ToInt64 a * int32ToInt64 b of + ab | ab < min32 || ab > max32 -> error "overflow" + | otherwise -> int64ToInt32 ab + where min32 = -0x80000000 :: Int64 + max32 = 0x7fffffff +{-# INLINE mul32 #-} +infixl 7 `mul32` + +intToInt64 :: Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Int +int64ToInt = fromIntegral + +intToInt32 :: Int -> Int32 +intToInt32 = fromIntegral + +int32ToInt :: Int32 -> Int +int32ToInt = fromIntegral + +int32ToInt64 :: Int32 -> Int64 +int32ToInt64 = fromIntegral + +int64ToInt32 :: Int64 -> Int32 +int64ToInt32 = fromIntegral + +-- $internals +-- +-- Internally, the 'Text' type is represented as an array of 'Word8' +-- UTF-8 code units. The offset and length fields in the constructor +-- are in these units, /not/ units of 'Char'. +-- +-- Invariants that all functions must maintain: +-- +-- * Since the 'Text' type uses UTF-8 internally, it cannot represent +-- characters in the reserved surrogate code point range U+D800 to +-- U+DFFF. To maintain this invariant, the 'safe' function maps +-- 'Char' values in this range to the replacement character (U+FFFD, +-- \'�\'). +-- +-- * Offset and length must point to a valid UTF-8 sequence of bytes. +-- Violation of this may cause memory access violation and divergence. + +-- ----------------------------------------------------------------------------- +-- * Conversion to/from 'Text' + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- Performs replacement on invalid scalar values, so @'Data.Text.unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.unpack (pack "\55555") +-- "\65533" +pack :: String -> Text +pack [] = empty +pack xs = runST $ do + -- It's tempting to allocate a buffer of 4 * length xs bytes, + -- but not only it's wasteful for predominantly ASCII arguments, + -- the computation of length xs would force allocation of the entire xs at once. + let dstLen = 64 + dst <- A.new dstLen + outer dst dstLen 0 xs + where + outer :: forall s. A.MArray s -> Int -> Int -> String -> ST s Text + outer !dst !dstLen = inner + where + inner !dstOff [] = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + inner !dstOff ccs@(c : cs) + -- Each 'Char' takes up to 4 bytes + | dstOff + 4 > dstLen = do + -- Double size of the buffer + let !dstLen' = dstLen * 2 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' dstOff ccs + | otherwise = do + d <- unsafeWrite dst dstOff (safe c) + inner (dstOff + d) cs +{-# NOINLINE [0] pack #-}
src/Data/Text/Internal/Builder.hs view
@@ -1,346 +1,350 @@-{-# LANGUAGE BangPatterns, CPP, RankNTypes #-}-{-# OPTIONS_HADDOCK not-home #-}---------------------------------------------------------------------------------- |--- Module : Data.Text.Internal.Builder--- Copyright : (c) 2013 Bryan O'Sullivan--- (c) 2010 Johan Tibell--- License : BSD-style (see LICENSE)------ Maintainer : Johan Tibell <johan.tibell@gmail.com>--- Stability : experimental--- Portability : portable to Hugs and GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Efficient construction of lazy @Text@ values. The principal--- operations on a @Builder@ are @singleton@, @fromText@, and--- @fromLazyText@, which construct new builders, and 'mappend', which--- concatenates two builders.------ To get maximum performance when building lazy @Text@ values using a--- builder, associate @mappend@ calls to the right. For example,--- prefer------ > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c')------ to------ > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c'------ as the latter associates @mappend@ to the left.-----------------------------------------------------------------------------------module Data.Text.Internal.Builder- ( -- * Public API- -- ** The Builder type- Builder- , LazyTextBuilder- , toLazyText- , toLazyTextWith-- -- ** Constructing Builders- , singleton- , fromText- , fromLazyText- , fromString-- -- ** Flushing the buffer state- , flush-- -- * Internal functions- , append'- , ensureFree- , writeN- ) where--import Control.Monad.ST (ST, runST)-import Data.Monoid (Monoid(..))-#if !MIN_VERSION_base(4,11,0)-import Data.Semigroup (Semigroup(..))-#endif-import Data.Text.Internal (Text(..), safe)-import Data.Text.Internal.Lazy (smallChunkSize)-import Data.Text.Unsafe (inlineInterleaveST)-import Data.Text.Internal.Unsafe.Char (unsafeWrite)-import Prelude hiding (map, putChar)--import qualified Data.String as String-import qualified Data.Text as S-import qualified Data.Text.Array as A-import qualified Data.Text.Lazy as L--#if defined(ASSERTS)-import GHC.Stack (HasCallStack)-#endif------------------------------------------------------------------------------ | A @Builder@ is an efficient way to build lazy @Text@ values.--- There are several functions for constructing builders, but only one--- to inspect them: to extract any data, you have to turn them into--- lazy @Text@ values using @toLazyText@.------ Internally, a builder constructs a lazy @Text@ by filling arrays--- piece by piece. As each buffer is filled, it is \'popped\' off, to--- become a new chunk of the resulting lazy @Text@. All this is--- hidden from the user of the @Builder@.-newtype Builder = Builder {- -- Invariant (from Data.Text.Lazy):- -- The lists include no null Texts.- runBuilder :: forall s. (Buffer s -> ST s [S.Text])- -> Buffer s- -> ST s [S.Text]- }--type LazyTextBuilder = Builder--instance Semigroup Builder where- (<>) = append- {-# INLINE (<>) #-}--instance Monoid Builder where- mempty = empty- {-# INLINE mempty #-}- mappend = (<>)- {-# INLINE mappend #-}- mconcat = foldr mappend Data.Monoid.mempty- {-# INLINE mconcat #-}---- | Performs replacement on invalid scalar values:------ >>> :set -XOverloadedStrings--- >>> "\55555" :: Builder--- "\65533"-instance String.IsString Builder where- fromString = fromString- {-# INLINE fromString #-}--instance Show Builder where- show = show . toLazyText--instance Eq Builder where- a == b = toLazyText a == toLazyText b--instance Ord Builder where- a <= b = toLazyText a <= toLazyText b------------------------------------------------------------------------------ | /O(1)./ The empty @Builder@, satisfying------ * @'toLazyText' 'empty' = 'L.empty'@----empty :: Builder-empty = Builder (\ k buf -> k buf)-{-# INLINE empty #-}---- | /O(1)./ A @Builder@ taking a single character, satisfying------ * @'toLazyText' ('singleton' c) = 'L.singleton' c@----singleton ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> Builder-singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c)-{-# INLINE singleton #-}------------------------------------------------------------------------------ | /O(1)./ The concatenation of two builders, an associative--- operation with identity 'empty', satisfying------ * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@----append :: Builder -> Builder -> Builder-append (Builder f) (Builder g) = Builder (f . g)-{-# INLINE [0] append #-}---- TODO: Experiment to find the right threshold.-copyLimit :: Int-copyLimit = 128---- This function attempts to merge small @Text@ values instead of--- treating each value as its own chunk. We may not always want this.---- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying------ * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@----fromText :: S.Text -> Builder-fromText t@(Text arr off l)- | S.null t = empty- | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off- | otherwise = flush `append` mapBuilder (t :)-{-# INLINE [1] fromText #-}--{-# RULES-"fromText/pack" forall s .- fromText (S.pack s) = fromString s- #-}---- | /O(1)./ A Builder taking a @String@, satisfying------ * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@------ Performs replacement on invalid scalar values:------ >>> fromString "\55555"--- "\65533"------ @since 1.2.0.0-fromString :: String -> Builder-fromString str = Builder $ \k (Buffer p0 o0 u0 l0) ->- let loop !marr !o !u !l [] = k (Buffer marr o u l)- loop marr o u l s@(c:cs)- | l <= 3 = do- A.shrinkM marr (o + u)- arr <- A.unsafeFreeze marr- let !t = Text arr o u- marr' <- A.new chunkSize- ts <- inlineInterleaveST (loop marr' 0 0 chunkSize s)- return $ t : ts- | otherwise = do- n <- unsafeWrite marr (o+u) (safe c)- loop marr o (u+n) (l-n) cs- in loop p0 o0 u0 l0 str- where- chunkSize = smallChunkSize-{-# INLINEABLE fromString #-}---- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying------ * @'toLazyText' ('fromLazyText' t) = t@----fromLazyText :: L.Text -> Builder-fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++)-{-# INLINE fromLazyText #-}------------------------------------------------------------------------------ Our internal buffer type-data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s)- {-# UNPACK #-} !Int -- offset- {-# UNPACK #-} !Int -- used units- {-# UNPACK #-} !Int -- length left------------------------------------------------------------------------------ | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default--- buffer size. The construction work takes place if and when the--- relevant part of the lazy @Text@ is demanded.-toLazyText :: Builder -> L.Text-toLazyText = toLazyTextWith smallChunkSize---- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given--- size for the initial buffer. The construction work takes place if--- and when the relevant part of the lazy @Text@ is demanded.------ If the initial buffer is too small to hold all data, subsequent--- buffers will be the default buffer size.-toLazyTextWith :: Int -> Builder -> L.Text-toLazyTextWith chunkSize m = L.fromChunks (runST $- newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return [])))---- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any,--- yielding a new chunk in the result lazy @Text@.-flush :: Builder-flush = Builder $ \ k buf@(Buffer p o u l) ->- if u == 0- then k buf- else do arr <- A.unsafeFreeze p- let !b = Buffer p (o+u) 0 l- !t = Text arr o u- ts <- inlineInterleaveST (k b)- return $! t : ts-{-# INLINE [1] flush #-}--- defer inlining so that flush/flush rule may fire.------------------------------------------------------------------------------ | Sequence an ST operation on the buffer-withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder-withBuffer f = Builder $ \k buf -> f buf >>= k-{-# INLINE withBuffer #-}---- | Get the size of the buffer-withSize :: (Int -> Builder) -> Builder-withSize f = Builder $ \ k buf@(Buffer _ _ _ l) ->- runBuilder (f l) k buf-{-# INLINE withSize #-}---- | Map the resulting list of texts.-mapBuilder :: ([S.Text] -> [S.Text]) -> Builder-mapBuilder f = Builder (fmap f .)------------------------------------------------------------------------------ | Ensure that there are at least @n@ many elements available.-ensureFree :: Int -> Builder-ensureFree !n = withSize $ \ l ->- if n <= l- then empty- else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize)))-{-# INLINE [0] ensureFree #-}--writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder-writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f)-{-# INLINE [0] writeAtMost #-}---- | Ensure that @n@ many elements are available, and then use @f@ to--- write some elements into the memory.-writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder-writeN n f = writeAtMost n (\ p o -> f p o >> return n)-{-# INLINE writeN #-}--writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s)-writeBuffer f (Buffer p o u l) = do- n <- f p (o+u)- return $! Buffer p o (u+n) (l-n)-{-# INLINE writeBuffer #-}--newBuffer :: Int -> ST s (Buffer s)-newBuffer size = do- arr <- A.new size- return $! Buffer arr 0 0 size-{-# INLINE newBuffer #-}----------------------------------------------------------------------------- Some nice rules for Builder---- This function makes GHC understand that 'writeN' and 'ensureFree'--- are *not* recursive in the precense of the rewrite rules below.--- This is not needed with GHC 7+.-append' :: Builder -> Builder -> Builder-append' (Builder f) (Builder g) = Builder (f . g)-{-# INLINE append' #-}--{-# RULES--"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)- (g::forall s. A.MArray s -> Int -> ST s Int) ws.- append (writeAtMost a f) (append (writeAtMost b g) ws) =- append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->- g marr (o+n) >>= \ m ->- let s = n+m in s `seq` return s)) ws--"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int)- (g::forall s. A.MArray s -> Int -> ST s Int).- append (writeAtMost a f) (writeAtMost b g) =- writeAtMost (a+b) (\marr o -> f marr o >>= \ n ->- g marr (o+n) >>= \ m ->- let s = n+m in s `seq` return s)--"ensureFree/ensureFree" forall a b .- append (ensureFree a) (ensureFree b) = ensureFree (max a b)--"flush/flush"- append flush flush = flush-- #-}+{-# LANGUAGE BangPatterns, CPP, RankNTypes #-} +{-# OPTIONS_HADDOCK not-home #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Data.Text.Internal.Builder +-- Copyright : (c) 2013 Bryan O'Sullivan +-- (c) 2010 Johan Tibell +-- License : BSD-style (see LICENSE) +-- +-- Maintainer : Johan Tibell <johan.tibell@gmail.com> +-- Stability : experimental +-- Portability : portable to Hugs and GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Efficient construction of lazy @Text@ values. The principal +-- operations on a @Builder@ are @singleton@, @fromText@, and +-- @fromLazyText@, which construct new builders, and 'mappend', which +-- concatenates two builders. +-- +-- To get maximum performance when building lazy @Text@ values using a +-- builder, associate @mappend@ calls to the right. For example, +-- prefer +-- +-- > singleton 'a' `mappend` (singleton 'b' `mappend` singleton 'c') +-- +-- to +-- +-- > singleton 'a' `mappend` singleton 'b' `mappend` singleton 'c' +-- +-- as the latter associates @mappend@ to the left. +-- +----------------------------------------------------------------------------- + +module Data.Text.Internal.Builder + ( -- * Public API + -- ** The Builder type + Builder + , LazyTextBuilder + , toLazyText + , toLazyTextWith + + -- ** Constructing Builders + , singleton + , fromText + , fromLazyText + , fromString + + -- ** Flushing the buffer state + , flush + + -- * Internal functions + , append' + , ensureFree + , writeN + ) where + +import Control.Monad.ST (ST, runST) +import Data.Monoid (Monoid(..)) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup (Semigroup(..)) +#endif +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Lazy (smallChunkSize) +import Data.Text.Unsafe (inlineInterleaveST) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Prelude hiding (map, putChar) + +import qualified Data.String as String +import qualified Data.Text as S +import qualified Data.Text.Array as A +import qualified Data.Text.Lazy as L + +#if defined(ASSERTS) +import GHC.Stack (HasCallStack) +#endif + +------------------------------------------------------------------------ + +-- | A @Builder@ is an efficient way to build lazy @Text@ values. +-- There are several functions for constructing builders, but only one +-- to inspect them: to extract any data, you have to turn them into +-- lazy @Text@ values using @toLazyText@. +-- +-- Internally, a builder constructs a lazy @Text@ by filling arrays +-- piece by piece. As each buffer is filled, it is \'popped\' off, to +-- become a new chunk of the resulting lazy @Text@. All this is +-- hidden from the user of the @Builder@. +newtype Builder = Builder { + -- Invariant (from Data.Text.Lazy): + -- The lists include no null Texts. + runBuilder :: forall s. (Buffer s -> ST s [S.Text]) + -> Buffer s + -> ST s [S.Text] + } + +-- | @since 2.1.2 +type LazyTextBuilder = Builder + +instance Semigroup Builder where + (<>) = append + {-# INLINE (<>) #-} + +instance Monoid Builder where + mempty = empty + {-# INLINE mempty #-} + mappend = (<>) + {-# INLINE mappend #-} + mconcat = foldr mappend Data.Monoid.mempty + {-# INLINE mconcat #-} + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Builder +-- "\65533" +instance String.IsString Builder where + fromString = fromString + {-# INLINE fromString #-} + +instance Show Builder where + show = show . toLazyText + +instance Eq Builder where + a == b = toLazyText a == toLazyText b + +instance Ord Builder where + a <= b = toLazyText a <= toLazyText b + +------------------------------------------------------------------------ + +-- | /O(1)./ The empty @Builder@, satisfying +-- +-- * @'toLazyText' 'empty' = 'L.empty'@ +-- +empty :: Builder +empty = Builder (\ k buf -> k buf) +{-# INLINE empty #-} + +-- | /O(1)./ A @Builder@ taking a single character, satisfying +-- +-- * @'toLazyText' ('singleton' c) = 'L.singleton' c@ +-- +singleton :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> Builder +singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) +{-# INLINE singleton #-} + +------------------------------------------------------------------------ + +-- | /O(1)./ The concatenation of two builders, an associative +-- operation with identity 'empty', satisfying +-- +-- * @'toLazyText' ('append' x y) = 'L.append' ('toLazyText' x) ('toLazyText' y)@ +-- +append :: Builder -> Builder -> Builder +append (Builder f) (Builder g) = Builder (f . g) +{-# INLINE [0] append #-} + +-- TODO: Experiment to find the right threshold. +copyLimit :: Int +copyLimit = 128 + +-- This function attempts to merge small @Text@ values instead of +-- treating each value as its own chunk. We may not always want this. + +-- | /O(1)./ A @Builder@ taking a 'S.Text', satisfying +-- +-- * @'toLazyText' ('fromText' t) = 'L.fromChunks' [t]@ +-- +fromText :: S.Text -> Builder +fromText t@(Text arr off l) + | S.null t = empty + | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off + | otherwise = flush `append` mapBuilder (t :) +{-# INLINE [1] fromText #-} + +{-# RULES +"fromText/pack" forall s . + fromText (S.pack s) = fromString s + #-} + +-- | /O(1)./ A Builder taking a @String@, satisfying +-- +-- * @'toLazyText' ('fromString' s) = 'L.fromChunks' [S.pack s]@ +-- +-- Performs replacement on invalid scalar values: +-- +-- >>> fromString "\55555" +-- "\65533" +-- +-- @since 1.2.0.0 +fromString :: String -> Builder +fromString str = Builder $ \k (Buffer p0 o0 u0) -> do + len <- A.getSizeofMArray p0 + -- `end` is 3 bytes before the actual end of `marr` + -- to make sure there's room for a 4-byte UTF-8 code point. + let loop !marr !o !u !_ [] = k (Buffer marr o u) + loop marr o u end s@(c:cs) + | u >= end = do + A.shrinkM marr (o + u) + arr <- A.unsafeFreeze marr + let !t = Text arr o u + marr' <- A.new chunkSize + ts <- inlineInterleaveST (loop marr' 0 0 (chunkSize - 3) s) + return $ t : ts + | otherwise = do + n <- unsafeWrite marr (o+u) (safe c) + loop marr o (u+n) end cs + loop p0 o0 u0 (len - o0 - 3) str + where + chunkSize = smallChunkSize +{-# INLINEABLE fromString #-} + +-- | /O(1)./ A @Builder@ taking a lazy @Text@, satisfying +-- +-- * @'toLazyText' ('fromLazyText' t) = t@ +-- +fromLazyText :: L.Text -> Builder +fromLazyText ts = flush `append` mapBuilder (L.toChunks ts ++) +{-# INLINE fromLazyText #-} + +------------------------------------------------------------------------ + +-- Our internal buffer type +data Buffer s = Buffer {-# UNPACK #-} !(A.MArray s) + {-# UNPACK #-} !Int -- offset + {-# UNPACK #-} !Int -- used units + +------------------------------------------------------------------------ + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@ with a default +-- buffer size. The construction work takes place if and when the +-- relevant part of the lazy @Text@ is demanded. +toLazyText :: Builder -> L.Text +toLazyText = toLazyTextWith smallChunkSize + +-- | /O(n)./ Extract a lazy @Text@ from a @Builder@, using the given +-- size for the initial buffer. The construction work takes place if +-- and when the relevant part of the lazy @Text@ is demanded. +-- +-- If the initial buffer is too small to hold all data, subsequent +-- buffers will be the default buffer size. +toLazyTextWith :: Int -> Builder -> L.Text +toLazyTextWith chunkSize m = L.fromChunks (runST $ + newBuffer chunkSize >>= runBuilder (m `append` flush) (const (return []))) + +-- | /O(1)./ Pop the strict @Text@ we have constructed so far, if any, +-- yielding a new chunk in the result lazy @Text@. +flush :: Builder +flush = Builder $ \ k buf@(Buffer p o u) -> + if u == 0 + then k buf + else do arr <- A.unsafeFreeze p + let !b = Buffer p (o+u) 0 + !t = Text arr o u + ts <- inlineInterleaveST (k b) + return $! t : ts +{-# INLINE [1] flush #-} +-- defer inlining so that flush/flush rule may fire. + +------------------------------------------------------------------------ + +-- | Sequence an ST operation on the buffer +withBuffer :: (forall s. Buffer s -> ST s (Buffer s)) -> Builder +withBuffer f = Builder $ \k buf -> f buf >>= k +{-# INLINE withBuffer #-} + +-- | Get the size of the buffer +withSize :: (Int -> Builder) -> Builder +withSize f = Builder $ \ k buf@(Buffer arr offset used) -> do + len <- A.getSizeofMArray arr + runBuilder (f (len - offset - used)) k buf +{-# INLINE withSize #-} + +-- | Map the resulting list of texts. +mapBuilder :: ([S.Text] -> [S.Text]) -> Builder +mapBuilder f = Builder (fmap f .) + +------------------------------------------------------------------------ + +-- | Ensure that there are at least @n@ many elements available. +ensureFree :: Int -> Builder +ensureFree !n = withSize $ \ l -> + if n <= l + then empty + else flush `append'` withBuffer (const (newBuffer (max n smallChunkSize))) +{-# INLINE [0] ensureFree #-} + +writeAtMost :: Int -> (forall s. A.MArray s -> Int -> ST s Int) -> Builder +writeAtMost n f = ensureFree n `append'` withBuffer (writeBuffer f) +{-# INLINE [0] writeAtMost #-} + +-- | Ensure that @n@ many elements are available, and then use @f@ to +-- write some elements into the memory. +writeN :: Int -> (forall s. A.MArray s -> Int -> ST s ()) -> Builder +writeN n f = writeAtMost n (\ p o -> f p o >> return n) +{-# INLINE writeN #-} + +writeBuffer :: (A.MArray s -> Int -> ST s Int) -> Buffer s -> ST s (Buffer s) +writeBuffer f (Buffer p o u) = do + n <- f p (o+u) + return $! Buffer p o (u+n) +{-# INLINE writeBuffer #-} + +newBuffer :: Int -> ST s (Buffer s) +newBuffer size = do + arr <- A.new size + return $! Buffer arr 0 0 +{-# INLINE newBuffer #-} + +------------------------------------------------------------------------ +-- Some nice rules for Builder + +-- This function makes GHC understand that 'writeN' and 'ensureFree' +-- are *not* recursive in the presence of the rewrite rules below. +-- This is not needed with GHC 7+. +append' :: Builder -> Builder -> Builder +append' (Builder f) (Builder g) = Builder (f . g) +{-# INLINE append' #-} + +{-# RULES + +"append/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int) ws. + append (writeAtMost a f) (append (writeAtMost b g) ws) = + append (writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s)) ws + +"writeAtMost/writeAtMost" forall a b (f::forall s. A.MArray s -> Int -> ST s Int) + (g::forall s. A.MArray s -> Int -> ST s Int). + append (writeAtMost a f) (writeAtMost b g) = + writeAtMost (a+b) (\marr o -> f marr o >>= \ n -> + g marr (o+n) >>= \ m -> + let s = n+m in s `seq` return s) + +"ensureFree/ensureFree" forall a b . + append (ensureFree a) (ensureFree b) = ensureFree (max a b) + +"flush/flush" + append flush flush = flush + + #-}
src/Data/Text/Internal/Encoding/Utf8.hs view
@@ -1,298 +1,318 @@-{-# LANGUAGE CPP, MagicHash, BangPatterns #-}---- |--- Module : Data.Text.Internal.Encoding.Utf8--- Copyright : (c) 2008, 2009 Tom Harper,--- (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts--- (c) 2021 Andrew Lelechenko------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Basic UTF-8 validation and character manipulation.-module Data.Text.Internal.Encoding.Utf8- ( utf8Length- , utf8LengthByLeader- -- Decomposition- , ord2- , ord3- , ord4- -- Construction- , chr2- , chr3- , chr4- -- * Validation- , validate1- , validate2- , validate3- , validate4- -- * Naive decoding- , DecoderState(..)- , utf8AcceptState- , utf8RejectState- , updateDecoderState- , DecoderResult(..)- , CodePoint(..)- , utf8DecodeStart- , utf8DecodeContinue- ) where--#if defined(ASSERTS)-import Control.Exception (assert)-import GHC.Stack (HasCallStack)-#endif-import Data.Bits (Bits(..), FiniteBits(..))-import Data.Char (ord, chr)-import GHC.Exts-import GHC.Word (Word8(..))--#if !MIN_VERSION_base(4,16,0)--- harmless to import, except for warnings that it is unused.-import Data.Text.Internal.PrimCompat (word8ToWord#)-#endif--default(Int)--between :: Word8 -- ^ byte to check- -> Word8 -- ^ lower bound- -> Word8 -- ^ upper bound- -> Bool-between x y z = x >= y && x <= z-{-# INLINE between #-}---- This is a branchless version of--- utf8Length c--- | ord c < 0x80 = 1--- | ord c < 0x800 = 2--- | ord c < 0x10000 = 3--- | otherwise = 4--- Implementation suggested by Alex Mason.---- | @since 2.0-utf8Length :: Char -> Int-utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#)))-{-# INLINE utf8Length #-}---- This is a branchless version of--- utf8LengthByLeader w--- | w < 0x80 = 1--- | w < 0xE0 = 2--- | w < 0xF0 = 3--- | otherwise = 4------ c `xor` I# (c# <=# 0#) is a branchless equivalent of c `max` 1.--- It is crucial to write c# <=# 0# and not c# ==# 0#, otherwise--- GHC is tempted to "optimize" by introduction of branches.---- | @since 2.0-utf8LengthByLeader :: Word8 -> Int-utf8LengthByLeader w = c `xor` I# (c# <=# 0#)- where- !c@(I# c#) = countLeadingZeros (complement w)-{-# INLINE utf8LengthByLeader #-}--ord2 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8)-ord2 c =-#if defined(ASSERTS)- assert (n >= 0x80 && n <= 0x07ff)-#endif- (x1,x2)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 6) + 0xC0- x2 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord2 #-}--ord3 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8,Word8)-ord3 c =-#if defined(ASSERTS)- assert (n >= 0x0800 && n <= 0xffff)-#endif- (x1,x2,x3)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 12) + 0xE0- x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80- x3 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord3 #-}--ord4 ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Char -> (Word8,Word8,Word8,Word8)-ord4 c =-#if defined(ASSERTS)- assert (n >= 0x10000)-#endif- (x1,x2,x3,x4)- where- n = ord c- x1 = intToWord8 $ (n `shiftR` 18) + 0xF0- x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80- x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80- x4 = intToWord8 $ (n .&. 0x3F) + 0x80-{-# INLINE ord4 #-}--chr2 :: Word8 -> Word8 -> Char-chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6#- !z2# = y2# -# 0x80#-{-# INLINE chr2 #-}--chr3 :: Word8 -> Word8 -> Word8 -> Char-chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !y3# = word2Int# (word8ToWord# x3#)- !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12#- !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6#- !z3# = y3# -# 0x80#-{-# INLINE chr3 #-}--chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char-chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) =- C# (chr# (z1# +# z2# +# z3# +# z4#))- where- !y1# = word2Int# (word8ToWord# x1#)- !y2# = word2Int# (word8ToWord# x2#)- !y3# = word2Int# (word8ToWord# x3#)- !y4# = word2Int# (word8ToWord# x4#)- !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18#- !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12#- !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6#- !z4# = y4# -# 0x80#-{-# INLINE chr4 #-}--validate1 :: Word8 -> Bool-validate1 x1 = x1 <= 0x7F-{-# INLINE validate1 #-}--validate2 :: Word8 -> Word8 -> Bool-validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF-{-# INLINE validate2 #-}--validate3 :: Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate3 #-}-validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4- where- validate3_1 = (x1 == 0xE0) &&- between x2 0xA0 0xBF &&- between x3 0x80 0xBF- validate3_2 = between x1 0xE1 0xEC &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF- validate3_3 = x1 == 0xED &&- between x2 0x80 0x9F &&- between x3 0x80 0xBF- validate3_4 = between x1 0xEE 0xEF &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF--validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool-{-# INLINE validate4 #-}-validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3- where- validate4_1 = x1 == 0xF0 &&- between x2 0x90 0xBF &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF- validate4_2 = between x1 0xF1 0xF3 &&- between x2 0x80 0xBF &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF- validate4_3 = x1 == 0xF4 &&- between x2 0x80 0x8F &&- between x3 0x80 0xBF &&- between x4 0x80 0xBF--intToWord8 :: Int -> Word8-intToWord8 = fromIntegral--word8ToInt :: Word8 -> Int-word8ToInt = fromIntegral------------------------------------------------------------------------------------ Naive UTF8 decoder.--- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine.--newtype ByteClass = ByteClass Word8--byteToClass :: Word8 -> ByteClass-byteToClass n = ByteClass (W8# el#)- where- !(I# n#) = word8ToInt n- el# = indexWord8OffAddr# table# n#-- table# :: Addr#- table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"#--newtype DecoderState = DecoderState Word8- deriving (Eq, Show)--utf8AcceptState :: DecoderState-utf8AcceptState = DecoderState 0--utf8RejectState :: DecoderState-utf8RejectState = DecoderState 12--updateState :: ByteClass -> DecoderState -> DecoderState-updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#)- where- !(I# n#) = word8ToInt (c + s)- el# = indexWord8OffAddr# table# n#-- table# :: Addr#- table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"#--updateDecoderState :: Word8 -> DecoderState -> DecoderState-updateDecoderState b s = updateState (byteToClass b) s--newtype CodePoint = CodePoint Int---- | @since 2.0-data DecoderResult- = Accept !Char- | Incomplete !DecoderState !CodePoint- | Reject---- | @since 2.0-utf8DecodeStart :: Word8 -> DecoderResult-utf8DecodeStart !w- | st == utf8AcceptState = Accept (chr (word8ToInt w))- | st == utf8RejectState = Reject- | otherwise = Incomplete st (CodePoint cp)- where- cl@(ByteClass cl') = byteToClass w- st = updateState cl utf8AcceptState- cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w---- | @since 2.0-utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult-utf8DecodeContinue !w !st (CodePoint !cp)- | st' == utf8AcceptState = Accept (chr cp')- | st' == utf8RejectState = Reject- | otherwise = Incomplete st' (CodePoint cp')- where- cl = byteToClass w- st' = updateState cl st- cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f)+{-# LANGUAGE CPP, MagicHash, BangPatterns #-} + +-- | +-- Module : Data.Text.Internal.Encoding.Utf8 +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- (c) 2021 Andrew Lelechenko +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Basic UTF-8 validation and character manipulation. +module Data.Text.Internal.Encoding.Utf8 + ( utf8Length + , utf8LengthByLeader + -- Decomposition + , ord2 + , ord3 + , ord4 + -- Construction + , chr2 + , chr3 + , chr4 + -- * Validation + , validate1 + , validate2 + , validate3 + , validate4 + -- * Naive decoding + , DecoderState(..) + , utf8AcceptState + , utf8RejectState + , updateDecoderState + , DecoderResult(..) + , CodePoint(..) + , utf8DecodeStart + , utf8DecodeContinue + ) where + +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Bits (Bits(..)) +import Data.Char (ord, chr) +import GHC.Exts +import GHC.Word (Word8(..)) + +#if !MIN_VERSION_base(4,16,0) +-- harmless to import, except for warnings that it is unused. +import Data.Text.Internal.PrimCompat (word8ToWord#) +#endif + +default(Int) + +between :: Word8 -- ^ byte to check + -> Word8 -- ^ lower bound + -> Word8 -- ^ upper bound + -> Bool +between x y z = x >= y && x <= z +{-# INLINE between #-} + +-- This is a branchless version of +-- utf8Length c +-- | ord c < 0x80 = 1 +-- | ord c < 0x800 = 2 +-- | ord c < 0x10000 = 3 +-- | otherwise = 4 +-- Implementation suggested by Alex Mason. + +-- | Measure byte length of UTF-8 encoding for a given character. +-- +-- @since 2.0 +utf8Length :: Char -> Int +utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) +{-# INLINE utf8Length #-} + +-- | Measure byte length of UTF-8 encoding for characters, +-- starting with a given byte. +-- +-- @since 2.0 +utf8LengthByLeader :: Word8 -> Int +utf8LengthByLeader w + | w < 0x80 = 1 + | w < 0xE0 = 2 + | w < 0xF0 = 3 + | otherwise = 4 +{-# INLINE utf8LengthByLeader #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 2 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8) +ord2 c = +#if defined(ASSERTS) + assert (n >= 0x80 && n <= 0x07ff) +#endif + (x1,x2) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 + x2 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord2 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 3 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord3 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8) +ord3 c = +#if defined(ASSERTS) + assert (n >= 0x0800 && n <= 0xffff) +#endif + (x1,x2,x3) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 + x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x3 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord3 #-} + +-- | Encode a character as UTF-8 bytes assuming that exactly 4 are needed. +-- This precondition is not checked. +-- +-- @since 1.1.0.0 +ord4 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8,Word8) +ord4 c = +#if defined(ASSERTS) + assert (n >= 0x10000) +#endif + (x1,x2,x3,x4) + where + n = ord c + x1 = intToWord8 $ (n `shiftR` 18) + 0xF0 + x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 + x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 + x4 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord4 #-} + +-- | @since 1.1.0.0 +chr2 :: Word8 -> Word8 -> Char +chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6# + !z2# = y2# -# 0x80# +{-# INLINE chr2 #-} + +-- | @since 1.1.0.0 +chr3 :: Word8 -> Word8 -> Word8 -> Char +chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6# + !z3# = y3# -# 0x80# +{-# INLINE chr3 #-} + +-- | @since 1.1.0.0 +chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char +chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = + C# (chr# (z1# +# z2# +# z3# +# z4#)) + where + !y1# = word2Int# (word8ToWord# x1#) + !y2# = word2Int# (word8ToWord# x2#) + !y3# = word2Int# (word8ToWord# x3#) + !y4# = word2Int# (word8ToWord# x4#) + !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18# + !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12# + !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6# + !z4# = y4# -# 0x80# +{-# INLINE chr4 #-} + +-- | @since 1.1.0.0 +validate1 :: Word8 -> Bool +validate1 x1 = x1 <= 0x7F +{-# INLINE validate1 #-} + +-- | @since 1.1.0.0 +validate2 :: Word8 -> Word8 -> Bool +validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF +{-# INLINE validate2 #-} + +-- | @since 1.1.0.0 +validate3 :: Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate3 #-} +validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4 + where + validate3_1 = (x1 == 0xE0) && + between x2 0xA0 0xBF && + between x3 0x80 0xBF + validate3_2 = between x1 0xE1 0xEC && + between x2 0x80 0xBF && + between x3 0x80 0xBF + validate3_3 = x1 == 0xED && + between x2 0x80 0x9F && + between x3 0x80 0xBF + validate3_4 = between x1 0xEE 0xEF && + between x2 0x80 0xBF && + between x3 0x80 0xBF + +-- | @since 1.1.0.0 +validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool +{-# INLINE validate4 #-} +validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3 + where + validate4_1 = x1 == 0xF0 && + between x2 0x90 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_2 = between x1 0xF1 0xF3 && + between x2 0x80 0xBF && + between x3 0x80 0xBF && + between x4 0x80 0xBF + validate4_3 = x1 == 0xF4 && + between x2 0x80 0x8F && + between x3 0x80 0xBF && + between x4 0x80 0xBF + +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +------------------------------------------------------------------------------- +-- Naive UTF8 decoder. +-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. + +newtype ByteClass = ByteClass Word8 + +byteToClass :: Word8 -> ByteClass +byteToClass n = ByteClass (W8# el#) + where + !(I# n#) = word8ToInt n + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# + +-- | @since 2.0 +newtype DecoderState = DecoderState Word8 + deriving (Eq, Show) + +-- | @since 2.0.2 +utf8AcceptState :: DecoderState +utf8AcceptState = DecoderState 0 + +-- | @since 2.0.2 +utf8RejectState :: DecoderState +utf8RejectState = DecoderState 12 + +updateState :: ByteClass -> DecoderState -> DecoderState +updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) + where + !(I# n#) = word8ToInt (c + s) + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# + +-- | @since 2.0.2 +updateDecoderState :: Word8 -> DecoderState -> DecoderState +updateDecoderState b s = updateState (byteToClass b) s + +-- | @since 2.0 +newtype CodePoint = CodePoint Int + +-- | @since 2.0 +data DecoderResult + = Accept !Char + | Incomplete !DecoderState !CodePoint + | Reject + +-- | @since 2.0 +utf8DecodeStart :: Word8 -> DecoderResult +utf8DecodeStart !w + | st == utf8AcceptState = Accept (chr (word8ToInt w)) + | st == utf8RejectState = Reject + | otherwise = Incomplete st (CodePoint cp) + where + cl@(ByteClass cl') = byteToClass w + st = updateState cl utf8AcceptState + cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w + +-- | @since 2.0 +utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult +utf8DecodeContinue !w !st (CodePoint !cp) + | st' == utf8AcceptState = Accept (chr cp') + | st' == utf8RejectState = Reject + | otherwise = Incomplete st' (CodePoint cp') + where + cl = byteToClass w + st' = updateState cl st + cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f)
src/Data/Text/Internal/Fusion/CaseMapping.hs view
@@ -1,7706 +1,7984 @@--- AUTOMATICALLY GENERATED - DO NOT EDIT--- Generated by scripts/CaseMapping.hs--- CaseFolding-16.0.0.txt--- Date: 2024-04-30, 21:48:11 GMT--- SpecialCasing-16.0.0.txt--- Date: 2024-05-10, 22:49:00 GMT--{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}-{-# OPTIONS_GHC -Wno-partial-type-signatures #-}-module Data.Text.Internal.Fusion.CaseMapping where-import GHC.Int-import GHC.Exts-unI64 :: Int64 -> _ {- unboxed Int64 -}-unI64 (I64# n) = n--upperMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE upperMapping #-}-upperMapping = \case- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 174063699- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 146800710- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 153092166- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 159383622- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 321057542111302- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 334251681644614- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 176160851- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 176160851- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2856322357- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2831156548- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2795504964- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2808087876- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2831156558- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2812282180- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 163578556- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429849- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429861- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778634- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373256- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390036- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584343- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584345- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200769- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459557- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987429- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498533- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720293- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025681- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025687- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918745- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429849- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025689- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651609- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918757- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429861- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459553- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025701- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651621- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025705- -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI- '\x1f80'# -> unI64 1931484936- -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI- '\x1f81'# -> unI64 1931484937- -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f82'# -> unI64 1931484938- -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f83'# -> unI64 1931484939- -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f84'# -> unI64 1931484940- -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f85'# -> unI64 1931484941- -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f86'# -> unI64 1931484942- -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f87'# -> unI64 1931484943- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI- '\x1f88'# -> unI64 1931484936- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI- '\x1f89'# -> unI64 1931484937- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f8a'# -> unI64 1931484938- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f8b'# -> unI64 1931484939- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f8c'# -> unI64 1931484940- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f8d'# -> unI64 1931484941- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8e'# -> unI64 1931484942- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8f'# -> unI64 1931484943- -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI- '\x1f90'# -> unI64 1931484968- -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI- '\x1f91'# -> unI64 1931484969- -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f92'# -> unI64 1931484970- -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f93'# -> unI64 1931484971- -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f94'# -> unI64 1931484972- -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f95'# -> unI64 1931484973- -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f96'# -> unI64 1931484974- -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f97'# -> unI64 1931484975- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI- '\x1f98'# -> unI64 1931484968- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI- '\x1f99'# -> unI64 1931484969- -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f9a'# -> unI64 1931484970- -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f9b'# -> unI64 1931484971- -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f9c'# -> unI64 1931484972- -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f9d'# -> unI64 1931484973- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9e'# -> unI64 1931484974- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9f'# -> unI64 1931484975- -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI- '\x1fa0'# -> unI64 1931485032- -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI- '\x1fa1'# -> unI64 1931485033- -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1fa2'# -> unI64 1931485034- -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1fa3'# -> unI64 1931485035- -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1fa4'# -> unI64 1931485036- -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1fa5'# -> unI64 1931485037- -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa6'# -> unI64 1931485038- -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa7'# -> unI64 1931485039- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI- '\x1fa8'# -> unI64 1931485032- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI- '\x1fa9'# -> unI64 1931485033- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1faa'# -> unI64 1931485034- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1fab'# -> unI64 1931485035- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1fac'# -> unI64 1931485036- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1fad'# -> unI64 1931485037- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1fae'# -> unI64 1931485038- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1faf'# -> unI64 1931485039- -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI- '\x1fb3'# -> unI64 1931477905- -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI- '\x1fbc'# -> unI64 1931477905- -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI- '\x1fc3'# -> unI64 1931477911- -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI- '\x1fcc'# -> unI64 1931477911- -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI- '\x1ff3'# -> unI64 1931477929- -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI- '\x1ffc'# -> unI64 1931477929- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1931485114- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1931477894- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1931485130- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1931477897- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1931485178- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1931477903- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 4050602585752465- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 4050602585752471- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 4050602585752489- '\x0061'# -> unI64 65- '\x0062'# -> unI64 66- '\x0063'# -> unI64 67- '\x0064'# -> unI64 68- '\x0065'# -> unI64 69- '\x0066'# -> unI64 70- '\x0067'# -> unI64 71- '\x0068'# -> unI64 72- '\x0069'# -> unI64 73- '\x006a'# -> unI64 74- '\x006b'# -> unI64 75- '\x006c'# -> unI64 76- '\x006d'# -> unI64 77- '\x006e'# -> unI64 78- '\x006f'# -> unI64 79- '\x0070'# -> unI64 80- '\x0071'# -> unI64 81- '\x0072'# -> unI64 82- '\x0073'# -> unI64 83- '\x0074'# -> unI64 84- '\x0075'# -> unI64 85- '\x0076'# -> unI64 86- '\x0077'# -> unI64 87- '\x0078'# -> unI64 88- '\x0079'# -> unI64 89- '\x007a'# -> unI64 90- '\x00b5'# -> unI64 924- '\x00e0'# -> unI64 192- '\x00e1'# -> unI64 193- '\x00e2'# -> unI64 194- '\x00e3'# -> unI64 195- '\x00e4'# -> unI64 196- '\x00e5'# -> unI64 197- '\x00e6'# -> unI64 198- '\x00e7'# -> unI64 199- '\x00e8'# -> unI64 200- '\x00e9'# -> unI64 201- '\x00ea'# -> unI64 202- '\x00eb'# -> unI64 203- '\x00ec'# -> unI64 204- '\x00ed'# -> unI64 205- '\x00ee'# -> unI64 206- '\x00ef'# -> unI64 207- '\x00f0'# -> unI64 208- '\x00f1'# -> unI64 209- '\x00f2'# -> unI64 210- '\x00f3'# -> unI64 211- '\x00f4'# -> unI64 212- '\x00f5'# -> unI64 213- '\x00f6'# -> unI64 214- '\x00f8'# -> unI64 216- '\x00f9'# -> unI64 217- '\x00fa'# -> unI64 218- '\x00fb'# -> unI64 219- '\x00fc'# -> unI64 220- '\x00fd'# -> unI64 221- '\x00fe'# -> unI64 222- '\x00ff'# -> unI64 376- '\x0101'# -> unI64 256- '\x0103'# -> unI64 258- '\x0105'# -> unI64 260- '\x0107'# -> unI64 262- '\x0109'# -> unI64 264- '\x010b'# -> unI64 266- '\x010d'# -> unI64 268- '\x010f'# -> unI64 270- '\x0111'# -> unI64 272- '\x0113'# -> unI64 274- '\x0115'# -> unI64 276- '\x0117'# -> unI64 278- '\x0119'# -> unI64 280- '\x011b'# -> unI64 282- '\x011d'# -> unI64 284- '\x011f'# -> unI64 286- '\x0121'# -> unI64 288- '\x0123'# -> unI64 290- '\x0125'# -> unI64 292- '\x0127'# -> unI64 294- '\x0129'# -> unI64 296- '\x012b'# -> unI64 298- '\x012d'# -> unI64 300- '\x012f'# -> unI64 302- '\x0131'# -> unI64 73- '\x0133'# -> unI64 306- '\x0135'# -> unI64 308- '\x0137'# -> unI64 310- '\x013a'# -> unI64 313- '\x013c'# -> unI64 315- '\x013e'# -> unI64 317- '\x0140'# -> unI64 319- '\x0142'# -> unI64 321- '\x0144'# -> unI64 323- '\x0146'# -> unI64 325- '\x0148'# -> unI64 327- '\x014b'# -> unI64 330- '\x014d'# -> unI64 332- '\x014f'# -> unI64 334- '\x0151'# -> unI64 336- '\x0153'# -> unI64 338- '\x0155'# -> unI64 340- '\x0157'# -> unI64 342- '\x0159'# -> unI64 344- '\x015b'# -> unI64 346- '\x015d'# -> unI64 348- '\x015f'# -> unI64 350- '\x0161'# -> unI64 352- '\x0163'# -> unI64 354- '\x0165'# -> unI64 356- '\x0167'# -> unI64 358- '\x0169'# -> unI64 360- '\x016b'# -> unI64 362- '\x016d'# -> unI64 364- '\x016f'# -> unI64 366- '\x0171'# -> unI64 368- '\x0173'# -> unI64 370- '\x0175'# -> unI64 372- '\x0177'# -> unI64 374- '\x017a'# -> unI64 377- '\x017c'# -> unI64 379- '\x017e'# -> unI64 381- '\x017f'# -> unI64 83- '\x0180'# -> unI64 579- '\x0183'# -> unI64 386- '\x0185'# -> unI64 388- '\x0188'# -> unI64 391- '\x018c'# -> unI64 395- '\x0192'# -> unI64 401- '\x0195'# -> unI64 502- '\x0199'# -> unI64 408- '\x019a'# -> unI64 573- '\x019e'# -> unI64 544- '\x01a1'# -> unI64 416- '\x01a3'# -> unI64 418- '\x01a5'# -> unI64 420- '\x01a8'# -> unI64 423- '\x01ad'# -> unI64 428- '\x01b0'# -> unI64 431- '\x01b4'# -> unI64 435- '\x01b6'# -> unI64 437- '\x01b9'# -> unI64 440- '\x01bd'# -> unI64 444- '\x01bf'# -> unI64 503- '\x01c5'# -> unI64 452- '\x01c6'# -> unI64 452- '\x01c8'# -> unI64 455- '\x01c9'# -> unI64 455- '\x01cb'# -> unI64 458- '\x01cc'# -> unI64 458- '\x01ce'# -> unI64 461- '\x01d0'# -> unI64 463- '\x01d2'# -> unI64 465- '\x01d4'# -> unI64 467- '\x01d6'# -> unI64 469- '\x01d8'# -> unI64 471- '\x01da'# -> unI64 473- '\x01dc'# -> unI64 475- '\x01dd'# -> unI64 398- '\x01df'# -> unI64 478- '\x01e1'# -> unI64 480- '\x01e3'# -> unI64 482- '\x01e5'# -> unI64 484- '\x01e7'# -> unI64 486- '\x01e9'# -> unI64 488- '\x01eb'# -> unI64 490- '\x01ed'# -> unI64 492- '\x01ef'# -> unI64 494- '\x01f2'# -> unI64 497- '\x01f3'# -> unI64 497- '\x01f5'# -> unI64 500- '\x01f9'# -> unI64 504- '\x01fb'# -> unI64 506- '\x01fd'# -> unI64 508- '\x01ff'# -> unI64 510- '\x0201'# -> unI64 512- '\x0203'# -> unI64 514- '\x0205'# -> unI64 516- '\x0207'# -> unI64 518- '\x0209'# -> unI64 520- '\x020b'# -> unI64 522- '\x020d'# -> unI64 524- '\x020f'# -> unI64 526- '\x0211'# -> unI64 528- '\x0213'# -> unI64 530- '\x0215'# -> unI64 532- '\x0217'# -> unI64 534- '\x0219'# -> unI64 536- '\x021b'# -> unI64 538- '\x021d'# -> unI64 540- '\x021f'# -> unI64 542- '\x0223'# -> unI64 546- '\x0225'# -> unI64 548- '\x0227'# -> unI64 550- '\x0229'# -> unI64 552- '\x022b'# -> unI64 554- '\x022d'# -> unI64 556- '\x022f'# -> unI64 558- '\x0231'# -> unI64 560- '\x0233'# -> unI64 562- '\x023c'# -> unI64 571- '\x023f'# -> unI64 11390- '\x0240'# -> unI64 11391- '\x0242'# -> unI64 577- '\x0247'# -> unI64 582- '\x0249'# -> unI64 584- '\x024b'# -> unI64 586- '\x024d'# -> unI64 588- '\x024f'# -> unI64 590- '\x0250'# -> unI64 11375- '\x0251'# -> unI64 11373- '\x0252'# -> unI64 11376- '\x0253'# -> unI64 385- '\x0254'# -> unI64 390- '\x0256'# -> unI64 393- '\x0257'# -> unI64 394- '\x0259'# -> unI64 399- '\x025b'# -> unI64 400- '\x025c'# -> unI64 42923- '\x0260'# -> unI64 403- '\x0261'# -> unI64 42924- '\x0263'# -> unI64 404- '\x0265'# -> unI64 42893- '\x0266'# -> unI64 42922- '\x0268'# -> unI64 407- '\x0269'# -> unI64 406- '\x026a'# -> unI64 42926- '\x026b'# -> unI64 11362- '\x026c'# -> unI64 42925- '\x026f'# -> unI64 412- '\x0271'# -> unI64 11374- '\x0272'# -> unI64 413- '\x0275'# -> unI64 415- '\x027d'# -> unI64 11364- '\x0280'# -> unI64 422- '\x0282'# -> unI64 42949- '\x0283'# -> unI64 425- '\x0287'# -> unI64 42929- '\x0288'# -> unI64 430- '\x0289'# -> unI64 580- '\x028a'# -> unI64 433- '\x028b'# -> unI64 434- '\x028c'# -> unI64 581- '\x0292'# -> unI64 439- '\x029d'# -> unI64 42930- '\x029e'# -> unI64 42928- '\x0345'# -> unI64 921- '\x0371'# -> unI64 880- '\x0373'# -> unI64 882- '\x0377'# -> unI64 886- '\x037b'# -> unI64 1021- '\x037c'# -> unI64 1022- '\x037d'# -> unI64 1023- '\x03ac'# -> unI64 902- '\x03ad'# -> unI64 904- '\x03ae'# -> unI64 905- '\x03af'# -> unI64 906- '\x03b1'# -> unI64 913- '\x03b2'# -> unI64 914- '\x03b3'# -> unI64 915- '\x03b4'# -> unI64 916- '\x03b5'# -> unI64 917- '\x03b6'# -> unI64 918- '\x03b7'# -> unI64 919- '\x03b8'# -> unI64 920- '\x03b9'# -> unI64 921- '\x03ba'# -> unI64 922- '\x03bb'# -> unI64 923- '\x03bc'# -> unI64 924- '\x03bd'# -> unI64 925- '\x03be'# -> unI64 926- '\x03bf'# -> unI64 927- '\x03c0'# -> unI64 928- '\x03c1'# -> unI64 929- '\x03c2'# -> unI64 931- '\x03c3'# -> unI64 931- '\x03c4'# -> unI64 932- '\x03c5'# -> unI64 933- '\x03c6'# -> unI64 934- '\x03c7'# -> unI64 935- '\x03c8'# -> unI64 936- '\x03c9'# -> unI64 937- '\x03ca'# -> unI64 938- '\x03cb'# -> unI64 939- '\x03cc'# -> unI64 908- '\x03cd'# -> unI64 910- '\x03ce'# -> unI64 911- '\x03d0'# -> unI64 914- '\x03d1'# -> unI64 920- '\x03d5'# -> unI64 934- '\x03d6'# -> unI64 928- '\x03d7'# -> unI64 975- '\x03d9'# -> unI64 984- '\x03db'# -> unI64 986- '\x03dd'# -> unI64 988- '\x03df'# -> unI64 990- '\x03e1'# -> unI64 992- '\x03e3'# -> unI64 994- '\x03e5'# -> unI64 996- '\x03e7'# -> unI64 998- '\x03e9'# -> unI64 1000- '\x03eb'# -> unI64 1002- '\x03ed'# -> unI64 1004- '\x03ef'# -> unI64 1006- '\x03f0'# -> unI64 922- '\x03f1'# -> unI64 929- '\x03f2'# -> unI64 1017- '\x03f3'# -> unI64 895- '\x03f5'# -> unI64 917- '\x03f8'# -> unI64 1015- '\x03fb'# -> unI64 1018- '\x0430'# -> unI64 1040- '\x0431'# -> unI64 1041- '\x0432'# -> unI64 1042- '\x0433'# -> unI64 1043- '\x0434'# -> unI64 1044- '\x0435'# -> unI64 1045- '\x0436'# -> unI64 1046- '\x0437'# -> unI64 1047- '\x0438'# -> unI64 1048- '\x0439'# -> unI64 1049- '\x043a'# -> unI64 1050- '\x043b'# -> unI64 1051- '\x043c'# -> unI64 1052- '\x043d'# -> unI64 1053- '\x043e'# -> unI64 1054- '\x043f'# -> unI64 1055- '\x0440'# -> unI64 1056- '\x0441'# -> unI64 1057- '\x0442'# -> unI64 1058- '\x0443'# -> unI64 1059- '\x0444'# -> unI64 1060- '\x0445'# -> unI64 1061- '\x0446'# -> unI64 1062- '\x0447'# -> unI64 1063- '\x0448'# -> unI64 1064- '\x0449'# -> unI64 1065- '\x044a'# -> unI64 1066- '\x044b'# -> unI64 1067- '\x044c'# -> unI64 1068- '\x044d'# -> unI64 1069- '\x044e'# -> unI64 1070- '\x044f'# -> unI64 1071- '\x0450'# -> unI64 1024- '\x0451'# -> unI64 1025- '\x0452'# -> unI64 1026- '\x0453'# -> unI64 1027- '\x0454'# -> unI64 1028- '\x0455'# -> unI64 1029- '\x0456'# -> unI64 1030- '\x0457'# -> unI64 1031- '\x0458'# -> unI64 1032- '\x0459'# -> unI64 1033- '\x045a'# -> unI64 1034- '\x045b'# -> unI64 1035- '\x045c'# -> unI64 1036- '\x045d'# -> unI64 1037- '\x045e'# -> unI64 1038- '\x045f'# -> unI64 1039- '\x0461'# -> unI64 1120- '\x0463'# -> unI64 1122- '\x0465'# -> unI64 1124- '\x0467'# -> unI64 1126- '\x0469'# -> unI64 1128- '\x046b'# -> unI64 1130- '\x046d'# -> unI64 1132- '\x046f'# -> unI64 1134- '\x0471'# -> unI64 1136- '\x0473'# -> unI64 1138- '\x0475'# -> unI64 1140- '\x0477'# -> unI64 1142- '\x0479'# -> unI64 1144- '\x047b'# -> unI64 1146- '\x047d'# -> unI64 1148- '\x047f'# -> unI64 1150- '\x0481'# -> unI64 1152- '\x048b'# -> unI64 1162- '\x048d'# -> unI64 1164- '\x048f'# -> unI64 1166- '\x0491'# -> unI64 1168- '\x0493'# -> unI64 1170- '\x0495'# -> unI64 1172- '\x0497'# -> unI64 1174- '\x0499'# -> unI64 1176- '\x049b'# -> unI64 1178- '\x049d'# -> unI64 1180- '\x049f'# -> unI64 1182- '\x04a1'# -> unI64 1184- '\x04a3'# -> unI64 1186- '\x04a5'# -> unI64 1188- '\x04a7'# -> unI64 1190- '\x04a9'# -> unI64 1192- '\x04ab'# -> unI64 1194- '\x04ad'# -> unI64 1196- '\x04af'# -> unI64 1198- '\x04b1'# -> unI64 1200- '\x04b3'# -> unI64 1202- '\x04b5'# -> unI64 1204- '\x04b7'# -> unI64 1206- '\x04b9'# -> unI64 1208- '\x04bb'# -> unI64 1210- '\x04bd'# -> unI64 1212- '\x04bf'# -> unI64 1214- '\x04c2'# -> unI64 1217- '\x04c4'# -> unI64 1219- '\x04c6'# -> unI64 1221- '\x04c8'# -> unI64 1223- '\x04ca'# -> unI64 1225- '\x04cc'# -> unI64 1227- '\x04ce'# -> unI64 1229- '\x04cf'# -> unI64 1216- '\x04d1'# -> unI64 1232- '\x04d3'# -> unI64 1234- '\x04d5'# -> unI64 1236- '\x04d7'# -> unI64 1238- '\x04d9'# -> unI64 1240- '\x04db'# -> unI64 1242- '\x04dd'# -> unI64 1244- '\x04df'# -> unI64 1246- '\x04e1'# -> unI64 1248- '\x04e3'# -> unI64 1250- '\x04e5'# -> unI64 1252- '\x04e7'# -> unI64 1254- '\x04e9'# -> unI64 1256- '\x04eb'# -> unI64 1258- '\x04ed'# -> unI64 1260- '\x04ef'# -> unI64 1262- '\x04f1'# -> unI64 1264- '\x04f3'# -> unI64 1266- '\x04f5'# -> unI64 1268- '\x04f7'# -> unI64 1270- '\x04f9'# -> unI64 1272- '\x04fb'# -> unI64 1274- '\x04fd'# -> unI64 1276- '\x04ff'# -> unI64 1278- '\x0501'# -> unI64 1280- '\x0503'# -> unI64 1282- '\x0505'# -> unI64 1284- '\x0507'# -> unI64 1286- '\x0509'# -> unI64 1288- '\x050b'# -> unI64 1290- '\x050d'# -> unI64 1292- '\x050f'# -> unI64 1294- '\x0511'# -> unI64 1296- '\x0513'# -> unI64 1298- '\x0515'# -> unI64 1300- '\x0517'# -> unI64 1302- '\x0519'# -> unI64 1304- '\x051b'# -> unI64 1306- '\x051d'# -> unI64 1308- '\x051f'# -> unI64 1310- '\x0521'# -> unI64 1312- '\x0523'# -> unI64 1314- '\x0525'# -> unI64 1316- '\x0527'# -> unI64 1318- '\x0529'# -> unI64 1320- '\x052b'# -> unI64 1322- '\x052d'# -> unI64 1324- '\x052f'# -> unI64 1326- '\x0561'# -> unI64 1329- '\x0562'# -> unI64 1330- '\x0563'# -> unI64 1331- '\x0564'# -> unI64 1332- '\x0565'# -> unI64 1333- '\x0566'# -> unI64 1334- '\x0567'# -> unI64 1335- '\x0568'# -> unI64 1336- '\x0569'# -> unI64 1337- '\x056a'# -> unI64 1338- '\x056b'# -> unI64 1339- '\x056c'# -> unI64 1340- '\x056d'# -> unI64 1341- '\x056e'# -> unI64 1342- '\x056f'# -> unI64 1343- '\x0570'# -> unI64 1344- '\x0571'# -> unI64 1345- '\x0572'# -> unI64 1346- '\x0573'# -> unI64 1347- '\x0574'# -> unI64 1348- '\x0575'# -> unI64 1349- '\x0576'# -> unI64 1350- '\x0577'# -> unI64 1351- '\x0578'# -> unI64 1352- '\x0579'# -> unI64 1353- '\x057a'# -> unI64 1354- '\x057b'# -> unI64 1355- '\x057c'# -> unI64 1356- '\x057d'# -> unI64 1357- '\x057e'# -> unI64 1358- '\x057f'# -> unI64 1359- '\x0580'# -> unI64 1360- '\x0581'# -> unI64 1361- '\x0582'# -> unI64 1362- '\x0583'# -> unI64 1363- '\x0584'# -> unI64 1364- '\x0585'# -> unI64 1365- '\x0586'# -> unI64 1366- '\x10d0'# -> unI64 7312- '\x10d1'# -> unI64 7313- '\x10d2'# -> unI64 7314- '\x10d3'# -> unI64 7315- '\x10d4'# -> unI64 7316- '\x10d5'# -> unI64 7317- '\x10d6'# -> unI64 7318- '\x10d7'# -> unI64 7319- '\x10d8'# -> unI64 7320- '\x10d9'# -> unI64 7321- '\x10da'# -> unI64 7322- '\x10db'# -> unI64 7323- '\x10dc'# -> unI64 7324- '\x10dd'# -> unI64 7325- '\x10de'# -> unI64 7326- '\x10df'# -> unI64 7327- '\x10e0'# -> unI64 7328- '\x10e1'# -> unI64 7329- '\x10e2'# -> unI64 7330- '\x10e3'# -> unI64 7331- '\x10e4'# -> unI64 7332- '\x10e5'# -> unI64 7333- '\x10e6'# -> unI64 7334- '\x10e7'# -> unI64 7335- '\x10e8'# -> unI64 7336- '\x10e9'# -> unI64 7337- '\x10ea'# -> unI64 7338- '\x10eb'# -> unI64 7339- '\x10ec'# -> unI64 7340- '\x10ed'# -> unI64 7341- '\x10ee'# -> unI64 7342- '\x10ef'# -> unI64 7343- '\x10f0'# -> unI64 7344- '\x10f1'# -> unI64 7345- '\x10f2'# -> unI64 7346- '\x10f3'# -> unI64 7347- '\x10f4'# -> unI64 7348- '\x10f5'# -> unI64 7349- '\x10f6'# -> unI64 7350- '\x10f7'# -> unI64 7351- '\x10f8'# -> unI64 7352- '\x10f9'# -> unI64 7353- '\x10fa'# -> unI64 7354- '\x10fd'# -> unI64 7357- '\x10fe'# -> unI64 7358- '\x10ff'# -> unI64 7359- '\x13f8'# -> unI64 5104- '\x13f9'# -> unI64 5105- '\x13fa'# -> unI64 5106- '\x13fb'# -> unI64 5107- '\x13fc'# -> unI64 5108- '\x13fd'# -> unI64 5109- '\x1c80'# -> unI64 1042- '\x1c81'# -> unI64 1044- '\x1c82'# -> unI64 1054- '\x1c83'# -> unI64 1057- '\x1c84'# -> unI64 1058- '\x1c85'# -> unI64 1058- '\x1c86'# -> unI64 1066- '\x1c87'# -> unI64 1122- '\x1c88'# -> unI64 42570- '\x1d79'# -> unI64 42877- '\x1d7d'# -> unI64 11363- '\x1d8e'# -> unI64 42950- '\x1e01'# -> unI64 7680- '\x1e03'# -> unI64 7682- '\x1e05'# -> unI64 7684- '\x1e07'# -> unI64 7686- '\x1e09'# -> unI64 7688- '\x1e0b'# -> unI64 7690- '\x1e0d'# -> unI64 7692- '\x1e0f'# -> unI64 7694- '\x1e11'# -> unI64 7696- '\x1e13'# -> unI64 7698- '\x1e15'# -> unI64 7700- '\x1e17'# -> unI64 7702- '\x1e19'# -> unI64 7704- '\x1e1b'# -> unI64 7706- '\x1e1d'# -> unI64 7708- '\x1e1f'# -> unI64 7710- '\x1e21'# -> unI64 7712- '\x1e23'# -> unI64 7714- '\x1e25'# -> unI64 7716- '\x1e27'# -> unI64 7718- '\x1e29'# -> unI64 7720- '\x1e2b'# -> unI64 7722- '\x1e2d'# -> unI64 7724- '\x1e2f'# -> unI64 7726- '\x1e31'# -> unI64 7728- '\x1e33'# -> unI64 7730- '\x1e35'# -> unI64 7732- '\x1e37'# -> unI64 7734- '\x1e39'# -> unI64 7736- '\x1e3b'# -> unI64 7738- '\x1e3d'# -> unI64 7740- '\x1e3f'# -> unI64 7742- '\x1e41'# -> unI64 7744- '\x1e43'# -> unI64 7746- '\x1e45'# -> unI64 7748- '\x1e47'# -> unI64 7750- '\x1e49'# -> unI64 7752- '\x1e4b'# -> unI64 7754- '\x1e4d'# -> unI64 7756- '\x1e4f'# -> unI64 7758- '\x1e51'# -> unI64 7760- '\x1e53'# -> unI64 7762- '\x1e55'# -> unI64 7764- '\x1e57'# -> unI64 7766- '\x1e59'# -> unI64 7768- '\x1e5b'# -> unI64 7770- '\x1e5d'# -> unI64 7772- '\x1e5f'# -> unI64 7774- '\x1e61'# -> unI64 7776- '\x1e63'# -> unI64 7778- '\x1e65'# -> unI64 7780- '\x1e67'# -> unI64 7782- '\x1e69'# -> unI64 7784- '\x1e6b'# -> unI64 7786- '\x1e6d'# -> unI64 7788- '\x1e6f'# -> unI64 7790- '\x1e71'# -> unI64 7792- '\x1e73'# -> unI64 7794- '\x1e75'# -> unI64 7796- '\x1e77'# -> unI64 7798- '\x1e79'# -> unI64 7800- '\x1e7b'# -> unI64 7802- '\x1e7d'# -> unI64 7804- '\x1e7f'# -> unI64 7806- '\x1e81'# -> unI64 7808- '\x1e83'# -> unI64 7810- '\x1e85'# -> unI64 7812- '\x1e87'# -> unI64 7814- '\x1e89'# -> unI64 7816- '\x1e8b'# -> unI64 7818- '\x1e8d'# -> unI64 7820- '\x1e8f'# -> unI64 7822- '\x1e91'# -> unI64 7824- '\x1e93'# -> unI64 7826- '\x1e95'# -> unI64 7828- '\x1e9b'# -> unI64 7776- '\x1ea1'# -> unI64 7840- '\x1ea3'# -> unI64 7842- '\x1ea5'# -> unI64 7844- '\x1ea7'# -> unI64 7846- '\x1ea9'# -> unI64 7848- '\x1eab'# -> unI64 7850- '\x1ead'# -> unI64 7852- '\x1eaf'# -> unI64 7854- '\x1eb1'# -> unI64 7856- '\x1eb3'# -> unI64 7858- '\x1eb5'# -> unI64 7860- '\x1eb7'# -> unI64 7862- '\x1eb9'# -> unI64 7864- '\x1ebb'# -> unI64 7866- '\x1ebd'# -> unI64 7868- '\x1ebf'# -> unI64 7870- '\x1ec1'# -> unI64 7872- '\x1ec3'# -> unI64 7874- '\x1ec5'# -> unI64 7876- '\x1ec7'# -> unI64 7878- '\x1ec9'# -> unI64 7880- '\x1ecb'# -> unI64 7882- '\x1ecd'# -> unI64 7884- '\x1ecf'# -> unI64 7886- '\x1ed1'# -> unI64 7888- '\x1ed3'# -> unI64 7890- '\x1ed5'# -> unI64 7892- '\x1ed7'# -> unI64 7894- '\x1ed9'# -> unI64 7896- '\x1edb'# -> unI64 7898- '\x1edd'# -> unI64 7900- '\x1edf'# -> unI64 7902- '\x1ee1'# -> unI64 7904- '\x1ee3'# -> unI64 7906- '\x1ee5'# -> unI64 7908- '\x1ee7'# -> unI64 7910- '\x1ee9'# -> unI64 7912- '\x1eeb'# -> unI64 7914- '\x1eed'# -> unI64 7916- '\x1eef'# -> unI64 7918- '\x1ef1'# -> unI64 7920- '\x1ef3'# -> unI64 7922- '\x1ef5'# -> unI64 7924- '\x1ef7'# -> unI64 7926- '\x1ef9'# -> unI64 7928- '\x1efb'# -> unI64 7930- '\x1efd'# -> unI64 7932- '\x1eff'# -> unI64 7934- '\x1f00'# -> unI64 7944- '\x1f01'# -> unI64 7945- '\x1f02'# -> unI64 7946- '\x1f03'# -> unI64 7947- '\x1f04'# -> unI64 7948- '\x1f05'# -> unI64 7949- '\x1f06'# -> unI64 7950- '\x1f07'# -> unI64 7951- '\x1f10'# -> unI64 7960- '\x1f11'# -> unI64 7961- '\x1f12'# -> unI64 7962- '\x1f13'# -> unI64 7963- '\x1f14'# -> unI64 7964- '\x1f15'# -> unI64 7965- '\x1f20'# -> unI64 7976- '\x1f21'# -> unI64 7977- '\x1f22'# -> unI64 7978- '\x1f23'# -> unI64 7979- '\x1f24'# -> unI64 7980- '\x1f25'# -> unI64 7981- '\x1f26'# -> unI64 7982- '\x1f27'# -> unI64 7983- '\x1f30'# -> unI64 7992- '\x1f31'# -> unI64 7993- '\x1f32'# -> unI64 7994- '\x1f33'# -> unI64 7995- '\x1f34'# -> unI64 7996- '\x1f35'# -> unI64 7997- '\x1f36'# -> unI64 7998- '\x1f37'# -> unI64 7999- '\x1f40'# -> unI64 8008- '\x1f41'# -> unI64 8009- '\x1f42'# -> unI64 8010- '\x1f43'# -> unI64 8011- '\x1f44'# -> unI64 8012- '\x1f45'# -> unI64 8013- '\x1f51'# -> unI64 8025- '\x1f53'# -> unI64 8027- '\x1f55'# -> unI64 8029- '\x1f57'# -> unI64 8031- '\x1f60'# -> unI64 8040- '\x1f61'# -> unI64 8041- '\x1f62'# -> unI64 8042- '\x1f63'# -> unI64 8043- '\x1f64'# -> unI64 8044- '\x1f65'# -> unI64 8045- '\x1f66'# -> unI64 8046- '\x1f67'# -> unI64 8047- '\x1f70'# -> unI64 8122- '\x1f71'# -> unI64 8123- '\x1f72'# -> unI64 8136- '\x1f73'# -> unI64 8137- '\x1f74'# -> unI64 8138- '\x1f75'# -> unI64 8139- '\x1f76'# -> unI64 8154- '\x1f77'# -> unI64 8155- '\x1f78'# -> unI64 8184- '\x1f79'# -> unI64 8185- '\x1f7a'# -> unI64 8170- '\x1f7b'# -> unI64 8171- '\x1f7c'# -> unI64 8186- '\x1f7d'# -> unI64 8187- '\x1fb0'# -> unI64 8120- '\x1fb1'# -> unI64 8121- '\x1fbe'# -> unI64 921- '\x1fd0'# -> unI64 8152- '\x1fd1'# -> unI64 8153- '\x1fe0'# -> unI64 8168- '\x1fe1'# -> unI64 8169- '\x1fe5'# -> unI64 8172- '\x214e'# -> unI64 8498- '\x2170'# -> unI64 8544- '\x2171'# -> unI64 8545- '\x2172'# -> unI64 8546- '\x2173'# -> unI64 8547- '\x2174'# -> unI64 8548- '\x2175'# -> unI64 8549- '\x2176'# -> unI64 8550- '\x2177'# -> unI64 8551- '\x2178'# -> unI64 8552- '\x2179'# -> unI64 8553- '\x217a'# -> unI64 8554- '\x217b'# -> unI64 8555- '\x217c'# -> unI64 8556- '\x217d'# -> unI64 8557- '\x217e'# -> unI64 8558- '\x217f'# -> unI64 8559- '\x2184'# -> unI64 8579- '\x24d0'# -> unI64 9398- '\x24d1'# -> unI64 9399- '\x24d2'# -> unI64 9400- '\x24d3'# -> unI64 9401- '\x24d4'# -> unI64 9402- '\x24d5'# -> unI64 9403- '\x24d6'# -> unI64 9404- '\x24d7'# -> unI64 9405- '\x24d8'# -> unI64 9406- '\x24d9'# -> unI64 9407- '\x24da'# -> unI64 9408- '\x24db'# -> unI64 9409- '\x24dc'# -> unI64 9410- '\x24dd'# -> unI64 9411- '\x24de'# -> unI64 9412- '\x24df'# -> unI64 9413- '\x24e0'# -> unI64 9414- '\x24e1'# -> unI64 9415- '\x24e2'# -> unI64 9416- '\x24e3'# -> unI64 9417- '\x24e4'# -> unI64 9418- '\x24e5'# -> unI64 9419- '\x24e6'# -> unI64 9420- '\x24e7'# -> unI64 9421- '\x24e8'# -> unI64 9422- '\x24e9'# -> unI64 9423- '\x2c30'# -> unI64 11264- '\x2c31'# -> unI64 11265- '\x2c32'# -> unI64 11266- '\x2c33'# -> unI64 11267- '\x2c34'# -> unI64 11268- '\x2c35'# -> unI64 11269- '\x2c36'# -> unI64 11270- '\x2c37'# -> unI64 11271- '\x2c38'# -> unI64 11272- '\x2c39'# -> unI64 11273- '\x2c3a'# -> unI64 11274- '\x2c3b'# -> unI64 11275- '\x2c3c'# -> unI64 11276- '\x2c3d'# -> unI64 11277- '\x2c3e'# -> unI64 11278- '\x2c3f'# -> unI64 11279- '\x2c40'# -> unI64 11280- '\x2c41'# -> unI64 11281- '\x2c42'# -> unI64 11282- '\x2c43'# -> unI64 11283- '\x2c44'# -> unI64 11284- '\x2c45'# -> unI64 11285- '\x2c46'# -> unI64 11286- '\x2c47'# -> unI64 11287- '\x2c48'# -> unI64 11288- '\x2c49'# -> unI64 11289- '\x2c4a'# -> unI64 11290- '\x2c4b'# -> unI64 11291- '\x2c4c'# -> unI64 11292- '\x2c4d'# -> unI64 11293- '\x2c4e'# -> unI64 11294- '\x2c4f'# -> unI64 11295- '\x2c50'# -> unI64 11296- '\x2c51'# -> unI64 11297- '\x2c52'# -> unI64 11298- '\x2c53'# -> unI64 11299- '\x2c54'# -> unI64 11300- '\x2c55'# -> unI64 11301- '\x2c56'# -> unI64 11302- '\x2c57'# -> unI64 11303- '\x2c58'# -> unI64 11304- '\x2c59'# -> unI64 11305- '\x2c5a'# -> unI64 11306- '\x2c5b'# -> unI64 11307- '\x2c5c'# -> unI64 11308- '\x2c5d'# -> unI64 11309- '\x2c5e'# -> unI64 11310- '\x2c5f'# -> unI64 11311- '\x2c61'# -> unI64 11360- '\x2c65'# -> unI64 570- '\x2c66'# -> unI64 574- '\x2c68'# -> unI64 11367- '\x2c6a'# -> unI64 11369- '\x2c6c'# -> unI64 11371- '\x2c73'# -> unI64 11378- '\x2c76'# -> unI64 11381- '\x2c81'# -> unI64 11392- '\x2c83'# -> unI64 11394- '\x2c85'# -> unI64 11396- '\x2c87'# -> unI64 11398- '\x2c89'# -> unI64 11400- '\x2c8b'# -> unI64 11402- '\x2c8d'# -> unI64 11404- '\x2c8f'# -> unI64 11406- '\x2c91'# -> unI64 11408- '\x2c93'# -> unI64 11410- '\x2c95'# -> unI64 11412- '\x2c97'# -> unI64 11414- '\x2c99'# -> unI64 11416- '\x2c9b'# -> unI64 11418- '\x2c9d'# -> unI64 11420- '\x2c9f'# -> unI64 11422- '\x2ca1'# -> unI64 11424- '\x2ca3'# -> unI64 11426- '\x2ca5'# -> unI64 11428- '\x2ca7'# -> unI64 11430- '\x2ca9'# -> unI64 11432- '\x2cab'# -> unI64 11434- '\x2cad'# -> unI64 11436- '\x2caf'# -> unI64 11438- '\x2cb1'# -> unI64 11440- '\x2cb3'# -> unI64 11442- '\x2cb5'# -> unI64 11444- '\x2cb7'# -> unI64 11446- '\x2cb9'# -> unI64 11448- '\x2cbb'# -> unI64 11450- '\x2cbd'# -> unI64 11452- '\x2cbf'# -> unI64 11454- '\x2cc1'# -> unI64 11456- '\x2cc3'# -> unI64 11458- '\x2cc5'# -> unI64 11460- '\x2cc7'# -> unI64 11462- '\x2cc9'# -> unI64 11464- '\x2ccb'# -> unI64 11466- '\x2ccd'# -> unI64 11468- '\x2ccf'# -> unI64 11470- '\x2cd1'# -> unI64 11472- '\x2cd3'# -> unI64 11474- '\x2cd5'# -> unI64 11476- '\x2cd7'# -> unI64 11478- '\x2cd9'# -> unI64 11480- '\x2cdb'# -> unI64 11482- '\x2cdd'# -> unI64 11484- '\x2cdf'# -> unI64 11486- '\x2ce1'# -> unI64 11488- '\x2ce3'# -> unI64 11490- '\x2cec'# -> unI64 11499- '\x2cee'# -> unI64 11501- '\x2cf3'# -> unI64 11506- '\x2d00'# -> unI64 4256- '\x2d01'# -> unI64 4257- '\x2d02'# -> unI64 4258- '\x2d03'# -> unI64 4259- '\x2d04'# -> unI64 4260- '\x2d05'# -> unI64 4261- '\x2d06'# -> unI64 4262- '\x2d07'# -> unI64 4263- '\x2d08'# -> unI64 4264- '\x2d09'# -> unI64 4265- '\x2d0a'# -> unI64 4266- '\x2d0b'# -> unI64 4267- '\x2d0c'# -> unI64 4268- '\x2d0d'# -> unI64 4269- '\x2d0e'# -> unI64 4270- '\x2d0f'# -> unI64 4271- '\x2d10'# -> unI64 4272- '\x2d11'# -> unI64 4273- '\x2d12'# -> unI64 4274- '\x2d13'# -> unI64 4275- '\x2d14'# -> unI64 4276- '\x2d15'# -> unI64 4277- '\x2d16'# -> unI64 4278- '\x2d17'# -> unI64 4279- '\x2d18'# -> unI64 4280- '\x2d19'# -> unI64 4281- '\x2d1a'# -> unI64 4282- '\x2d1b'# -> unI64 4283- '\x2d1c'# -> unI64 4284- '\x2d1d'# -> unI64 4285- '\x2d1e'# -> unI64 4286- '\x2d1f'# -> unI64 4287- '\x2d20'# -> unI64 4288- '\x2d21'# -> unI64 4289- '\x2d22'# -> unI64 4290- '\x2d23'# -> unI64 4291- '\x2d24'# -> unI64 4292- '\x2d25'# -> unI64 4293- '\x2d27'# -> unI64 4295- '\x2d2d'# -> unI64 4301- '\xa641'# -> unI64 42560- '\xa643'# -> unI64 42562- '\xa645'# -> unI64 42564- '\xa647'# -> unI64 42566- '\xa649'# -> unI64 42568- '\xa64b'# -> unI64 42570- '\xa64d'# -> unI64 42572- '\xa64f'# -> unI64 42574- '\xa651'# -> unI64 42576- '\xa653'# -> unI64 42578- '\xa655'# -> unI64 42580- '\xa657'# -> unI64 42582- '\xa659'# -> unI64 42584- '\xa65b'# -> unI64 42586- '\xa65d'# -> unI64 42588- '\xa65f'# -> unI64 42590- '\xa661'# -> unI64 42592- '\xa663'# -> unI64 42594- '\xa665'# -> unI64 42596- '\xa667'# -> unI64 42598- '\xa669'# -> unI64 42600- '\xa66b'# -> unI64 42602- '\xa66d'# -> unI64 42604- '\xa681'# -> unI64 42624- '\xa683'# -> unI64 42626- '\xa685'# -> unI64 42628- '\xa687'# -> unI64 42630- '\xa689'# -> unI64 42632- '\xa68b'# -> unI64 42634- '\xa68d'# -> unI64 42636- '\xa68f'# -> unI64 42638- '\xa691'# -> unI64 42640- '\xa693'# -> unI64 42642- '\xa695'# -> unI64 42644- '\xa697'# -> unI64 42646- '\xa699'# -> unI64 42648- '\xa69b'# -> unI64 42650- '\xa723'# -> unI64 42786- '\xa725'# -> unI64 42788- '\xa727'# -> unI64 42790- '\xa729'# -> unI64 42792- '\xa72b'# -> unI64 42794- '\xa72d'# -> unI64 42796- '\xa72f'# -> unI64 42798- '\xa733'# -> unI64 42802- '\xa735'# -> unI64 42804- '\xa737'# -> unI64 42806- '\xa739'# -> unI64 42808- '\xa73b'# -> unI64 42810- '\xa73d'# -> unI64 42812- '\xa73f'# -> unI64 42814- '\xa741'# -> unI64 42816- '\xa743'# -> unI64 42818- '\xa745'# -> unI64 42820- '\xa747'# -> unI64 42822- '\xa749'# -> unI64 42824- '\xa74b'# -> unI64 42826- '\xa74d'# -> unI64 42828- '\xa74f'# -> unI64 42830- '\xa751'# -> unI64 42832- '\xa753'# -> unI64 42834- '\xa755'# -> unI64 42836- '\xa757'# -> unI64 42838- '\xa759'# -> unI64 42840- '\xa75b'# -> unI64 42842- '\xa75d'# -> unI64 42844- '\xa75f'# -> unI64 42846- '\xa761'# -> unI64 42848- '\xa763'# -> unI64 42850- '\xa765'# -> unI64 42852- '\xa767'# -> unI64 42854- '\xa769'# -> unI64 42856- '\xa76b'# -> unI64 42858- '\xa76d'# -> unI64 42860- '\xa76f'# -> unI64 42862- '\xa77a'# -> unI64 42873- '\xa77c'# -> unI64 42875- '\xa77f'# -> unI64 42878- '\xa781'# -> unI64 42880- '\xa783'# -> unI64 42882- '\xa785'# -> unI64 42884- '\xa787'# -> unI64 42886- '\xa78c'# -> unI64 42891- '\xa791'# -> unI64 42896- '\xa793'# -> unI64 42898- '\xa794'# -> unI64 42948- '\xa797'# -> unI64 42902- '\xa799'# -> unI64 42904- '\xa79b'# -> unI64 42906- '\xa79d'# -> unI64 42908- '\xa79f'# -> unI64 42910- '\xa7a1'# -> unI64 42912- '\xa7a3'# -> unI64 42914- '\xa7a5'# -> unI64 42916- '\xa7a7'# -> unI64 42918- '\xa7a9'# -> unI64 42920- '\xa7b5'# -> unI64 42932- '\xa7b7'# -> unI64 42934- '\xa7b9'# -> unI64 42936- '\xa7bb'# -> unI64 42938- '\xa7bd'# -> unI64 42940- '\xa7bf'# -> unI64 42942- '\xa7c1'# -> unI64 42944- '\xa7c3'# -> unI64 42946- '\xa7c8'# -> unI64 42951- '\xa7ca'# -> unI64 42953- '\xa7d1'# -> unI64 42960- '\xa7d7'# -> unI64 42966- '\xa7d9'# -> unI64 42968- '\xa7f6'# -> unI64 42997- '\xab53'# -> unI64 42931- '\xab70'# -> unI64 5024- '\xab71'# -> unI64 5025- '\xab72'# -> unI64 5026- '\xab73'# -> unI64 5027- '\xab74'# -> unI64 5028- '\xab75'# -> unI64 5029- '\xab76'# -> unI64 5030- '\xab77'# -> unI64 5031- '\xab78'# -> unI64 5032- '\xab79'# -> unI64 5033- '\xab7a'# -> unI64 5034- '\xab7b'# -> unI64 5035- '\xab7c'# -> unI64 5036- '\xab7d'# -> unI64 5037- '\xab7e'# -> unI64 5038- '\xab7f'# -> unI64 5039- '\xab80'# -> unI64 5040- '\xab81'# -> unI64 5041- '\xab82'# -> unI64 5042- '\xab83'# -> unI64 5043- '\xab84'# -> unI64 5044- '\xab85'# -> unI64 5045- '\xab86'# -> unI64 5046- '\xab87'# -> unI64 5047- '\xab88'# -> unI64 5048- '\xab89'# -> unI64 5049- '\xab8a'# -> unI64 5050- '\xab8b'# -> unI64 5051- '\xab8c'# -> unI64 5052- '\xab8d'# -> unI64 5053- '\xab8e'# -> unI64 5054- '\xab8f'# -> unI64 5055- '\xab90'# -> unI64 5056- '\xab91'# -> unI64 5057- '\xab92'# -> unI64 5058- '\xab93'# -> unI64 5059- '\xab94'# -> unI64 5060- '\xab95'# -> unI64 5061- '\xab96'# -> unI64 5062- '\xab97'# -> unI64 5063- '\xab98'# -> unI64 5064- '\xab99'# -> unI64 5065- '\xab9a'# -> unI64 5066- '\xab9b'# -> unI64 5067- '\xab9c'# -> unI64 5068- '\xab9d'# -> unI64 5069- '\xab9e'# -> unI64 5070- '\xab9f'# -> unI64 5071- '\xaba0'# -> unI64 5072- '\xaba1'# -> unI64 5073- '\xaba2'# -> unI64 5074- '\xaba3'# -> unI64 5075- '\xaba4'# -> unI64 5076- '\xaba5'# -> unI64 5077- '\xaba6'# -> unI64 5078- '\xaba7'# -> unI64 5079- '\xaba8'# -> unI64 5080- '\xaba9'# -> unI64 5081- '\xabaa'# -> unI64 5082- '\xabab'# -> unI64 5083- '\xabac'# -> unI64 5084- '\xabad'# -> unI64 5085- '\xabae'# -> unI64 5086- '\xabaf'# -> unI64 5087- '\xabb0'# -> unI64 5088- '\xabb1'# -> unI64 5089- '\xabb2'# -> unI64 5090- '\xabb3'# -> unI64 5091- '\xabb4'# -> unI64 5092- '\xabb5'# -> unI64 5093- '\xabb6'# -> unI64 5094- '\xabb7'# -> unI64 5095- '\xabb8'# -> unI64 5096- '\xabb9'# -> unI64 5097- '\xabba'# -> unI64 5098- '\xabbb'# -> unI64 5099- '\xabbc'# -> unI64 5100- '\xabbd'# -> unI64 5101- '\xabbe'# -> unI64 5102- '\xabbf'# -> unI64 5103- '\xff41'# -> unI64 65313- '\xff42'# -> unI64 65314- '\xff43'# -> unI64 65315- '\xff44'# -> unI64 65316- '\xff45'# -> unI64 65317- '\xff46'# -> unI64 65318- '\xff47'# -> unI64 65319- '\xff48'# -> unI64 65320- '\xff49'# -> unI64 65321- '\xff4a'# -> unI64 65322- '\xff4b'# -> unI64 65323- '\xff4c'# -> unI64 65324- '\xff4d'# -> unI64 65325- '\xff4e'# -> unI64 65326- '\xff4f'# -> unI64 65327- '\xff50'# -> unI64 65328- '\xff51'# -> unI64 65329- '\xff52'# -> unI64 65330- '\xff53'# -> unI64 65331- '\xff54'# -> unI64 65332- '\xff55'# -> unI64 65333- '\xff56'# -> unI64 65334- '\xff57'# -> unI64 65335- '\xff58'# -> unI64 65336- '\xff59'# -> unI64 65337- '\xff5a'# -> unI64 65338- '\x10428'# -> unI64 66560- '\x10429'# -> unI64 66561- '\x1042a'# -> unI64 66562- '\x1042b'# -> unI64 66563- '\x1042c'# -> unI64 66564- '\x1042d'# -> unI64 66565- '\x1042e'# -> unI64 66566- '\x1042f'# -> unI64 66567- '\x10430'# -> unI64 66568- '\x10431'# -> unI64 66569- '\x10432'# -> unI64 66570- '\x10433'# -> unI64 66571- '\x10434'# -> unI64 66572- '\x10435'# -> unI64 66573- '\x10436'# -> unI64 66574- '\x10437'# -> unI64 66575- '\x10438'# -> unI64 66576- '\x10439'# -> unI64 66577- '\x1043a'# -> unI64 66578- '\x1043b'# -> unI64 66579- '\x1043c'# -> unI64 66580- '\x1043d'# -> unI64 66581- '\x1043e'# -> unI64 66582- '\x1043f'# -> unI64 66583- '\x10440'# -> unI64 66584- '\x10441'# -> unI64 66585- '\x10442'# -> unI64 66586- '\x10443'# -> unI64 66587- '\x10444'# -> unI64 66588- '\x10445'# -> unI64 66589- '\x10446'# -> unI64 66590- '\x10447'# -> unI64 66591- '\x10448'# -> unI64 66592- '\x10449'# -> unI64 66593- '\x1044a'# -> unI64 66594- '\x1044b'# -> unI64 66595- '\x1044c'# -> unI64 66596- '\x1044d'# -> unI64 66597- '\x1044e'# -> unI64 66598- '\x1044f'# -> unI64 66599- '\x104d8'# -> unI64 66736- '\x104d9'# -> unI64 66737- '\x104da'# -> unI64 66738- '\x104db'# -> unI64 66739- '\x104dc'# -> unI64 66740- '\x104dd'# -> unI64 66741- '\x104de'# -> unI64 66742- '\x104df'# -> unI64 66743- '\x104e0'# -> unI64 66744- '\x104e1'# -> unI64 66745- '\x104e2'# -> unI64 66746- '\x104e3'# -> unI64 66747- '\x104e4'# -> unI64 66748- '\x104e5'# -> unI64 66749- '\x104e6'# -> unI64 66750- '\x104e7'# -> unI64 66751- '\x104e8'# -> unI64 66752- '\x104e9'# -> unI64 66753- '\x104ea'# -> unI64 66754- '\x104eb'# -> unI64 66755- '\x104ec'# -> unI64 66756- '\x104ed'# -> unI64 66757- '\x104ee'# -> unI64 66758- '\x104ef'# -> unI64 66759- '\x104f0'# -> unI64 66760- '\x104f1'# -> unI64 66761- '\x104f2'# -> unI64 66762- '\x104f3'# -> unI64 66763- '\x104f4'# -> unI64 66764- '\x104f5'# -> unI64 66765- '\x104f6'# -> unI64 66766- '\x104f7'# -> unI64 66767- '\x104f8'# -> unI64 66768- '\x104f9'# -> unI64 66769- '\x104fa'# -> unI64 66770- '\x104fb'# -> unI64 66771- '\x10597'# -> unI64 66928- '\x10598'# -> unI64 66929- '\x10599'# -> unI64 66930- '\x1059a'# -> unI64 66931- '\x1059b'# -> unI64 66932- '\x1059c'# -> unI64 66933- '\x1059d'# -> unI64 66934- '\x1059e'# -> unI64 66935- '\x1059f'# -> unI64 66936- '\x105a0'# -> unI64 66937- '\x105a1'# -> unI64 66938- '\x105a3'# -> unI64 66940- '\x105a4'# -> unI64 66941- '\x105a5'# -> unI64 66942- '\x105a6'# -> unI64 66943- '\x105a7'# -> unI64 66944- '\x105a8'# -> unI64 66945- '\x105a9'# -> unI64 66946- '\x105aa'# -> unI64 66947- '\x105ab'# -> unI64 66948- '\x105ac'# -> unI64 66949- '\x105ad'# -> unI64 66950- '\x105ae'# -> unI64 66951- '\x105af'# -> unI64 66952- '\x105b0'# -> unI64 66953- '\x105b1'# -> unI64 66954- '\x105b3'# -> unI64 66956- '\x105b4'# -> unI64 66957- '\x105b5'# -> unI64 66958- '\x105b6'# -> unI64 66959- '\x105b7'# -> unI64 66960- '\x105b8'# -> unI64 66961- '\x105b9'# -> unI64 66962- '\x105bb'# -> unI64 66964- '\x105bc'# -> unI64 66965- '\x10cc0'# -> unI64 68736- '\x10cc1'# -> unI64 68737- '\x10cc2'# -> unI64 68738- '\x10cc3'# -> unI64 68739- '\x10cc4'# -> unI64 68740- '\x10cc5'# -> unI64 68741- '\x10cc6'# -> unI64 68742- '\x10cc7'# -> unI64 68743- '\x10cc8'# -> unI64 68744- '\x10cc9'# -> unI64 68745- '\x10cca'# -> unI64 68746- '\x10ccb'# -> unI64 68747- '\x10ccc'# -> unI64 68748- '\x10ccd'# -> unI64 68749- '\x10cce'# -> unI64 68750- '\x10ccf'# -> unI64 68751- '\x10cd0'# -> unI64 68752- '\x10cd1'# -> unI64 68753- '\x10cd2'# -> unI64 68754- '\x10cd3'# -> unI64 68755- '\x10cd4'# -> unI64 68756- '\x10cd5'# -> unI64 68757- '\x10cd6'# -> unI64 68758- '\x10cd7'# -> unI64 68759- '\x10cd8'# -> unI64 68760- '\x10cd9'# -> unI64 68761- '\x10cda'# -> unI64 68762- '\x10cdb'# -> unI64 68763- '\x10cdc'# -> unI64 68764- '\x10cdd'# -> unI64 68765- '\x10cde'# -> unI64 68766- '\x10cdf'# -> unI64 68767- '\x10ce0'# -> unI64 68768- '\x10ce1'# -> unI64 68769- '\x10ce2'# -> unI64 68770- '\x10ce3'# -> unI64 68771- '\x10ce4'# -> unI64 68772- '\x10ce5'# -> unI64 68773- '\x10ce6'# -> unI64 68774- '\x10ce7'# -> unI64 68775- '\x10ce8'# -> unI64 68776- '\x10ce9'# -> unI64 68777- '\x10cea'# -> unI64 68778- '\x10ceb'# -> unI64 68779- '\x10cec'# -> unI64 68780- '\x10ced'# -> unI64 68781- '\x10cee'# -> unI64 68782- '\x10cef'# -> unI64 68783- '\x10cf0'# -> unI64 68784- '\x10cf1'# -> unI64 68785- '\x10cf2'# -> unI64 68786- '\x118c0'# -> unI64 71840- '\x118c1'# -> unI64 71841- '\x118c2'# -> unI64 71842- '\x118c3'# -> unI64 71843- '\x118c4'# -> unI64 71844- '\x118c5'# -> unI64 71845- '\x118c6'# -> unI64 71846- '\x118c7'# -> unI64 71847- '\x118c8'# -> unI64 71848- '\x118c9'# -> unI64 71849- '\x118ca'# -> unI64 71850- '\x118cb'# -> unI64 71851- '\x118cc'# -> unI64 71852- '\x118cd'# -> unI64 71853- '\x118ce'# -> unI64 71854- '\x118cf'# -> unI64 71855- '\x118d0'# -> unI64 71856- '\x118d1'# -> unI64 71857- '\x118d2'# -> unI64 71858- '\x118d3'# -> unI64 71859- '\x118d4'# -> unI64 71860- '\x118d5'# -> unI64 71861- '\x118d6'# -> unI64 71862- '\x118d7'# -> unI64 71863- '\x118d8'# -> unI64 71864- '\x118d9'# -> unI64 71865- '\x118da'# -> unI64 71866- '\x118db'# -> unI64 71867- '\x118dc'# -> unI64 71868- '\x118dd'# -> unI64 71869- '\x118de'# -> unI64 71870- '\x118df'# -> unI64 71871- '\x16e60'# -> unI64 93760- '\x16e61'# -> unI64 93761- '\x16e62'# -> unI64 93762- '\x16e63'# -> unI64 93763- '\x16e64'# -> unI64 93764- '\x16e65'# -> unI64 93765- '\x16e66'# -> unI64 93766- '\x16e67'# -> unI64 93767- '\x16e68'# -> unI64 93768- '\x16e69'# -> unI64 93769- '\x16e6a'# -> unI64 93770- '\x16e6b'# -> unI64 93771- '\x16e6c'# -> unI64 93772- '\x16e6d'# -> unI64 93773- '\x16e6e'# -> unI64 93774- '\x16e6f'# -> unI64 93775- '\x16e70'# -> unI64 93776- '\x16e71'# -> unI64 93777- '\x16e72'# -> unI64 93778- '\x16e73'# -> unI64 93779- '\x16e74'# -> unI64 93780- '\x16e75'# -> unI64 93781- '\x16e76'# -> unI64 93782- '\x16e77'# -> unI64 93783- '\x16e78'# -> unI64 93784- '\x16e79'# -> unI64 93785- '\x16e7a'# -> unI64 93786- '\x16e7b'# -> unI64 93787- '\x16e7c'# -> unI64 93788- '\x16e7d'# -> unI64 93789- '\x16e7e'# -> unI64 93790- '\x16e7f'# -> unI64 93791- '\x1e922'# -> unI64 125184- '\x1e923'# -> unI64 125185- '\x1e924'# -> unI64 125186- '\x1e925'# -> unI64 125187- '\x1e926'# -> unI64 125188- '\x1e927'# -> unI64 125189- '\x1e928'# -> unI64 125190- '\x1e929'# -> unI64 125191- '\x1e92a'# -> unI64 125192- '\x1e92b'# -> unI64 125193- '\x1e92c'# -> unI64 125194- '\x1e92d'# -> unI64 125195- '\x1e92e'# -> unI64 125196- '\x1e92f'# -> unI64 125197- '\x1e930'# -> unI64 125198- '\x1e931'# -> unI64 125199- '\x1e932'# -> unI64 125200- '\x1e933'# -> unI64 125201- '\x1e934'# -> unI64 125202- '\x1e935'# -> unI64 125203- '\x1e936'# -> unI64 125204- '\x1e937'# -> unI64 125205- '\x1e938'# -> unI64 125206- '\x1e939'# -> unI64 125207- '\x1e93a'# -> unI64 125208- '\x1e93b'# -> unI64 125209- '\x1e93c'# -> unI64 125210- '\x1e93d'# -> unI64 125211- '\x1e93e'# -> unI64 125212- '\x1e93f'# -> unI64 125213- '\x1e940'# -> unI64 125214- '\x1e941'# -> unI64 125215- '\x1e942'# -> unI64 125216- '\x1e943'# -> unI64 125217- _ -> unI64 0-lowerMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE lowerMapping #-}-lowerMapping = \case- -- LATIN CAPITAL LETTER I WITH DOT ABOVE- '\x0130'# -> unI64 1625292905- '\x0041'# -> unI64 97- '\x0042'# -> unI64 98- '\x0043'# -> unI64 99- '\x0044'# -> unI64 100- '\x0045'# -> unI64 101- '\x0046'# -> unI64 102- '\x0047'# -> unI64 103- '\x0048'# -> unI64 104- '\x0049'# -> unI64 105- '\x004a'# -> unI64 106- '\x004b'# -> unI64 107- '\x004c'# -> unI64 108- '\x004d'# -> unI64 109- '\x004e'# -> unI64 110- '\x004f'# -> unI64 111- '\x0050'# -> unI64 112- '\x0051'# -> unI64 113- '\x0052'# -> unI64 114- '\x0053'# -> unI64 115- '\x0054'# -> unI64 116- '\x0055'# -> unI64 117- '\x0056'# -> unI64 118- '\x0057'# -> unI64 119- '\x0058'# -> unI64 120- '\x0059'# -> unI64 121- '\x005a'# -> unI64 122- '\x00c0'# -> unI64 224- '\x00c1'# -> unI64 225- '\x00c2'# -> unI64 226- '\x00c3'# -> unI64 227- '\x00c4'# -> unI64 228- '\x00c5'# -> unI64 229- '\x00c6'# -> unI64 230- '\x00c7'# -> unI64 231- '\x00c8'# -> unI64 232- '\x00c9'# -> unI64 233- '\x00ca'# -> unI64 234- '\x00cb'# -> unI64 235- '\x00cc'# -> unI64 236- '\x00cd'# -> unI64 237- '\x00ce'# -> unI64 238- '\x00cf'# -> unI64 239- '\x00d0'# -> unI64 240- '\x00d1'# -> unI64 241- '\x00d2'# -> unI64 242- '\x00d3'# -> unI64 243- '\x00d4'# -> unI64 244- '\x00d5'# -> unI64 245- '\x00d6'# -> unI64 246- '\x00d8'# -> unI64 248- '\x00d9'# -> unI64 249- '\x00da'# -> unI64 250- '\x00db'# -> unI64 251- '\x00dc'# -> unI64 252- '\x00dd'# -> unI64 253- '\x00de'# -> unI64 254- '\x0100'# -> unI64 257- '\x0102'# -> unI64 259- '\x0104'# -> unI64 261- '\x0106'# -> unI64 263- '\x0108'# -> unI64 265- '\x010a'# -> unI64 267- '\x010c'# -> unI64 269- '\x010e'# -> unI64 271- '\x0110'# -> unI64 273- '\x0112'# -> unI64 275- '\x0114'# -> unI64 277- '\x0116'# -> unI64 279- '\x0118'# -> unI64 281- '\x011a'# -> unI64 283- '\x011c'# -> unI64 285- '\x011e'# -> unI64 287- '\x0120'# -> unI64 289- '\x0122'# -> unI64 291- '\x0124'# -> unI64 293- '\x0126'# -> unI64 295- '\x0128'# -> unI64 297- '\x012a'# -> unI64 299- '\x012c'# -> unI64 301- '\x012e'# -> unI64 303- '\x0132'# -> unI64 307- '\x0134'# -> unI64 309- '\x0136'# -> unI64 311- '\x0139'# -> unI64 314- '\x013b'# -> unI64 316- '\x013d'# -> unI64 318- '\x013f'# -> unI64 320- '\x0141'# -> unI64 322- '\x0143'# -> unI64 324- '\x0145'# -> unI64 326- '\x0147'# -> unI64 328- '\x014a'# -> unI64 331- '\x014c'# -> unI64 333- '\x014e'# -> unI64 335- '\x0150'# -> unI64 337- '\x0152'# -> unI64 339- '\x0154'# -> unI64 341- '\x0156'# -> unI64 343- '\x0158'# -> unI64 345- '\x015a'# -> unI64 347- '\x015c'# -> unI64 349- '\x015e'# -> unI64 351- '\x0160'# -> unI64 353- '\x0162'# -> unI64 355- '\x0164'# -> unI64 357- '\x0166'# -> unI64 359- '\x0168'# -> unI64 361- '\x016a'# -> unI64 363- '\x016c'# -> unI64 365- '\x016e'# -> unI64 367- '\x0170'# -> unI64 369- '\x0172'# -> unI64 371- '\x0174'# -> unI64 373- '\x0176'# -> unI64 375- '\x0178'# -> unI64 255- '\x0179'# -> unI64 378- '\x017b'# -> unI64 380- '\x017d'# -> unI64 382- '\x0181'# -> unI64 595- '\x0182'# -> unI64 387- '\x0184'# -> unI64 389- '\x0186'# -> unI64 596- '\x0187'# -> unI64 392- '\x0189'# -> unI64 598- '\x018a'# -> unI64 599- '\x018b'# -> unI64 396- '\x018e'# -> unI64 477- '\x018f'# -> unI64 601- '\x0190'# -> unI64 603- '\x0191'# -> unI64 402- '\x0193'# -> unI64 608- '\x0194'# -> unI64 611- '\x0196'# -> unI64 617- '\x0197'# -> unI64 616- '\x0198'# -> unI64 409- '\x019c'# -> unI64 623- '\x019d'# -> unI64 626- '\x019f'# -> unI64 629- '\x01a0'# -> unI64 417- '\x01a2'# -> unI64 419- '\x01a4'# -> unI64 421- '\x01a6'# -> unI64 640- '\x01a7'# -> unI64 424- '\x01a9'# -> unI64 643- '\x01ac'# -> unI64 429- '\x01ae'# -> unI64 648- '\x01af'# -> unI64 432- '\x01b1'# -> unI64 650- '\x01b2'# -> unI64 651- '\x01b3'# -> unI64 436- '\x01b5'# -> unI64 438- '\x01b7'# -> unI64 658- '\x01b8'# -> unI64 441- '\x01bc'# -> unI64 445- '\x01c4'# -> unI64 454- '\x01c5'# -> unI64 454- '\x01c7'# -> unI64 457- '\x01c8'# -> unI64 457- '\x01ca'# -> unI64 460- '\x01cb'# -> unI64 460- '\x01cd'# -> unI64 462- '\x01cf'# -> unI64 464- '\x01d1'# -> unI64 466- '\x01d3'# -> unI64 468- '\x01d5'# -> unI64 470- '\x01d7'# -> unI64 472- '\x01d9'# -> unI64 474- '\x01db'# -> unI64 476- '\x01de'# -> unI64 479- '\x01e0'# -> unI64 481- '\x01e2'# -> unI64 483- '\x01e4'# -> unI64 485- '\x01e6'# -> unI64 487- '\x01e8'# -> unI64 489- '\x01ea'# -> unI64 491- '\x01ec'# -> unI64 493- '\x01ee'# -> unI64 495- '\x01f1'# -> unI64 499- '\x01f2'# -> unI64 499- '\x01f4'# -> unI64 501- '\x01f6'# -> unI64 405- '\x01f7'# -> unI64 447- '\x01f8'# -> unI64 505- '\x01fa'# -> unI64 507- '\x01fc'# -> unI64 509- '\x01fe'# -> unI64 511- '\x0200'# -> unI64 513- '\x0202'# -> unI64 515- '\x0204'# -> unI64 517- '\x0206'# -> unI64 519- '\x0208'# -> unI64 521- '\x020a'# -> unI64 523- '\x020c'# -> unI64 525- '\x020e'# -> unI64 527- '\x0210'# -> unI64 529- '\x0212'# -> unI64 531- '\x0214'# -> unI64 533- '\x0216'# -> unI64 535- '\x0218'# -> unI64 537- '\x021a'# -> unI64 539- '\x021c'# -> unI64 541- '\x021e'# -> unI64 543- '\x0220'# -> unI64 414- '\x0222'# -> unI64 547- '\x0224'# -> unI64 549- '\x0226'# -> unI64 551- '\x0228'# -> unI64 553- '\x022a'# -> unI64 555- '\x022c'# -> unI64 557- '\x022e'# -> unI64 559- '\x0230'# -> unI64 561- '\x0232'# -> unI64 563- '\x023a'# -> unI64 11365- '\x023b'# -> unI64 572- '\x023d'# -> unI64 410- '\x023e'# -> unI64 11366- '\x0241'# -> unI64 578- '\x0243'# -> unI64 384- '\x0244'# -> unI64 649- '\x0245'# -> unI64 652- '\x0246'# -> unI64 583- '\x0248'# -> unI64 585- '\x024a'# -> unI64 587- '\x024c'# -> unI64 589- '\x024e'# -> unI64 591- '\x0370'# -> unI64 881- '\x0372'# -> unI64 883- '\x0376'# -> unI64 887- '\x037f'# -> unI64 1011- '\x0386'# -> unI64 940- '\x0388'# -> unI64 941- '\x0389'# -> unI64 942- '\x038a'# -> unI64 943- '\x038c'# -> unI64 972- '\x038e'# -> unI64 973- '\x038f'# -> unI64 974- '\x0391'# -> unI64 945- '\x0392'# -> unI64 946- '\x0393'# -> unI64 947- '\x0394'# -> unI64 948- '\x0395'# -> unI64 949- '\x0396'# -> unI64 950- '\x0397'# -> unI64 951- '\x0398'# -> unI64 952- '\x0399'# -> unI64 953- '\x039a'# -> unI64 954- '\x039b'# -> unI64 955- '\x039c'# -> unI64 956- '\x039d'# -> unI64 957- '\x039e'# -> unI64 958- '\x039f'# -> unI64 959- '\x03a0'# -> unI64 960- '\x03a1'# -> unI64 961- '\x03a3'# -> unI64 963- '\x03a4'# -> unI64 964- '\x03a5'# -> unI64 965- '\x03a6'# -> unI64 966- '\x03a7'# -> unI64 967- '\x03a8'# -> unI64 968- '\x03a9'# -> unI64 969- '\x03aa'# -> unI64 970- '\x03ab'# -> unI64 971- '\x03cf'# -> unI64 983- '\x03d8'# -> unI64 985- '\x03da'# -> unI64 987- '\x03dc'# -> unI64 989- '\x03de'# -> unI64 991- '\x03e0'# -> unI64 993- '\x03e2'# -> unI64 995- '\x03e4'# -> unI64 997- '\x03e6'# -> unI64 999- '\x03e8'# -> unI64 1001- '\x03ea'# -> unI64 1003- '\x03ec'# -> unI64 1005- '\x03ee'# -> unI64 1007- '\x03f4'# -> unI64 952- '\x03f7'# -> unI64 1016- '\x03f9'# -> unI64 1010- '\x03fa'# -> unI64 1019- '\x03fd'# -> unI64 891- '\x03fe'# -> unI64 892- '\x03ff'# -> unI64 893- '\x0400'# -> unI64 1104- '\x0401'# -> unI64 1105- '\x0402'# -> unI64 1106- '\x0403'# -> unI64 1107- '\x0404'# -> unI64 1108- '\x0405'# -> unI64 1109- '\x0406'# -> unI64 1110- '\x0407'# -> unI64 1111- '\x0408'# -> unI64 1112- '\x0409'# -> unI64 1113- '\x040a'# -> unI64 1114- '\x040b'# -> unI64 1115- '\x040c'# -> unI64 1116- '\x040d'# -> unI64 1117- '\x040e'# -> unI64 1118- '\x040f'# -> unI64 1119- '\x0410'# -> unI64 1072- '\x0411'# -> unI64 1073- '\x0412'# -> unI64 1074- '\x0413'# -> unI64 1075- '\x0414'# -> unI64 1076- '\x0415'# -> unI64 1077- '\x0416'# -> unI64 1078- '\x0417'# -> unI64 1079- '\x0418'# -> unI64 1080- '\x0419'# -> unI64 1081- '\x041a'# -> unI64 1082- '\x041b'# -> unI64 1083- '\x041c'# -> unI64 1084- '\x041d'# -> unI64 1085- '\x041e'# -> unI64 1086- '\x041f'# -> unI64 1087- '\x0420'# -> unI64 1088- '\x0421'# -> unI64 1089- '\x0422'# -> unI64 1090- '\x0423'# -> unI64 1091- '\x0424'# -> unI64 1092- '\x0425'# -> unI64 1093- '\x0426'# -> unI64 1094- '\x0427'# -> unI64 1095- '\x0428'# -> unI64 1096- '\x0429'# -> unI64 1097- '\x042a'# -> unI64 1098- '\x042b'# -> unI64 1099- '\x042c'# -> unI64 1100- '\x042d'# -> unI64 1101- '\x042e'# -> unI64 1102- '\x042f'# -> unI64 1103- '\x0460'# -> unI64 1121- '\x0462'# -> unI64 1123- '\x0464'# -> unI64 1125- '\x0466'# -> unI64 1127- '\x0468'# -> unI64 1129- '\x046a'# -> unI64 1131- '\x046c'# -> unI64 1133- '\x046e'# -> unI64 1135- '\x0470'# -> unI64 1137- '\x0472'# -> unI64 1139- '\x0474'# -> unI64 1141- '\x0476'# -> unI64 1143- '\x0478'# -> unI64 1145- '\x047a'# -> unI64 1147- '\x047c'# -> unI64 1149- '\x047e'# -> unI64 1151- '\x0480'# -> unI64 1153- '\x048a'# -> unI64 1163- '\x048c'# -> unI64 1165- '\x048e'# -> unI64 1167- '\x0490'# -> unI64 1169- '\x0492'# -> unI64 1171- '\x0494'# -> unI64 1173- '\x0496'# -> unI64 1175- '\x0498'# -> unI64 1177- '\x049a'# -> unI64 1179- '\x049c'# -> unI64 1181- '\x049e'# -> unI64 1183- '\x04a0'# -> unI64 1185- '\x04a2'# -> unI64 1187- '\x04a4'# -> unI64 1189- '\x04a6'# -> unI64 1191- '\x04a8'# -> unI64 1193- '\x04aa'# -> unI64 1195- '\x04ac'# -> unI64 1197- '\x04ae'# -> unI64 1199- '\x04b0'# -> unI64 1201- '\x04b2'# -> unI64 1203- '\x04b4'# -> unI64 1205- '\x04b6'# -> unI64 1207- '\x04b8'# -> unI64 1209- '\x04ba'# -> unI64 1211- '\x04bc'# -> unI64 1213- '\x04be'# -> unI64 1215- '\x04c0'# -> unI64 1231- '\x04c1'# -> unI64 1218- '\x04c3'# -> unI64 1220- '\x04c5'# -> unI64 1222- '\x04c7'# -> unI64 1224- '\x04c9'# -> unI64 1226- '\x04cb'# -> unI64 1228- '\x04cd'# -> unI64 1230- '\x04d0'# -> unI64 1233- '\x04d2'# -> unI64 1235- '\x04d4'# -> unI64 1237- '\x04d6'# -> unI64 1239- '\x04d8'# -> unI64 1241- '\x04da'# -> unI64 1243- '\x04dc'# -> unI64 1245- '\x04de'# -> unI64 1247- '\x04e0'# -> unI64 1249- '\x04e2'# -> unI64 1251- '\x04e4'# -> unI64 1253- '\x04e6'# -> unI64 1255- '\x04e8'# -> unI64 1257- '\x04ea'# -> unI64 1259- '\x04ec'# -> unI64 1261- '\x04ee'# -> unI64 1263- '\x04f0'# -> unI64 1265- '\x04f2'# -> unI64 1267- '\x04f4'# -> unI64 1269- '\x04f6'# -> unI64 1271- '\x04f8'# -> unI64 1273- '\x04fa'# -> unI64 1275- '\x04fc'# -> unI64 1277- '\x04fe'# -> unI64 1279- '\x0500'# -> unI64 1281- '\x0502'# -> unI64 1283- '\x0504'# -> unI64 1285- '\x0506'# -> unI64 1287- '\x0508'# -> unI64 1289- '\x050a'# -> unI64 1291- '\x050c'# -> unI64 1293- '\x050e'# -> unI64 1295- '\x0510'# -> unI64 1297- '\x0512'# -> unI64 1299- '\x0514'# -> unI64 1301- '\x0516'# -> unI64 1303- '\x0518'# -> unI64 1305- '\x051a'# -> unI64 1307- '\x051c'# -> unI64 1309- '\x051e'# -> unI64 1311- '\x0520'# -> unI64 1313- '\x0522'# -> unI64 1315- '\x0524'# -> unI64 1317- '\x0526'# -> unI64 1319- '\x0528'# -> unI64 1321- '\x052a'# -> unI64 1323- '\x052c'# -> unI64 1325- '\x052e'# -> unI64 1327- '\x0531'# -> unI64 1377- '\x0532'# -> unI64 1378- '\x0533'# -> unI64 1379- '\x0534'# -> unI64 1380- '\x0535'# -> unI64 1381- '\x0536'# -> unI64 1382- '\x0537'# -> unI64 1383- '\x0538'# -> unI64 1384- '\x0539'# -> unI64 1385- '\x053a'# -> unI64 1386- '\x053b'# -> unI64 1387- '\x053c'# -> unI64 1388- '\x053d'# -> unI64 1389- '\x053e'# -> unI64 1390- '\x053f'# -> unI64 1391- '\x0540'# -> unI64 1392- '\x0541'# -> unI64 1393- '\x0542'# -> unI64 1394- '\x0543'# -> unI64 1395- '\x0544'# -> unI64 1396- '\x0545'# -> unI64 1397- '\x0546'# -> unI64 1398- '\x0547'# -> unI64 1399- '\x0548'# -> unI64 1400- '\x0549'# -> unI64 1401- '\x054a'# -> unI64 1402- '\x054b'# -> unI64 1403- '\x054c'# -> unI64 1404- '\x054d'# -> unI64 1405- '\x054e'# -> unI64 1406- '\x054f'# -> unI64 1407- '\x0550'# -> unI64 1408- '\x0551'# -> unI64 1409- '\x0552'# -> unI64 1410- '\x0553'# -> unI64 1411- '\x0554'# -> unI64 1412- '\x0555'# -> unI64 1413- '\x0556'# -> unI64 1414- '\x10a0'# -> unI64 11520- '\x10a1'# -> unI64 11521- '\x10a2'# -> unI64 11522- '\x10a3'# -> unI64 11523- '\x10a4'# -> unI64 11524- '\x10a5'# -> unI64 11525- '\x10a6'# -> unI64 11526- '\x10a7'# -> unI64 11527- '\x10a8'# -> unI64 11528- '\x10a9'# -> unI64 11529- '\x10aa'# -> unI64 11530- '\x10ab'# -> unI64 11531- '\x10ac'# -> unI64 11532- '\x10ad'# -> unI64 11533- '\x10ae'# -> unI64 11534- '\x10af'# -> unI64 11535- '\x10b0'# -> unI64 11536- '\x10b1'# -> unI64 11537- '\x10b2'# -> unI64 11538- '\x10b3'# -> unI64 11539- '\x10b4'# -> unI64 11540- '\x10b5'# -> unI64 11541- '\x10b6'# -> unI64 11542- '\x10b7'# -> unI64 11543- '\x10b8'# -> unI64 11544- '\x10b9'# -> unI64 11545- '\x10ba'# -> unI64 11546- '\x10bb'# -> unI64 11547- '\x10bc'# -> unI64 11548- '\x10bd'# -> unI64 11549- '\x10be'# -> unI64 11550- '\x10bf'# -> unI64 11551- '\x10c0'# -> unI64 11552- '\x10c1'# -> unI64 11553- '\x10c2'# -> unI64 11554- '\x10c3'# -> unI64 11555- '\x10c4'# -> unI64 11556- '\x10c5'# -> unI64 11557- '\x10c7'# -> unI64 11559- '\x10cd'# -> unI64 11565- '\x13a0'# -> unI64 43888- '\x13a1'# -> unI64 43889- '\x13a2'# -> unI64 43890- '\x13a3'# -> unI64 43891- '\x13a4'# -> unI64 43892- '\x13a5'# -> unI64 43893- '\x13a6'# -> unI64 43894- '\x13a7'# -> unI64 43895- '\x13a8'# -> unI64 43896- '\x13a9'# -> unI64 43897- '\x13aa'# -> unI64 43898- '\x13ab'# -> unI64 43899- '\x13ac'# -> unI64 43900- '\x13ad'# -> unI64 43901- '\x13ae'# -> unI64 43902- '\x13af'# -> unI64 43903- '\x13b0'# -> unI64 43904- '\x13b1'# -> unI64 43905- '\x13b2'# -> unI64 43906- '\x13b3'# -> unI64 43907- '\x13b4'# -> unI64 43908- '\x13b5'# -> unI64 43909- '\x13b6'# -> unI64 43910- '\x13b7'# -> unI64 43911- '\x13b8'# -> unI64 43912- '\x13b9'# -> unI64 43913- '\x13ba'# -> unI64 43914- '\x13bb'# -> unI64 43915- '\x13bc'# -> unI64 43916- '\x13bd'# -> unI64 43917- '\x13be'# -> unI64 43918- '\x13bf'# -> unI64 43919- '\x13c0'# -> unI64 43920- '\x13c1'# -> unI64 43921- '\x13c2'# -> unI64 43922- '\x13c3'# -> unI64 43923- '\x13c4'# -> unI64 43924- '\x13c5'# -> unI64 43925- '\x13c6'# -> unI64 43926- '\x13c7'# -> unI64 43927- '\x13c8'# -> unI64 43928- '\x13c9'# -> unI64 43929- '\x13ca'# -> unI64 43930- '\x13cb'# -> unI64 43931- '\x13cc'# -> unI64 43932- '\x13cd'# -> unI64 43933- '\x13ce'# -> unI64 43934- '\x13cf'# -> unI64 43935- '\x13d0'# -> unI64 43936- '\x13d1'# -> unI64 43937- '\x13d2'# -> unI64 43938- '\x13d3'# -> unI64 43939- '\x13d4'# -> unI64 43940- '\x13d5'# -> unI64 43941- '\x13d6'# -> unI64 43942- '\x13d7'# -> unI64 43943- '\x13d8'# -> unI64 43944- '\x13d9'# -> unI64 43945- '\x13da'# -> unI64 43946- '\x13db'# -> unI64 43947- '\x13dc'# -> unI64 43948- '\x13dd'# -> unI64 43949- '\x13de'# -> unI64 43950- '\x13df'# -> unI64 43951- '\x13e0'# -> unI64 43952- '\x13e1'# -> unI64 43953- '\x13e2'# -> unI64 43954- '\x13e3'# -> unI64 43955- '\x13e4'# -> unI64 43956- '\x13e5'# -> unI64 43957- '\x13e6'# -> unI64 43958- '\x13e7'# -> unI64 43959- '\x13e8'# -> unI64 43960- '\x13e9'# -> unI64 43961- '\x13ea'# -> unI64 43962- '\x13eb'# -> unI64 43963- '\x13ec'# -> unI64 43964- '\x13ed'# -> unI64 43965- '\x13ee'# -> unI64 43966- '\x13ef'# -> unI64 43967- '\x13f0'# -> unI64 5112- '\x13f1'# -> unI64 5113- '\x13f2'# -> unI64 5114- '\x13f3'# -> unI64 5115- '\x13f4'# -> unI64 5116- '\x13f5'# -> unI64 5117- '\x1c90'# -> unI64 4304- '\x1c91'# -> unI64 4305- '\x1c92'# -> unI64 4306- '\x1c93'# -> unI64 4307- '\x1c94'# -> unI64 4308- '\x1c95'# -> unI64 4309- '\x1c96'# -> unI64 4310- '\x1c97'# -> unI64 4311- '\x1c98'# -> unI64 4312- '\x1c99'# -> unI64 4313- '\x1c9a'# -> unI64 4314- '\x1c9b'# -> unI64 4315- '\x1c9c'# -> unI64 4316- '\x1c9d'# -> unI64 4317- '\x1c9e'# -> unI64 4318- '\x1c9f'# -> unI64 4319- '\x1ca0'# -> unI64 4320- '\x1ca1'# -> unI64 4321- '\x1ca2'# -> unI64 4322- '\x1ca3'# -> unI64 4323- '\x1ca4'# -> unI64 4324- '\x1ca5'# -> unI64 4325- '\x1ca6'# -> unI64 4326- '\x1ca7'# -> unI64 4327- '\x1ca8'# -> unI64 4328- '\x1ca9'# -> unI64 4329- '\x1caa'# -> unI64 4330- '\x1cab'# -> unI64 4331- '\x1cac'# -> unI64 4332- '\x1cad'# -> unI64 4333- '\x1cae'# -> unI64 4334- '\x1caf'# -> unI64 4335- '\x1cb0'# -> unI64 4336- '\x1cb1'# -> unI64 4337- '\x1cb2'# -> unI64 4338- '\x1cb3'# -> unI64 4339- '\x1cb4'# -> unI64 4340- '\x1cb5'# -> unI64 4341- '\x1cb6'# -> unI64 4342- '\x1cb7'# -> unI64 4343- '\x1cb8'# -> unI64 4344- '\x1cb9'# -> unI64 4345- '\x1cba'# -> unI64 4346- '\x1cbd'# -> unI64 4349- '\x1cbe'# -> unI64 4350- '\x1cbf'# -> unI64 4351- '\x1e00'# -> unI64 7681- '\x1e02'# -> unI64 7683- '\x1e04'# -> unI64 7685- '\x1e06'# -> unI64 7687- '\x1e08'# -> unI64 7689- '\x1e0a'# -> unI64 7691- '\x1e0c'# -> unI64 7693- '\x1e0e'# -> unI64 7695- '\x1e10'# -> unI64 7697- '\x1e12'# -> unI64 7699- '\x1e14'# -> unI64 7701- '\x1e16'# -> unI64 7703- '\x1e18'# -> unI64 7705- '\x1e1a'# -> unI64 7707- '\x1e1c'# -> unI64 7709- '\x1e1e'# -> unI64 7711- '\x1e20'# -> unI64 7713- '\x1e22'# -> unI64 7715- '\x1e24'# -> unI64 7717- '\x1e26'# -> unI64 7719- '\x1e28'# -> unI64 7721- '\x1e2a'# -> unI64 7723- '\x1e2c'# -> unI64 7725- '\x1e2e'# -> unI64 7727- '\x1e30'# -> unI64 7729- '\x1e32'# -> unI64 7731- '\x1e34'# -> unI64 7733- '\x1e36'# -> unI64 7735- '\x1e38'# -> unI64 7737- '\x1e3a'# -> unI64 7739- '\x1e3c'# -> unI64 7741- '\x1e3e'# -> unI64 7743- '\x1e40'# -> unI64 7745- '\x1e42'# -> unI64 7747- '\x1e44'# -> unI64 7749- '\x1e46'# -> unI64 7751- '\x1e48'# -> unI64 7753- '\x1e4a'# -> unI64 7755- '\x1e4c'# -> unI64 7757- '\x1e4e'# -> unI64 7759- '\x1e50'# -> unI64 7761- '\x1e52'# -> unI64 7763- '\x1e54'# -> unI64 7765- '\x1e56'# -> unI64 7767- '\x1e58'# -> unI64 7769- '\x1e5a'# -> unI64 7771- '\x1e5c'# -> unI64 7773- '\x1e5e'# -> unI64 7775- '\x1e60'# -> unI64 7777- '\x1e62'# -> unI64 7779- '\x1e64'# -> unI64 7781- '\x1e66'# -> unI64 7783- '\x1e68'# -> unI64 7785- '\x1e6a'# -> unI64 7787- '\x1e6c'# -> unI64 7789- '\x1e6e'# -> unI64 7791- '\x1e70'# -> unI64 7793- '\x1e72'# -> unI64 7795- '\x1e74'# -> unI64 7797- '\x1e76'# -> unI64 7799- '\x1e78'# -> unI64 7801- '\x1e7a'# -> unI64 7803- '\x1e7c'# -> unI64 7805- '\x1e7e'# -> unI64 7807- '\x1e80'# -> unI64 7809- '\x1e82'# -> unI64 7811- '\x1e84'# -> unI64 7813- '\x1e86'# -> unI64 7815- '\x1e88'# -> unI64 7817- '\x1e8a'# -> unI64 7819- '\x1e8c'# -> unI64 7821- '\x1e8e'# -> unI64 7823- '\x1e90'# -> unI64 7825- '\x1e92'# -> unI64 7827- '\x1e94'# -> unI64 7829- '\x1e9e'# -> unI64 223- '\x1ea0'# -> unI64 7841- '\x1ea2'# -> unI64 7843- '\x1ea4'# -> unI64 7845- '\x1ea6'# -> unI64 7847- '\x1ea8'# -> unI64 7849- '\x1eaa'# -> unI64 7851- '\x1eac'# -> unI64 7853- '\x1eae'# -> unI64 7855- '\x1eb0'# -> unI64 7857- '\x1eb2'# -> unI64 7859- '\x1eb4'# -> unI64 7861- '\x1eb6'# -> unI64 7863- '\x1eb8'# -> unI64 7865- '\x1eba'# -> unI64 7867- '\x1ebc'# -> unI64 7869- '\x1ebe'# -> unI64 7871- '\x1ec0'# -> unI64 7873- '\x1ec2'# -> unI64 7875- '\x1ec4'# -> unI64 7877- '\x1ec6'# -> unI64 7879- '\x1ec8'# -> unI64 7881- '\x1eca'# -> unI64 7883- '\x1ecc'# -> unI64 7885- '\x1ece'# -> unI64 7887- '\x1ed0'# -> unI64 7889- '\x1ed2'# -> unI64 7891- '\x1ed4'# -> unI64 7893- '\x1ed6'# -> unI64 7895- '\x1ed8'# -> unI64 7897- '\x1eda'# -> unI64 7899- '\x1edc'# -> unI64 7901- '\x1ede'# -> unI64 7903- '\x1ee0'# -> unI64 7905- '\x1ee2'# -> unI64 7907- '\x1ee4'# -> unI64 7909- '\x1ee6'# -> unI64 7911- '\x1ee8'# -> unI64 7913- '\x1eea'# -> unI64 7915- '\x1eec'# -> unI64 7917- '\x1eee'# -> unI64 7919- '\x1ef0'# -> unI64 7921- '\x1ef2'# -> unI64 7923- '\x1ef4'# -> unI64 7925- '\x1ef6'# -> unI64 7927- '\x1ef8'# -> unI64 7929- '\x1efa'# -> unI64 7931- '\x1efc'# -> unI64 7933- '\x1efe'# -> unI64 7935- '\x1f08'# -> unI64 7936- '\x1f09'# -> unI64 7937- '\x1f0a'# -> unI64 7938- '\x1f0b'# -> unI64 7939- '\x1f0c'# -> unI64 7940- '\x1f0d'# -> unI64 7941- '\x1f0e'# -> unI64 7942- '\x1f0f'# -> unI64 7943- '\x1f18'# -> unI64 7952- '\x1f19'# -> unI64 7953- '\x1f1a'# -> unI64 7954- '\x1f1b'# -> unI64 7955- '\x1f1c'# -> unI64 7956- '\x1f1d'# -> unI64 7957- '\x1f28'# -> unI64 7968- '\x1f29'# -> unI64 7969- '\x1f2a'# -> unI64 7970- '\x1f2b'# -> unI64 7971- '\x1f2c'# -> unI64 7972- '\x1f2d'# -> unI64 7973- '\x1f2e'# -> unI64 7974- '\x1f2f'# -> unI64 7975- '\x1f38'# -> unI64 7984- '\x1f39'# -> unI64 7985- '\x1f3a'# -> unI64 7986- '\x1f3b'# -> unI64 7987- '\x1f3c'# -> unI64 7988- '\x1f3d'# -> unI64 7989- '\x1f3e'# -> unI64 7990- '\x1f3f'# -> unI64 7991- '\x1f48'# -> unI64 8000- '\x1f49'# -> unI64 8001- '\x1f4a'# -> unI64 8002- '\x1f4b'# -> unI64 8003- '\x1f4c'# -> unI64 8004- '\x1f4d'# -> unI64 8005- '\x1f59'# -> unI64 8017- '\x1f5b'# -> unI64 8019- '\x1f5d'# -> unI64 8021- '\x1f5f'# -> unI64 8023- '\x1f68'# -> unI64 8032- '\x1f69'# -> unI64 8033- '\x1f6a'# -> unI64 8034- '\x1f6b'# -> unI64 8035- '\x1f6c'# -> unI64 8036- '\x1f6d'# -> unI64 8037- '\x1f6e'# -> unI64 8038- '\x1f6f'# -> unI64 8039- '\x1f88'# -> unI64 8064- '\x1f89'# -> unI64 8065- '\x1f8a'# -> unI64 8066- '\x1f8b'# -> unI64 8067- '\x1f8c'# -> unI64 8068- '\x1f8d'# -> unI64 8069- '\x1f8e'# -> unI64 8070- '\x1f8f'# -> unI64 8071- '\x1f98'# -> unI64 8080- '\x1f99'# -> unI64 8081- '\x1f9a'# -> unI64 8082- '\x1f9b'# -> unI64 8083- '\x1f9c'# -> unI64 8084- '\x1f9d'# -> unI64 8085- '\x1f9e'# -> unI64 8086- '\x1f9f'# -> unI64 8087- '\x1fa8'# -> unI64 8096- '\x1fa9'# -> unI64 8097- '\x1faa'# -> unI64 8098- '\x1fab'# -> unI64 8099- '\x1fac'# -> unI64 8100- '\x1fad'# -> unI64 8101- '\x1fae'# -> unI64 8102- '\x1faf'# -> unI64 8103- '\x1fb8'# -> unI64 8112- '\x1fb9'# -> unI64 8113- '\x1fba'# -> unI64 8048- '\x1fbb'# -> unI64 8049- '\x1fbc'# -> unI64 8115- '\x1fc8'# -> unI64 8050- '\x1fc9'# -> unI64 8051- '\x1fca'# -> unI64 8052- '\x1fcb'# -> unI64 8053- '\x1fcc'# -> unI64 8131- '\x1fd8'# -> unI64 8144- '\x1fd9'# -> unI64 8145- '\x1fda'# -> unI64 8054- '\x1fdb'# -> unI64 8055- '\x1fe8'# -> unI64 8160- '\x1fe9'# -> unI64 8161- '\x1fea'# -> unI64 8058- '\x1feb'# -> unI64 8059- '\x1fec'# -> unI64 8165- '\x1ff8'# -> unI64 8056- '\x1ff9'# -> unI64 8057- '\x1ffa'# -> unI64 8060- '\x1ffb'# -> unI64 8061- '\x1ffc'# -> unI64 8179- '\x2126'# -> unI64 969- '\x212a'# -> unI64 107- '\x212b'# -> unI64 229- '\x2132'# -> unI64 8526- '\x2160'# -> unI64 8560- '\x2161'# -> unI64 8561- '\x2162'# -> unI64 8562- '\x2163'# -> unI64 8563- '\x2164'# -> unI64 8564- '\x2165'# -> unI64 8565- '\x2166'# -> unI64 8566- '\x2167'# -> unI64 8567- '\x2168'# -> unI64 8568- '\x2169'# -> unI64 8569- '\x216a'# -> unI64 8570- '\x216b'# -> unI64 8571- '\x216c'# -> unI64 8572- '\x216d'# -> unI64 8573- '\x216e'# -> unI64 8574- '\x216f'# -> unI64 8575- '\x2183'# -> unI64 8580- '\x24b6'# -> unI64 9424- '\x24b7'# -> unI64 9425- '\x24b8'# -> unI64 9426- '\x24b9'# -> unI64 9427- '\x24ba'# -> unI64 9428- '\x24bb'# -> unI64 9429- '\x24bc'# -> unI64 9430- '\x24bd'# -> unI64 9431- '\x24be'# -> unI64 9432- '\x24bf'# -> unI64 9433- '\x24c0'# -> unI64 9434- '\x24c1'# -> unI64 9435- '\x24c2'# -> unI64 9436- '\x24c3'# -> unI64 9437- '\x24c4'# -> unI64 9438- '\x24c5'# -> unI64 9439- '\x24c6'# -> unI64 9440- '\x24c7'# -> unI64 9441- '\x24c8'# -> unI64 9442- '\x24c9'# -> unI64 9443- '\x24ca'# -> unI64 9444- '\x24cb'# -> unI64 9445- '\x24cc'# -> unI64 9446- '\x24cd'# -> unI64 9447- '\x24ce'# -> unI64 9448- '\x24cf'# -> unI64 9449- '\x2c00'# -> unI64 11312- '\x2c01'# -> unI64 11313- '\x2c02'# -> unI64 11314- '\x2c03'# -> unI64 11315- '\x2c04'# -> unI64 11316- '\x2c05'# -> unI64 11317- '\x2c06'# -> unI64 11318- '\x2c07'# -> unI64 11319- '\x2c08'# -> unI64 11320- '\x2c09'# -> unI64 11321- '\x2c0a'# -> unI64 11322- '\x2c0b'# -> unI64 11323- '\x2c0c'# -> unI64 11324- '\x2c0d'# -> unI64 11325- '\x2c0e'# -> unI64 11326- '\x2c0f'# -> unI64 11327- '\x2c10'# -> unI64 11328- '\x2c11'# -> unI64 11329- '\x2c12'# -> unI64 11330- '\x2c13'# -> unI64 11331- '\x2c14'# -> unI64 11332- '\x2c15'# -> unI64 11333- '\x2c16'# -> unI64 11334- '\x2c17'# -> unI64 11335- '\x2c18'# -> unI64 11336- '\x2c19'# -> unI64 11337- '\x2c1a'# -> unI64 11338- '\x2c1b'# -> unI64 11339- '\x2c1c'# -> unI64 11340- '\x2c1d'# -> unI64 11341- '\x2c1e'# -> unI64 11342- '\x2c1f'# -> unI64 11343- '\x2c20'# -> unI64 11344- '\x2c21'# -> unI64 11345- '\x2c22'# -> unI64 11346- '\x2c23'# -> unI64 11347- '\x2c24'# -> unI64 11348- '\x2c25'# -> unI64 11349- '\x2c26'# -> unI64 11350- '\x2c27'# -> unI64 11351- '\x2c28'# -> unI64 11352- '\x2c29'# -> unI64 11353- '\x2c2a'# -> unI64 11354- '\x2c2b'# -> unI64 11355- '\x2c2c'# -> unI64 11356- '\x2c2d'# -> unI64 11357- '\x2c2e'# -> unI64 11358- '\x2c2f'# -> unI64 11359- '\x2c60'# -> unI64 11361- '\x2c62'# -> unI64 619- '\x2c63'# -> unI64 7549- '\x2c64'# -> unI64 637- '\x2c67'# -> unI64 11368- '\x2c69'# -> unI64 11370- '\x2c6b'# -> unI64 11372- '\x2c6d'# -> unI64 593- '\x2c6e'# -> unI64 625- '\x2c6f'# -> unI64 592- '\x2c70'# -> unI64 594- '\x2c72'# -> unI64 11379- '\x2c75'# -> unI64 11382- '\x2c7e'# -> unI64 575- '\x2c7f'# -> unI64 576- '\x2c80'# -> unI64 11393- '\x2c82'# -> unI64 11395- '\x2c84'# -> unI64 11397- '\x2c86'# -> unI64 11399- '\x2c88'# -> unI64 11401- '\x2c8a'# -> unI64 11403- '\x2c8c'# -> unI64 11405- '\x2c8e'# -> unI64 11407- '\x2c90'# -> unI64 11409- '\x2c92'# -> unI64 11411- '\x2c94'# -> unI64 11413- '\x2c96'# -> unI64 11415- '\x2c98'# -> unI64 11417- '\x2c9a'# -> unI64 11419- '\x2c9c'# -> unI64 11421- '\x2c9e'# -> unI64 11423- '\x2ca0'# -> unI64 11425- '\x2ca2'# -> unI64 11427- '\x2ca4'# -> unI64 11429- '\x2ca6'# -> unI64 11431- '\x2ca8'# -> unI64 11433- '\x2caa'# -> unI64 11435- '\x2cac'# -> unI64 11437- '\x2cae'# -> unI64 11439- '\x2cb0'# -> unI64 11441- '\x2cb2'# -> unI64 11443- '\x2cb4'# -> unI64 11445- '\x2cb6'# -> unI64 11447- '\x2cb8'# -> unI64 11449- '\x2cba'# -> unI64 11451- '\x2cbc'# -> unI64 11453- '\x2cbe'# -> unI64 11455- '\x2cc0'# -> unI64 11457- '\x2cc2'# -> unI64 11459- '\x2cc4'# -> unI64 11461- '\x2cc6'# -> unI64 11463- '\x2cc8'# -> unI64 11465- '\x2cca'# -> unI64 11467- '\x2ccc'# -> unI64 11469- '\x2cce'# -> unI64 11471- '\x2cd0'# -> unI64 11473- '\x2cd2'# -> unI64 11475- '\x2cd4'# -> unI64 11477- '\x2cd6'# -> unI64 11479- '\x2cd8'# -> unI64 11481- '\x2cda'# -> unI64 11483- '\x2cdc'# -> unI64 11485- '\x2cde'# -> unI64 11487- '\x2ce0'# -> unI64 11489- '\x2ce2'# -> unI64 11491- '\x2ceb'# -> unI64 11500- '\x2ced'# -> unI64 11502- '\x2cf2'# -> unI64 11507- '\xa640'# -> unI64 42561- '\xa642'# -> unI64 42563- '\xa644'# -> unI64 42565- '\xa646'# -> unI64 42567- '\xa648'# -> unI64 42569- '\xa64a'# -> unI64 42571- '\xa64c'# -> unI64 42573- '\xa64e'# -> unI64 42575- '\xa650'# -> unI64 42577- '\xa652'# -> unI64 42579- '\xa654'# -> unI64 42581- '\xa656'# -> unI64 42583- '\xa658'# -> unI64 42585- '\xa65a'# -> unI64 42587- '\xa65c'# -> unI64 42589- '\xa65e'# -> unI64 42591- '\xa660'# -> unI64 42593- '\xa662'# -> unI64 42595- '\xa664'# -> unI64 42597- '\xa666'# -> unI64 42599- '\xa668'# -> unI64 42601- '\xa66a'# -> unI64 42603- '\xa66c'# -> unI64 42605- '\xa680'# -> unI64 42625- '\xa682'# -> unI64 42627- '\xa684'# -> unI64 42629- '\xa686'# -> unI64 42631- '\xa688'# -> unI64 42633- '\xa68a'# -> unI64 42635- '\xa68c'# -> unI64 42637- '\xa68e'# -> unI64 42639- '\xa690'# -> unI64 42641- '\xa692'# -> unI64 42643- '\xa694'# -> unI64 42645- '\xa696'# -> unI64 42647- '\xa698'# -> unI64 42649- '\xa69a'# -> unI64 42651- '\xa722'# -> unI64 42787- '\xa724'# -> unI64 42789- '\xa726'# -> unI64 42791- '\xa728'# -> unI64 42793- '\xa72a'# -> unI64 42795- '\xa72c'# -> unI64 42797- '\xa72e'# -> unI64 42799- '\xa732'# -> unI64 42803- '\xa734'# -> unI64 42805- '\xa736'# -> unI64 42807- '\xa738'# -> unI64 42809- '\xa73a'# -> unI64 42811- '\xa73c'# -> unI64 42813- '\xa73e'# -> unI64 42815- '\xa740'# -> unI64 42817- '\xa742'# -> unI64 42819- '\xa744'# -> unI64 42821- '\xa746'# -> unI64 42823- '\xa748'# -> unI64 42825- '\xa74a'# -> unI64 42827- '\xa74c'# -> unI64 42829- '\xa74e'# -> unI64 42831- '\xa750'# -> unI64 42833- '\xa752'# -> unI64 42835- '\xa754'# -> unI64 42837- '\xa756'# -> unI64 42839- '\xa758'# -> unI64 42841- '\xa75a'# -> unI64 42843- '\xa75c'# -> unI64 42845- '\xa75e'# -> unI64 42847- '\xa760'# -> unI64 42849- '\xa762'# -> unI64 42851- '\xa764'# -> unI64 42853- '\xa766'# -> unI64 42855- '\xa768'# -> unI64 42857- '\xa76a'# -> unI64 42859- '\xa76c'# -> unI64 42861- '\xa76e'# -> unI64 42863- '\xa779'# -> unI64 42874- '\xa77b'# -> unI64 42876- '\xa77d'# -> unI64 7545- '\xa77e'# -> unI64 42879- '\xa780'# -> unI64 42881- '\xa782'# -> unI64 42883- '\xa784'# -> unI64 42885- '\xa786'# -> unI64 42887- '\xa78b'# -> unI64 42892- '\xa78d'# -> unI64 613- '\xa790'# -> unI64 42897- '\xa792'# -> unI64 42899- '\xa796'# -> unI64 42903- '\xa798'# -> unI64 42905- '\xa79a'# -> unI64 42907- '\xa79c'# -> unI64 42909- '\xa79e'# -> unI64 42911- '\xa7a0'# -> unI64 42913- '\xa7a2'# -> unI64 42915- '\xa7a4'# -> unI64 42917- '\xa7a6'# -> unI64 42919- '\xa7a8'# -> unI64 42921- '\xa7aa'# -> unI64 614- '\xa7ab'# -> unI64 604- '\xa7ac'# -> unI64 609- '\xa7ad'# -> unI64 620- '\xa7ae'# -> unI64 618- '\xa7b0'# -> unI64 670- '\xa7b1'# -> unI64 647- '\xa7b2'# -> unI64 669- '\xa7b3'# -> unI64 43859- '\xa7b4'# -> unI64 42933- '\xa7b6'# -> unI64 42935- '\xa7b8'# -> unI64 42937- '\xa7ba'# -> unI64 42939- '\xa7bc'# -> unI64 42941- '\xa7be'# -> unI64 42943- '\xa7c0'# -> unI64 42945- '\xa7c2'# -> unI64 42947- '\xa7c4'# -> unI64 42900- '\xa7c5'# -> unI64 642- '\xa7c6'# -> unI64 7566- '\xa7c7'# -> unI64 42952- '\xa7c9'# -> unI64 42954- '\xa7d0'# -> unI64 42961- '\xa7d6'# -> unI64 42967- '\xa7d8'# -> unI64 42969- '\xa7f5'# -> unI64 42998- '\xff21'# -> unI64 65345- '\xff22'# -> unI64 65346- '\xff23'# -> unI64 65347- '\xff24'# -> unI64 65348- '\xff25'# -> unI64 65349- '\xff26'# -> unI64 65350- '\xff27'# -> unI64 65351- '\xff28'# -> unI64 65352- '\xff29'# -> unI64 65353- '\xff2a'# -> unI64 65354- '\xff2b'# -> unI64 65355- '\xff2c'# -> unI64 65356- '\xff2d'# -> unI64 65357- '\xff2e'# -> unI64 65358- '\xff2f'# -> unI64 65359- '\xff30'# -> unI64 65360- '\xff31'# -> unI64 65361- '\xff32'# -> unI64 65362- '\xff33'# -> unI64 65363- '\xff34'# -> unI64 65364- '\xff35'# -> unI64 65365- '\xff36'# -> unI64 65366- '\xff37'# -> unI64 65367- '\xff38'# -> unI64 65368- '\xff39'# -> unI64 65369- '\xff3a'# -> unI64 65370- '\x10400'# -> unI64 66600- '\x10401'# -> unI64 66601- '\x10402'# -> unI64 66602- '\x10403'# -> unI64 66603- '\x10404'# -> unI64 66604- '\x10405'# -> unI64 66605- '\x10406'# -> unI64 66606- '\x10407'# -> unI64 66607- '\x10408'# -> unI64 66608- '\x10409'# -> unI64 66609- '\x1040a'# -> unI64 66610- '\x1040b'# -> unI64 66611- '\x1040c'# -> unI64 66612- '\x1040d'# -> unI64 66613- '\x1040e'# -> unI64 66614- '\x1040f'# -> unI64 66615- '\x10410'# -> unI64 66616- '\x10411'# -> unI64 66617- '\x10412'# -> unI64 66618- '\x10413'# -> unI64 66619- '\x10414'# -> unI64 66620- '\x10415'# -> unI64 66621- '\x10416'# -> unI64 66622- '\x10417'# -> unI64 66623- '\x10418'# -> unI64 66624- '\x10419'# -> unI64 66625- '\x1041a'# -> unI64 66626- '\x1041b'# -> unI64 66627- '\x1041c'# -> unI64 66628- '\x1041d'# -> unI64 66629- '\x1041e'# -> unI64 66630- '\x1041f'# -> unI64 66631- '\x10420'# -> unI64 66632- '\x10421'# -> unI64 66633- '\x10422'# -> unI64 66634- '\x10423'# -> unI64 66635- '\x10424'# -> unI64 66636- '\x10425'# -> unI64 66637- '\x10426'# -> unI64 66638- '\x10427'# -> unI64 66639- '\x104b0'# -> unI64 66776- '\x104b1'# -> unI64 66777- '\x104b2'# -> unI64 66778- '\x104b3'# -> unI64 66779- '\x104b4'# -> unI64 66780- '\x104b5'# -> unI64 66781- '\x104b6'# -> unI64 66782- '\x104b7'# -> unI64 66783- '\x104b8'# -> unI64 66784- '\x104b9'# -> unI64 66785- '\x104ba'# -> unI64 66786- '\x104bb'# -> unI64 66787- '\x104bc'# -> unI64 66788- '\x104bd'# -> unI64 66789- '\x104be'# -> unI64 66790- '\x104bf'# -> unI64 66791- '\x104c0'# -> unI64 66792- '\x104c1'# -> unI64 66793- '\x104c2'# -> unI64 66794- '\x104c3'# -> unI64 66795- '\x104c4'# -> unI64 66796- '\x104c5'# -> unI64 66797- '\x104c6'# -> unI64 66798- '\x104c7'# -> unI64 66799- '\x104c8'# -> unI64 66800- '\x104c9'# -> unI64 66801- '\x104ca'# -> unI64 66802- '\x104cb'# -> unI64 66803- '\x104cc'# -> unI64 66804- '\x104cd'# -> unI64 66805- '\x104ce'# -> unI64 66806- '\x104cf'# -> unI64 66807- '\x104d0'# -> unI64 66808- '\x104d1'# -> unI64 66809- '\x104d2'# -> unI64 66810- '\x104d3'# -> unI64 66811- '\x10570'# -> unI64 66967- '\x10571'# -> unI64 66968- '\x10572'# -> unI64 66969- '\x10573'# -> unI64 66970- '\x10574'# -> unI64 66971- '\x10575'# -> unI64 66972- '\x10576'# -> unI64 66973- '\x10577'# -> unI64 66974- '\x10578'# -> unI64 66975- '\x10579'# -> unI64 66976- '\x1057a'# -> unI64 66977- '\x1057c'# -> unI64 66979- '\x1057d'# -> unI64 66980- '\x1057e'# -> unI64 66981- '\x1057f'# -> unI64 66982- '\x10580'# -> unI64 66983- '\x10581'# -> unI64 66984- '\x10582'# -> unI64 66985- '\x10583'# -> unI64 66986- '\x10584'# -> unI64 66987- '\x10585'# -> unI64 66988- '\x10586'# -> unI64 66989- '\x10587'# -> unI64 66990- '\x10588'# -> unI64 66991- '\x10589'# -> unI64 66992- '\x1058a'# -> unI64 66993- '\x1058c'# -> unI64 66995- '\x1058d'# -> unI64 66996- '\x1058e'# -> unI64 66997- '\x1058f'# -> unI64 66998- '\x10590'# -> unI64 66999- '\x10591'# -> unI64 67000- '\x10592'# -> unI64 67001- '\x10594'# -> unI64 67003- '\x10595'# -> unI64 67004- '\x10c80'# -> unI64 68800- '\x10c81'# -> unI64 68801- '\x10c82'# -> unI64 68802- '\x10c83'# -> unI64 68803- '\x10c84'# -> unI64 68804- '\x10c85'# -> unI64 68805- '\x10c86'# -> unI64 68806- '\x10c87'# -> unI64 68807- '\x10c88'# -> unI64 68808- '\x10c89'# -> unI64 68809- '\x10c8a'# -> unI64 68810- '\x10c8b'# -> unI64 68811- '\x10c8c'# -> unI64 68812- '\x10c8d'# -> unI64 68813- '\x10c8e'# -> unI64 68814- '\x10c8f'# -> unI64 68815- '\x10c90'# -> unI64 68816- '\x10c91'# -> unI64 68817- '\x10c92'# -> unI64 68818- '\x10c93'# -> unI64 68819- '\x10c94'# -> unI64 68820- '\x10c95'# -> unI64 68821- '\x10c96'# -> unI64 68822- '\x10c97'# -> unI64 68823- '\x10c98'# -> unI64 68824- '\x10c99'# -> unI64 68825- '\x10c9a'# -> unI64 68826- '\x10c9b'# -> unI64 68827- '\x10c9c'# -> unI64 68828- '\x10c9d'# -> unI64 68829- '\x10c9e'# -> unI64 68830- '\x10c9f'# -> unI64 68831- '\x10ca0'# -> unI64 68832- '\x10ca1'# -> unI64 68833- '\x10ca2'# -> unI64 68834- '\x10ca3'# -> unI64 68835- '\x10ca4'# -> unI64 68836- '\x10ca5'# -> unI64 68837- '\x10ca6'# -> unI64 68838- '\x10ca7'# -> unI64 68839- '\x10ca8'# -> unI64 68840- '\x10ca9'# -> unI64 68841- '\x10caa'# -> unI64 68842- '\x10cab'# -> unI64 68843- '\x10cac'# -> unI64 68844- '\x10cad'# -> unI64 68845- '\x10cae'# -> unI64 68846- '\x10caf'# -> unI64 68847- '\x10cb0'# -> unI64 68848- '\x10cb1'# -> unI64 68849- '\x10cb2'# -> unI64 68850- '\x118a0'# -> unI64 71872- '\x118a1'# -> unI64 71873- '\x118a2'# -> unI64 71874- '\x118a3'# -> unI64 71875- '\x118a4'# -> unI64 71876- '\x118a5'# -> unI64 71877- '\x118a6'# -> unI64 71878- '\x118a7'# -> unI64 71879- '\x118a8'# -> unI64 71880- '\x118a9'# -> unI64 71881- '\x118aa'# -> unI64 71882- '\x118ab'# -> unI64 71883- '\x118ac'# -> unI64 71884- '\x118ad'# -> unI64 71885- '\x118ae'# -> unI64 71886- '\x118af'# -> unI64 71887- '\x118b0'# -> unI64 71888- '\x118b1'# -> unI64 71889- '\x118b2'# -> unI64 71890- '\x118b3'# -> unI64 71891- '\x118b4'# -> unI64 71892- '\x118b5'# -> unI64 71893- '\x118b6'# -> unI64 71894- '\x118b7'# -> unI64 71895- '\x118b8'# -> unI64 71896- '\x118b9'# -> unI64 71897- '\x118ba'# -> unI64 71898- '\x118bb'# -> unI64 71899- '\x118bc'# -> unI64 71900- '\x118bd'# -> unI64 71901- '\x118be'# -> unI64 71902- '\x118bf'# -> unI64 71903- '\x16e40'# -> unI64 93792- '\x16e41'# -> unI64 93793- '\x16e42'# -> unI64 93794- '\x16e43'# -> unI64 93795- '\x16e44'# -> unI64 93796- '\x16e45'# -> unI64 93797- '\x16e46'# -> unI64 93798- '\x16e47'# -> unI64 93799- '\x16e48'# -> unI64 93800- '\x16e49'# -> unI64 93801- '\x16e4a'# -> unI64 93802- '\x16e4b'# -> unI64 93803- '\x16e4c'# -> unI64 93804- '\x16e4d'# -> unI64 93805- '\x16e4e'# -> unI64 93806- '\x16e4f'# -> unI64 93807- '\x16e50'# -> unI64 93808- '\x16e51'# -> unI64 93809- '\x16e52'# -> unI64 93810- '\x16e53'# -> unI64 93811- '\x16e54'# -> unI64 93812- '\x16e55'# -> unI64 93813- '\x16e56'# -> unI64 93814- '\x16e57'# -> unI64 93815- '\x16e58'# -> unI64 93816- '\x16e59'# -> unI64 93817- '\x16e5a'# -> unI64 93818- '\x16e5b'# -> unI64 93819- '\x16e5c'# -> unI64 93820- '\x16e5d'# -> unI64 93821- '\x16e5e'# -> unI64 93822- '\x16e5f'# -> unI64 93823- '\x1e900'# -> unI64 125218- '\x1e901'# -> unI64 125219- '\x1e902'# -> unI64 125220- '\x1e903'# -> unI64 125221- '\x1e904'# -> unI64 125222- '\x1e905'# -> unI64 125223- '\x1e906'# -> unI64 125224- '\x1e907'# -> unI64 125225- '\x1e908'# -> unI64 125226- '\x1e909'# -> unI64 125227- '\x1e90a'# -> unI64 125228- '\x1e90b'# -> unI64 125229- '\x1e90c'# -> unI64 125230- '\x1e90d'# -> unI64 125231- '\x1e90e'# -> unI64 125232- '\x1e90f'# -> unI64 125233- '\x1e910'# -> unI64 125234- '\x1e911'# -> unI64 125235- '\x1e912'# -> unI64 125236- '\x1e913'# -> unI64 125237- '\x1e914'# -> unI64 125238- '\x1e915'# -> unI64 125239- '\x1e916'# -> unI64 125240- '\x1e917'# -> unI64 125241- '\x1e918'# -> unI64 125242- '\x1e919'# -> unI64 125243- '\x1e91a'# -> unI64 125244- '\x1e91b'# -> unI64 125245- '\x1e91c'# -> unI64 125246- '\x1e91d'# -> unI64 125247- '\x1e91e'# -> unI64 125248- '\x1e91f'# -> unI64 125249- '\x1e920'# -> unI64 125250- '\x1e921'# -> unI64 125251- _ -> unI64 0-titleMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE titleMapping #-}-titleMapping = \case- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 241172563- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 213909574- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 220201030- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 226492486- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 461795097575494- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 474989237108806- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 243269715- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 243269715- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2956985653- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2931819844- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2896168260- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2908751172- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2931819854- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2912945476- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 163578556- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429849- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429861- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778634- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373256- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390036- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584343- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584345- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200769- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459557- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987429- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498533- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720293- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025681- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025687- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918745- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429849- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025689- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651609- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918757- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429861- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459553- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025701- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651621- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025705- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1755324346- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1755317126- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1755324362- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1755317129- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1755324410- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1755317135- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 3681166678819729- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 3681166678819735- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 3681166678819753- '\x0061'# -> unI64 65- '\x0062'# -> unI64 66- '\x0063'# -> unI64 67- '\x0064'# -> unI64 68- '\x0065'# -> unI64 69- '\x0066'# -> unI64 70- '\x0067'# -> unI64 71- '\x0068'# -> unI64 72- '\x0069'# -> unI64 73- '\x006a'# -> unI64 74- '\x006b'# -> unI64 75- '\x006c'# -> unI64 76- '\x006d'# -> unI64 77- '\x006e'# -> unI64 78- '\x006f'# -> unI64 79- '\x0070'# -> unI64 80- '\x0071'# -> unI64 81- '\x0072'# -> unI64 82- '\x0073'# -> unI64 83- '\x0074'# -> unI64 84- '\x0075'# -> unI64 85- '\x0076'# -> unI64 86- '\x0077'# -> unI64 87- '\x0078'# -> unI64 88- '\x0079'# -> unI64 89- '\x007a'# -> unI64 90- '\x00b5'# -> unI64 924- '\x00e0'# -> unI64 192- '\x00e1'# -> unI64 193- '\x00e2'# -> unI64 194- '\x00e3'# -> unI64 195- '\x00e4'# -> unI64 196- '\x00e5'# -> unI64 197- '\x00e6'# -> unI64 198- '\x00e7'# -> unI64 199- '\x00e8'# -> unI64 200- '\x00e9'# -> unI64 201- '\x00ea'# -> unI64 202- '\x00eb'# -> unI64 203- '\x00ec'# -> unI64 204- '\x00ed'# -> unI64 205- '\x00ee'# -> unI64 206- '\x00ef'# -> unI64 207- '\x00f0'# -> unI64 208- '\x00f1'# -> unI64 209- '\x00f2'# -> unI64 210- '\x00f3'# -> unI64 211- '\x00f4'# -> unI64 212- '\x00f5'# -> unI64 213- '\x00f6'# -> unI64 214- '\x00f8'# -> unI64 216- '\x00f9'# -> unI64 217- '\x00fa'# -> unI64 218- '\x00fb'# -> unI64 219- '\x00fc'# -> unI64 220- '\x00fd'# -> unI64 221- '\x00fe'# -> unI64 222- '\x00ff'# -> unI64 376- '\x0101'# -> unI64 256- '\x0103'# -> unI64 258- '\x0105'# -> unI64 260- '\x0107'# -> unI64 262- '\x0109'# -> unI64 264- '\x010b'# -> unI64 266- '\x010d'# -> unI64 268- '\x010f'# -> unI64 270- '\x0111'# -> unI64 272- '\x0113'# -> unI64 274- '\x0115'# -> unI64 276- '\x0117'# -> unI64 278- '\x0119'# -> unI64 280- '\x011b'# -> unI64 282- '\x011d'# -> unI64 284- '\x011f'# -> unI64 286- '\x0121'# -> unI64 288- '\x0123'# -> unI64 290- '\x0125'# -> unI64 292- '\x0127'# -> unI64 294- '\x0129'# -> unI64 296- '\x012b'# -> unI64 298- '\x012d'# -> unI64 300- '\x012f'# -> unI64 302- '\x0131'# -> unI64 73- '\x0133'# -> unI64 306- '\x0135'# -> unI64 308- '\x0137'# -> unI64 310- '\x013a'# -> unI64 313- '\x013c'# -> unI64 315- '\x013e'# -> unI64 317- '\x0140'# -> unI64 319- '\x0142'# -> unI64 321- '\x0144'# -> unI64 323- '\x0146'# -> unI64 325- '\x0148'# -> unI64 327- '\x014b'# -> unI64 330- '\x014d'# -> unI64 332- '\x014f'# -> unI64 334- '\x0151'# -> unI64 336- '\x0153'# -> unI64 338- '\x0155'# -> unI64 340- '\x0157'# -> unI64 342- '\x0159'# -> unI64 344- '\x015b'# -> unI64 346- '\x015d'# -> unI64 348- '\x015f'# -> unI64 350- '\x0161'# -> unI64 352- '\x0163'# -> unI64 354- '\x0165'# -> unI64 356- '\x0167'# -> unI64 358- '\x0169'# -> unI64 360- '\x016b'# -> unI64 362- '\x016d'# -> unI64 364- '\x016f'# -> unI64 366- '\x0171'# -> unI64 368- '\x0173'# -> unI64 370- '\x0175'# -> unI64 372- '\x0177'# -> unI64 374- '\x017a'# -> unI64 377- '\x017c'# -> unI64 379- '\x017e'# -> unI64 381- '\x017f'# -> unI64 83- '\x0180'# -> unI64 579- '\x0183'# -> unI64 386- '\x0185'# -> unI64 388- '\x0188'# -> unI64 391- '\x018c'# -> unI64 395- '\x0192'# -> unI64 401- '\x0195'# -> unI64 502- '\x0199'# -> unI64 408- '\x019a'# -> unI64 573- '\x019e'# -> unI64 544- '\x01a1'# -> unI64 416- '\x01a3'# -> unI64 418- '\x01a5'# -> unI64 420- '\x01a8'# -> unI64 423- '\x01ad'# -> unI64 428- '\x01b0'# -> unI64 431- '\x01b4'# -> unI64 435- '\x01b6'# -> unI64 437- '\x01b9'# -> unI64 440- '\x01bd'# -> unI64 444- '\x01bf'# -> unI64 503- '\x01c4'# -> unI64 453- '\x01c6'# -> unI64 453- '\x01c7'# -> unI64 456- '\x01c9'# -> unI64 456- '\x01ca'# -> unI64 459- '\x01cc'# -> unI64 459- '\x01ce'# -> unI64 461- '\x01d0'# -> unI64 463- '\x01d2'# -> unI64 465- '\x01d4'# -> unI64 467- '\x01d6'# -> unI64 469- '\x01d8'# -> unI64 471- '\x01da'# -> unI64 473- '\x01dc'# -> unI64 475- '\x01dd'# -> unI64 398- '\x01df'# -> unI64 478- '\x01e1'# -> unI64 480- '\x01e3'# -> unI64 482- '\x01e5'# -> unI64 484- '\x01e7'# -> unI64 486- '\x01e9'# -> unI64 488- '\x01eb'# -> unI64 490- '\x01ed'# -> unI64 492- '\x01ef'# -> unI64 494- '\x01f1'# -> unI64 498- '\x01f3'# -> unI64 498- '\x01f5'# -> unI64 500- '\x01f9'# -> unI64 504- '\x01fb'# -> unI64 506- '\x01fd'# -> unI64 508- '\x01ff'# -> unI64 510- '\x0201'# -> unI64 512- '\x0203'# -> unI64 514- '\x0205'# -> unI64 516- '\x0207'# -> unI64 518- '\x0209'# -> unI64 520- '\x020b'# -> unI64 522- '\x020d'# -> unI64 524- '\x020f'# -> unI64 526- '\x0211'# -> unI64 528- '\x0213'# -> unI64 530- '\x0215'# -> unI64 532- '\x0217'# -> unI64 534- '\x0219'# -> unI64 536- '\x021b'# -> unI64 538- '\x021d'# -> unI64 540- '\x021f'# -> unI64 542- '\x0223'# -> unI64 546- '\x0225'# -> unI64 548- '\x0227'# -> unI64 550- '\x0229'# -> unI64 552- '\x022b'# -> unI64 554- '\x022d'# -> unI64 556- '\x022f'# -> unI64 558- '\x0231'# -> unI64 560- '\x0233'# -> unI64 562- '\x023c'# -> unI64 571- '\x023f'# -> unI64 11390- '\x0240'# -> unI64 11391- '\x0242'# -> unI64 577- '\x0247'# -> unI64 582- '\x0249'# -> unI64 584- '\x024b'# -> unI64 586- '\x024d'# -> unI64 588- '\x024f'# -> unI64 590- '\x0250'# -> unI64 11375- '\x0251'# -> unI64 11373- '\x0252'# -> unI64 11376- '\x0253'# -> unI64 385- '\x0254'# -> unI64 390- '\x0256'# -> unI64 393- '\x0257'# -> unI64 394- '\x0259'# -> unI64 399- '\x025b'# -> unI64 400- '\x025c'# -> unI64 42923- '\x0260'# -> unI64 403- '\x0261'# -> unI64 42924- '\x0263'# -> unI64 404- '\x0265'# -> unI64 42893- '\x0266'# -> unI64 42922- '\x0268'# -> unI64 407- '\x0269'# -> unI64 406- '\x026a'# -> unI64 42926- '\x026b'# -> unI64 11362- '\x026c'# -> unI64 42925- '\x026f'# -> unI64 412- '\x0271'# -> unI64 11374- '\x0272'# -> unI64 413- '\x0275'# -> unI64 415- '\x027d'# -> unI64 11364- '\x0280'# -> unI64 422- '\x0282'# -> unI64 42949- '\x0283'# -> unI64 425- '\x0287'# -> unI64 42929- '\x0288'# -> unI64 430- '\x0289'# -> unI64 580- '\x028a'# -> unI64 433- '\x028b'# -> unI64 434- '\x028c'# -> unI64 581- '\x0292'# -> unI64 439- '\x029d'# -> unI64 42930- '\x029e'# -> unI64 42928- '\x0345'# -> unI64 921- '\x0371'# -> unI64 880- '\x0373'# -> unI64 882- '\x0377'# -> unI64 886- '\x037b'# -> unI64 1021- '\x037c'# -> unI64 1022- '\x037d'# -> unI64 1023- '\x03ac'# -> unI64 902- '\x03ad'# -> unI64 904- '\x03ae'# -> unI64 905- '\x03af'# -> unI64 906- '\x03b1'# -> unI64 913- '\x03b2'# -> unI64 914- '\x03b3'# -> unI64 915- '\x03b4'# -> unI64 916- '\x03b5'# -> unI64 917- '\x03b6'# -> unI64 918- '\x03b7'# -> unI64 919- '\x03b8'# -> unI64 920- '\x03b9'# -> unI64 921- '\x03ba'# -> unI64 922- '\x03bb'# -> unI64 923- '\x03bc'# -> unI64 924- '\x03bd'# -> unI64 925- '\x03be'# -> unI64 926- '\x03bf'# -> unI64 927- '\x03c0'# -> unI64 928- '\x03c1'# -> unI64 929- '\x03c2'# -> unI64 931- '\x03c3'# -> unI64 931- '\x03c4'# -> unI64 932- '\x03c5'# -> unI64 933- '\x03c6'# -> unI64 934- '\x03c7'# -> unI64 935- '\x03c8'# -> unI64 936- '\x03c9'# -> unI64 937- '\x03ca'# -> unI64 938- '\x03cb'# -> unI64 939- '\x03cc'# -> unI64 908- '\x03cd'# -> unI64 910- '\x03ce'# -> unI64 911- '\x03d0'# -> unI64 914- '\x03d1'# -> unI64 920- '\x03d5'# -> unI64 934- '\x03d6'# -> unI64 928- '\x03d7'# -> unI64 975- '\x03d9'# -> unI64 984- '\x03db'# -> unI64 986- '\x03dd'# -> unI64 988- '\x03df'# -> unI64 990- '\x03e1'# -> unI64 992- '\x03e3'# -> unI64 994- '\x03e5'# -> unI64 996- '\x03e7'# -> unI64 998- '\x03e9'# -> unI64 1000- '\x03eb'# -> unI64 1002- '\x03ed'# -> unI64 1004- '\x03ef'# -> unI64 1006- '\x03f0'# -> unI64 922- '\x03f1'# -> unI64 929- '\x03f2'# -> unI64 1017- '\x03f3'# -> unI64 895- '\x03f5'# -> unI64 917- '\x03f8'# -> unI64 1015- '\x03fb'# -> unI64 1018- '\x0430'# -> unI64 1040- '\x0431'# -> unI64 1041- '\x0432'# -> unI64 1042- '\x0433'# -> unI64 1043- '\x0434'# -> unI64 1044- '\x0435'# -> unI64 1045- '\x0436'# -> unI64 1046- '\x0437'# -> unI64 1047- '\x0438'# -> unI64 1048- '\x0439'# -> unI64 1049- '\x043a'# -> unI64 1050- '\x043b'# -> unI64 1051- '\x043c'# -> unI64 1052- '\x043d'# -> unI64 1053- '\x043e'# -> unI64 1054- '\x043f'# -> unI64 1055- '\x0440'# -> unI64 1056- '\x0441'# -> unI64 1057- '\x0442'# -> unI64 1058- '\x0443'# -> unI64 1059- '\x0444'# -> unI64 1060- '\x0445'# -> unI64 1061- '\x0446'# -> unI64 1062- '\x0447'# -> unI64 1063- '\x0448'# -> unI64 1064- '\x0449'# -> unI64 1065- '\x044a'# -> unI64 1066- '\x044b'# -> unI64 1067- '\x044c'# -> unI64 1068- '\x044d'# -> unI64 1069- '\x044e'# -> unI64 1070- '\x044f'# -> unI64 1071- '\x0450'# -> unI64 1024- '\x0451'# -> unI64 1025- '\x0452'# -> unI64 1026- '\x0453'# -> unI64 1027- '\x0454'# -> unI64 1028- '\x0455'# -> unI64 1029- '\x0456'# -> unI64 1030- '\x0457'# -> unI64 1031- '\x0458'# -> unI64 1032- '\x0459'# -> unI64 1033- '\x045a'# -> unI64 1034- '\x045b'# -> unI64 1035- '\x045c'# -> unI64 1036- '\x045d'# -> unI64 1037- '\x045e'# -> unI64 1038- '\x045f'# -> unI64 1039- '\x0461'# -> unI64 1120- '\x0463'# -> unI64 1122- '\x0465'# -> unI64 1124- '\x0467'# -> unI64 1126- '\x0469'# -> unI64 1128- '\x046b'# -> unI64 1130- '\x046d'# -> unI64 1132- '\x046f'# -> unI64 1134- '\x0471'# -> unI64 1136- '\x0473'# -> unI64 1138- '\x0475'# -> unI64 1140- '\x0477'# -> unI64 1142- '\x0479'# -> unI64 1144- '\x047b'# -> unI64 1146- '\x047d'# -> unI64 1148- '\x047f'# -> unI64 1150- '\x0481'# -> unI64 1152- '\x048b'# -> unI64 1162- '\x048d'# -> unI64 1164- '\x048f'# -> unI64 1166- '\x0491'# -> unI64 1168- '\x0493'# -> unI64 1170- '\x0495'# -> unI64 1172- '\x0497'# -> unI64 1174- '\x0499'# -> unI64 1176- '\x049b'# -> unI64 1178- '\x049d'# -> unI64 1180- '\x049f'# -> unI64 1182- '\x04a1'# -> unI64 1184- '\x04a3'# -> unI64 1186- '\x04a5'# -> unI64 1188- '\x04a7'# -> unI64 1190- '\x04a9'# -> unI64 1192- '\x04ab'# -> unI64 1194- '\x04ad'# -> unI64 1196- '\x04af'# -> unI64 1198- '\x04b1'# -> unI64 1200- '\x04b3'# -> unI64 1202- '\x04b5'# -> unI64 1204- '\x04b7'# -> unI64 1206- '\x04b9'# -> unI64 1208- '\x04bb'# -> unI64 1210- '\x04bd'# -> unI64 1212- '\x04bf'# -> unI64 1214- '\x04c2'# -> unI64 1217- '\x04c4'# -> unI64 1219- '\x04c6'# -> unI64 1221- '\x04c8'# -> unI64 1223- '\x04ca'# -> unI64 1225- '\x04cc'# -> unI64 1227- '\x04ce'# -> unI64 1229- '\x04cf'# -> unI64 1216- '\x04d1'# -> unI64 1232- '\x04d3'# -> unI64 1234- '\x04d5'# -> unI64 1236- '\x04d7'# -> unI64 1238- '\x04d9'# -> unI64 1240- '\x04db'# -> unI64 1242- '\x04dd'# -> unI64 1244- '\x04df'# -> unI64 1246- '\x04e1'# -> unI64 1248- '\x04e3'# -> unI64 1250- '\x04e5'# -> unI64 1252- '\x04e7'# -> unI64 1254- '\x04e9'# -> unI64 1256- '\x04eb'# -> unI64 1258- '\x04ed'# -> unI64 1260- '\x04ef'# -> unI64 1262- '\x04f1'# -> unI64 1264- '\x04f3'# -> unI64 1266- '\x04f5'# -> unI64 1268- '\x04f7'# -> unI64 1270- '\x04f9'# -> unI64 1272- '\x04fb'# -> unI64 1274- '\x04fd'# -> unI64 1276- '\x04ff'# -> unI64 1278- '\x0501'# -> unI64 1280- '\x0503'# -> unI64 1282- '\x0505'# -> unI64 1284- '\x0507'# -> unI64 1286- '\x0509'# -> unI64 1288- '\x050b'# -> unI64 1290- '\x050d'# -> unI64 1292- '\x050f'# -> unI64 1294- '\x0511'# -> unI64 1296- '\x0513'# -> unI64 1298- '\x0515'# -> unI64 1300- '\x0517'# -> unI64 1302- '\x0519'# -> unI64 1304- '\x051b'# -> unI64 1306- '\x051d'# -> unI64 1308- '\x051f'# -> unI64 1310- '\x0521'# -> unI64 1312- '\x0523'# -> unI64 1314- '\x0525'# -> unI64 1316- '\x0527'# -> unI64 1318- '\x0529'# -> unI64 1320- '\x052b'# -> unI64 1322- '\x052d'# -> unI64 1324- '\x052f'# -> unI64 1326- '\x0561'# -> unI64 1329- '\x0562'# -> unI64 1330- '\x0563'# -> unI64 1331- '\x0564'# -> unI64 1332- '\x0565'# -> unI64 1333- '\x0566'# -> unI64 1334- '\x0567'# -> unI64 1335- '\x0568'# -> unI64 1336- '\x0569'# -> unI64 1337- '\x056a'# -> unI64 1338- '\x056b'# -> unI64 1339- '\x056c'# -> unI64 1340- '\x056d'# -> unI64 1341- '\x056e'# -> unI64 1342- '\x056f'# -> unI64 1343- '\x0570'# -> unI64 1344- '\x0571'# -> unI64 1345- '\x0572'# -> unI64 1346- '\x0573'# -> unI64 1347- '\x0574'# -> unI64 1348- '\x0575'# -> unI64 1349- '\x0576'# -> unI64 1350- '\x0577'# -> unI64 1351- '\x0578'# -> unI64 1352- '\x0579'# -> unI64 1353- '\x057a'# -> unI64 1354- '\x057b'# -> unI64 1355- '\x057c'# -> unI64 1356- '\x057d'# -> unI64 1357- '\x057e'# -> unI64 1358- '\x057f'# -> unI64 1359- '\x0580'# -> unI64 1360- '\x0581'# -> unI64 1361- '\x0582'# -> unI64 1362- '\x0583'# -> unI64 1363- '\x0584'# -> unI64 1364- '\x0585'# -> unI64 1365- '\x0586'# -> unI64 1366- '\x13f8'# -> unI64 5104- '\x13f9'# -> unI64 5105- '\x13fa'# -> unI64 5106- '\x13fb'# -> unI64 5107- '\x13fc'# -> unI64 5108- '\x13fd'# -> unI64 5109- '\x1c80'# -> unI64 1042- '\x1c81'# -> unI64 1044- '\x1c82'# -> unI64 1054- '\x1c83'# -> unI64 1057- '\x1c84'# -> unI64 1058- '\x1c85'# -> unI64 1058- '\x1c86'# -> unI64 1066- '\x1c87'# -> unI64 1122- '\x1c88'# -> unI64 42570- '\x1d79'# -> unI64 42877- '\x1d7d'# -> unI64 11363- '\x1d8e'# -> unI64 42950- '\x1e01'# -> unI64 7680- '\x1e03'# -> unI64 7682- '\x1e05'# -> unI64 7684- '\x1e07'# -> unI64 7686- '\x1e09'# -> unI64 7688- '\x1e0b'# -> unI64 7690- '\x1e0d'# -> unI64 7692- '\x1e0f'# -> unI64 7694- '\x1e11'# -> unI64 7696- '\x1e13'# -> unI64 7698- '\x1e15'# -> unI64 7700- '\x1e17'# -> unI64 7702- '\x1e19'# -> unI64 7704- '\x1e1b'# -> unI64 7706- '\x1e1d'# -> unI64 7708- '\x1e1f'# -> unI64 7710- '\x1e21'# -> unI64 7712- '\x1e23'# -> unI64 7714- '\x1e25'# -> unI64 7716- '\x1e27'# -> unI64 7718- '\x1e29'# -> unI64 7720- '\x1e2b'# -> unI64 7722- '\x1e2d'# -> unI64 7724- '\x1e2f'# -> unI64 7726- '\x1e31'# -> unI64 7728- '\x1e33'# -> unI64 7730- '\x1e35'# -> unI64 7732- '\x1e37'# -> unI64 7734- '\x1e39'# -> unI64 7736- '\x1e3b'# -> unI64 7738- '\x1e3d'# -> unI64 7740- '\x1e3f'# -> unI64 7742- '\x1e41'# -> unI64 7744- '\x1e43'# -> unI64 7746- '\x1e45'# -> unI64 7748- '\x1e47'# -> unI64 7750- '\x1e49'# -> unI64 7752- '\x1e4b'# -> unI64 7754- '\x1e4d'# -> unI64 7756- '\x1e4f'# -> unI64 7758- '\x1e51'# -> unI64 7760- '\x1e53'# -> unI64 7762- '\x1e55'# -> unI64 7764- '\x1e57'# -> unI64 7766- '\x1e59'# -> unI64 7768- '\x1e5b'# -> unI64 7770- '\x1e5d'# -> unI64 7772- '\x1e5f'# -> unI64 7774- '\x1e61'# -> unI64 7776- '\x1e63'# -> unI64 7778- '\x1e65'# -> unI64 7780- '\x1e67'# -> unI64 7782- '\x1e69'# -> unI64 7784- '\x1e6b'# -> unI64 7786- '\x1e6d'# -> unI64 7788- '\x1e6f'# -> unI64 7790- '\x1e71'# -> unI64 7792- '\x1e73'# -> unI64 7794- '\x1e75'# -> unI64 7796- '\x1e77'# -> unI64 7798- '\x1e79'# -> unI64 7800- '\x1e7b'# -> unI64 7802- '\x1e7d'# -> unI64 7804- '\x1e7f'# -> unI64 7806- '\x1e81'# -> unI64 7808- '\x1e83'# -> unI64 7810- '\x1e85'# -> unI64 7812- '\x1e87'# -> unI64 7814- '\x1e89'# -> unI64 7816- '\x1e8b'# -> unI64 7818- '\x1e8d'# -> unI64 7820- '\x1e8f'# -> unI64 7822- '\x1e91'# -> unI64 7824- '\x1e93'# -> unI64 7826- '\x1e95'# -> unI64 7828- '\x1e9b'# -> unI64 7776- '\x1ea1'# -> unI64 7840- '\x1ea3'# -> unI64 7842- '\x1ea5'# -> unI64 7844- '\x1ea7'# -> unI64 7846- '\x1ea9'# -> unI64 7848- '\x1eab'# -> unI64 7850- '\x1ead'# -> unI64 7852- '\x1eaf'# -> unI64 7854- '\x1eb1'# -> unI64 7856- '\x1eb3'# -> unI64 7858- '\x1eb5'# -> unI64 7860- '\x1eb7'# -> unI64 7862- '\x1eb9'# -> unI64 7864- '\x1ebb'# -> unI64 7866- '\x1ebd'# -> unI64 7868- '\x1ebf'# -> unI64 7870- '\x1ec1'# -> unI64 7872- '\x1ec3'# -> unI64 7874- '\x1ec5'# -> unI64 7876- '\x1ec7'# -> unI64 7878- '\x1ec9'# -> unI64 7880- '\x1ecb'# -> unI64 7882- '\x1ecd'# -> unI64 7884- '\x1ecf'# -> unI64 7886- '\x1ed1'# -> unI64 7888- '\x1ed3'# -> unI64 7890- '\x1ed5'# -> unI64 7892- '\x1ed7'# -> unI64 7894- '\x1ed9'# -> unI64 7896- '\x1edb'# -> unI64 7898- '\x1edd'# -> unI64 7900- '\x1edf'# -> unI64 7902- '\x1ee1'# -> unI64 7904- '\x1ee3'# -> unI64 7906- '\x1ee5'# -> unI64 7908- '\x1ee7'# -> unI64 7910- '\x1ee9'# -> unI64 7912- '\x1eeb'# -> unI64 7914- '\x1eed'# -> unI64 7916- '\x1eef'# -> unI64 7918- '\x1ef1'# -> unI64 7920- '\x1ef3'# -> unI64 7922- '\x1ef5'# -> unI64 7924- '\x1ef7'# -> unI64 7926- '\x1ef9'# -> unI64 7928- '\x1efb'# -> unI64 7930- '\x1efd'# -> unI64 7932- '\x1eff'# -> unI64 7934- '\x1f00'# -> unI64 7944- '\x1f01'# -> unI64 7945- '\x1f02'# -> unI64 7946- '\x1f03'# -> unI64 7947- '\x1f04'# -> unI64 7948- '\x1f05'# -> unI64 7949- '\x1f06'# -> unI64 7950- '\x1f07'# -> unI64 7951- '\x1f10'# -> unI64 7960- '\x1f11'# -> unI64 7961- '\x1f12'# -> unI64 7962- '\x1f13'# -> unI64 7963- '\x1f14'# -> unI64 7964- '\x1f15'# -> unI64 7965- '\x1f20'# -> unI64 7976- '\x1f21'# -> unI64 7977- '\x1f22'# -> unI64 7978- '\x1f23'# -> unI64 7979- '\x1f24'# -> unI64 7980- '\x1f25'# -> unI64 7981- '\x1f26'# -> unI64 7982- '\x1f27'# -> unI64 7983- '\x1f30'# -> unI64 7992- '\x1f31'# -> unI64 7993- '\x1f32'# -> unI64 7994- '\x1f33'# -> unI64 7995- '\x1f34'# -> unI64 7996- '\x1f35'# -> unI64 7997- '\x1f36'# -> unI64 7998- '\x1f37'# -> unI64 7999- '\x1f40'# -> unI64 8008- '\x1f41'# -> unI64 8009- '\x1f42'# -> unI64 8010- '\x1f43'# -> unI64 8011- '\x1f44'# -> unI64 8012- '\x1f45'# -> unI64 8013- '\x1f51'# -> unI64 8025- '\x1f53'# -> unI64 8027- '\x1f55'# -> unI64 8029- '\x1f57'# -> unI64 8031- '\x1f60'# -> unI64 8040- '\x1f61'# -> unI64 8041- '\x1f62'# -> unI64 8042- '\x1f63'# -> unI64 8043- '\x1f64'# -> unI64 8044- '\x1f65'# -> unI64 8045- '\x1f66'# -> unI64 8046- '\x1f67'# -> unI64 8047- '\x1f70'# -> unI64 8122- '\x1f71'# -> unI64 8123- '\x1f72'# -> unI64 8136- '\x1f73'# -> unI64 8137- '\x1f74'# -> unI64 8138- '\x1f75'# -> unI64 8139- '\x1f76'# -> unI64 8154- '\x1f77'# -> unI64 8155- '\x1f78'# -> unI64 8184- '\x1f79'# -> unI64 8185- '\x1f7a'# -> unI64 8170- '\x1f7b'# -> unI64 8171- '\x1f7c'# -> unI64 8186- '\x1f7d'# -> unI64 8187- '\x1f80'# -> unI64 8072- '\x1f81'# -> unI64 8073- '\x1f82'# -> unI64 8074- '\x1f83'# -> unI64 8075- '\x1f84'# -> unI64 8076- '\x1f85'# -> unI64 8077- '\x1f86'# -> unI64 8078- '\x1f87'# -> unI64 8079- '\x1f90'# -> unI64 8088- '\x1f91'# -> unI64 8089- '\x1f92'# -> unI64 8090- '\x1f93'# -> unI64 8091- '\x1f94'# -> unI64 8092- '\x1f95'# -> unI64 8093- '\x1f96'# -> unI64 8094- '\x1f97'# -> unI64 8095- '\x1fa0'# -> unI64 8104- '\x1fa1'# -> unI64 8105- '\x1fa2'# -> unI64 8106- '\x1fa3'# -> unI64 8107- '\x1fa4'# -> unI64 8108- '\x1fa5'# -> unI64 8109- '\x1fa6'# -> unI64 8110- '\x1fa7'# -> unI64 8111- '\x1fb0'# -> unI64 8120- '\x1fb1'# -> unI64 8121- '\x1fb3'# -> unI64 8124- '\x1fbe'# -> unI64 921- '\x1fc3'# -> unI64 8140- '\x1fd0'# -> unI64 8152- '\x1fd1'# -> unI64 8153- '\x1fe0'# -> unI64 8168- '\x1fe1'# -> unI64 8169- '\x1fe5'# -> unI64 8172- '\x1ff3'# -> unI64 8188- '\x214e'# -> unI64 8498- '\x2170'# -> unI64 8544- '\x2171'# -> unI64 8545- '\x2172'# -> unI64 8546- '\x2173'# -> unI64 8547- '\x2174'# -> unI64 8548- '\x2175'# -> unI64 8549- '\x2176'# -> unI64 8550- '\x2177'# -> unI64 8551- '\x2178'# -> unI64 8552- '\x2179'# -> unI64 8553- '\x217a'# -> unI64 8554- '\x217b'# -> unI64 8555- '\x217c'# -> unI64 8556- '\x217d'# -> unI64 8557- '\x217e'# -> unI64 8558- '\x217f'# -> unI64 8559- '\x2184'# -> unI64 8579- '\x24d0'# -> unI64 9398- '\x24d1'# -> unI64 9399- '\x24d2'# -> unI64 9400- '\x24d3'# -> unI64 9401- '\x24d4'# -> unI64 9402- '\x24d5'# -> unI64 9403- '\x24d6'# -> unI64 9404- '\x24d7'# -> unI64 9405- '\x24d8'# -> unI64 9406- '\x24d9'# -> unI64 9407- '\x24da'# -> unI64 9408- '\x24db'# -> unI64 9409- '\x24dc'# -> unI64 9410- '\x24dd'# -> unI64 9411- '\x24de'# -> unI64 9412- '\x24df'# -> unI64 9413- '\x24e0'# -> unI64 9414- '\x24e1'# -> unI64 9415- '\x24e2'# -> unI64 9416- '\x24e3'# -> unI64 9417- '\x24e4'# -> unI64 9418- '\x24e5'# -> unI64 9419- '\x24e6'# -> unI64 9420- '\x24e7'# -> unI64 9421- '\x24e8'# -> unI64 9422- '\x24e9'# -> unI64 9423- '\x2c30'# -> unI64 11264- '\x2c31'# -> unI64 11265- '\x2c32'# -> unI64 11266- '\x2c33'# -> unI64 11267- '\x2c34'# -> unI64 11268- '\x2c35'# -> unI64 11269- '\x2c36'# -> unI64 11270- '\x2c37'# -> unI64 11271- '\x2c38'# -> unI64 11272- '\x2c39'# -> unI64 11273- '\x2c3a'# -> unI64 11274- '\x2c3b'# -> unI64 11275- '\x2c3c'# -> unI64 11276- '\x2c3d'# -> unI64 11277- '\x2c3e'# -> unI64 11278- '\x2c3f'# -> unI64 11279- '\x2c40'# -> unI64 11280- '\x2c41'# -> unI64 11281- '\x2c42'# -> unI64 11282- '\x2c43'# -> unI64 11283- '\x2c44'# -> unI64 11284- '\x2c45'# -> unI64 11285- '\x2c46'# -> unI64 11286- '\x2c47'# -> unI64 11287- '\x2c48'# -> unI64 11288- '\x2c49'# -> unI64 11289- '\x2c4a'# -> unI64 11290- '\x2c4b'# -> unI64 11291- '\x2c4c'# -> unI64 11292- '\x2c4d'# -> unI64 11293- '\x2c4e'# -> unI64 11294- '\x2c4f'# -> unI64 11295- '\x2c50'# -> unI64 11296- '\x2c51'# -> unI64 11297- '\x2c52'# -> unI64 11298- '\x2c53'# -> unI64 11299- '\x2c54'# -> unI64 11300- '\x2c55'# -> unI64 11301- '\x2c56'# -> unI64 11302- '\x2c57'# -> unI64 11303- '\x2c58'# -> unI64 11304- '\x2c59'# -> unI64 11305- '\x2c5a'# -> unI64 11306- '\x2c5b'# -> unI64 11307- '\x2c5c'# -> unI64 11308- '\x2c5d'# -> unI64 11309- '\x2c5e'# -> unI64 11310- '\x2c5f'# -> unI64 11311- '\x2c61'# -> unI64 11360- '\x2c65'# -> unI64 570- '\x2c66'# -> unI64 574- '\x2c68'# -> unI64 11367- '\x2c6a'# -> unI64 11369- '\x2c6c'# -> unI64 11371- '\x2c73'# -> unI64 11378- '\x2c76'# -> unI64 11381- '\x2c81'# -> unI64 11392- '\x2c83'# -> unI64 11394- '\x2c85'# -> unI64 11396- '\x2c87'# -> unI64 11398- '\x2c89'# -> unI64 11400- '\x2c8b'# -> unI64 11402- '\x2c8d'# -> unI64 11404- '\x2c8f'# -> unI64 11406- '\x2c91'# -> unI64 11408- '\x2c93'# -> unI64 11410- '\x2c95'# -> unI64 11412- '\x2c97'# -> unI64 11414- '\x2c99'# -> unI64 11416- '\x2c9b'# -> unI64 11418- '\x2c9d'# -> unI64 11420- '\x2c9f'# -> unI64 11422- '\x2ca1'# -> unI64 11424- '\x2ca3'# -> unI64 11426- '\x2ca5'# -> unI64 11428- '\x2ca7'# -> unI64 11430- '\x2ca9'# -> unI64 11432- '\x2cab'# -> unI64 11434- '\x2cad'# -> unI64 11436- '\x2caf'# -> unI64 11438- '\x2cb1'# -> unI64 11440- '\x2cb3'# -> unI64 11442- '\x2cb5'# -> unI64 11444- '\x2cb7'# -> unI64 11446- '\x2cb9'# -> unI64 11448- '\x2cbb'# -> unI64 11450- '\x2cbd'# -> unI64 11452- '\x2cbf'# -> unI64 11454- '\x2cc1'# -> unI64 11456- '\x2cc3'# -> unI64 11458- '\x2cc5'# -> unI64 11460- '\x2cc7'# -> unI64 11462- '\x2cc9'# -> unI64 11464- '\x2ccb'# -> unI64 11466- '\x2ccd'# -> unI64 11468- '\x2ccf'# -> unI64 11470- '\x2cd1'# -> unI64 11472- '\x2cd3'# -> unI64 11474- '\x2cd5'# -> unI64 11476- '\x2cd7'# -> unI64 11478- '\x2cd9'# -> unI64 11480- '\x2cdb'# -> unI64 11482- '\x2cdd'# -> unI64 11484- '\x2cdf'# -> unI64 11486- '\x2ce1'# -> unI64 11488- '\x2ce3'# -> unI64 11490- '\x2cec'# -> unI64 11499- '\x2cee'# -> unI64 11501- '\x2cf3'# -> unI64 11506- '\x2d00'# -> unI64 4256- '\x2d01'# -> unI64 4257- '\x2d02'# -> unI64 4258- '\x2d03'# -> unI64 4259- '\x2d04'# -> unI64 4260- '\x2d05'# -> unI64 4261- '\x2d06'# -> unI64 4262- '\x2d07'# -> unI64 4263- '\x2d08'# -> unI64 4264- '\x2d09'# -> unI64 4265- '\x2d0a'# -> unI64 4266- '\x2d0b'# -> unI64 4267- '\x2d0c'# -> unI64 4268- '\x2d0d'# -> unI64 4269- '\x2d0e'# -> unI64 4270- '\x2d0f'# -> unI64 4271- '\x2d10'# -> unI64 4272- '\x2d11'# -> unI64 4273- '\x2d12'# -> unI64 4274- '\x2d13'# -> unI64 4275- '\x2d14'# -> unI64 4276- '\x2d15'# -> unI64 4277- '\x2d16'# -> unI64 4278- '\x2d17'# -> unI64 4279- '\x2d18'# -> unI64 4280- '\x2d19'# -> unI64 4281- '\x2d1a'# -> unI64 4282- '\x2d1b'# -> unI64 4283- '\x2d1c'# -> unI64 4284- '\x2d1d'# -> unI64 4285- '\x2d1e'# -> unI64 4286- '\x2d1f'# -> unI64 4287- '\x2d20'# -> unI64 4288- '\x2d21'# -> unI64 4289- '\x2d22'# -> unI64 4290- '\x2d23'# -> unI64 4291- '\x2d24'# -> unI64 4292- '\x2d25'# -> unI64 4293- '\x2d27'# -> unI64 4295- '\x2d2d'# -> unI64 4301- '\xa641'# -> unI64 42560- '\xa643'# -> unI64 42562- '\xa645'# -> unI64 42564- '\xa647'# -> unI64 42566- '\xa649'# -> unI64 42568- '\xa64b'# -> unI64 42570- '\xa64d'# -> unI64 42572- '\xa64f'# -> unI64 42574- '\xa651'# -> unI64 42576- '\xa653'# -> unI64 42578- '\xa655'# -> unI64 42580- '\xa657'# -> unI64 42582- '\xa659'# -> unI64 42584- '\xa65b'# -> unI64 42586- '\xa65d'# -> unI64 42588- '\xa65f'# -> unI64 42590- '\xa661'# -> unI64 42592- '\xa663'# -> unI64 42594- '\xa665'# -> unI64 42596- '\xa667'# -> unI64 42598- '\xa669'# -> unI64 42600- '\xa66b'# -> unI64 42602- '\xa66d'# -> unI64 42604- '\xa681'# -> unI64 42624- '\xa683'# -> unI64 42626- '\xa685'# -> unI64 42628- '\xa687'# -> unI64 42630- '\xa689'# -> unI64 42632- '\xa68b'# -> unI64 42634- '\xa68d'# -> unI64 42636- '\xa68f'# -> unI64 42638- '\xa691'# -> unI64 42640- '\xa693'# -> unI64 42642- '\xa695'# -> unI64 42644- '\xa697'# -> unI64 42646- '\xa699'# -> unI64 42648- '\xa69b'# -> unI64 42650- '\xa723'# -> unI64 42786- '\xa725'# -> unI64 42788- '\xa727'# -> unI64 42790- '\xa729'# -> unI64 42792- '\xa72b'# -> unI64 42794- '\xa72d'# -> unI64 42796- '\xa72f'# -> unI64 42798- '\xa733'# -> unI64 42802- '\xa735'# -> unI64 42804- '\xa737'# -> unI64 42806- '\xa739'# -> unI64 42808- '\xa73b'# -> unI64 42810- '\xa73d'# -> unI64 42812- '\xa73f'# -> unI64 42814- '\xa741'# -> unI64 42816- '\xa743'# -> unI64 42818- '\xa745'# -> unI64 42820- '\xa747'# -> unI64 42822- '\xa749'# -> unI64 42824- '\xa74b'# -> unI64 42826- '\xa74d'# -> unI64 42828- '\xa74f'# -> unI64 42830- '\xa751'# -> unI64 42832- '\xa753'# -> unI64 42834- '\xa755'# -> unI64 42836- '\xa757'# -> unI64 42838- '\xa759'# -> unI64 42840- '\xa75b'# -> unI64 42842- '\xa75d'# -> unI64 42844- '\xa75f'# -> unI64 42846- '\xa761'# -> unI64 42848- '\xa763'# -> unI64 42850- '\xa765'# -> unI64 42852- '\xa767'# -> unI64 42854- '\xa769'# -> unI64 42856- '\xa76b'# -> unI64 42858- '\xa76d'# -> unI64 42860- '\xa76f'# -> unI64 42862- '\xa77a'# -> unI64 42873- '\xa77c'# -> unI64 42875- '\xa77f'# -> unI64 42878- '\xa781'# -> unI64 42880- '\xa783'# -> unI64 42882- '\xa785'# -> unI64 42884- '\xa787'# -> unI64 42886- '\xa78c'# -> unI64 42891- '\xa791'# -> unI64 42896- '\xa793'# -> unI64 42898- '\xa794'# -> unI64 42948- '\xa797'# -> unI64 42902- '\xa799'# -> unI64 42904- '\xa79b'# -> unI64 42906- '\xa79d'# -> unI64 42908- '\xa79f'# -> unI64 42910- '\xa7a1'# -> unI64 42912- '\xa7a3'# -> unI64 42914- '\xa7a5'# -> unI64 42916- '\xa7a7'# -> unI64 42918- '\xa7a9'# -> unI64 42920- '\xa7b5'# -> unI64 42932- '\xa7b7'# -> unI64 42934- '\xa7b9'# -> unI64 42936- '\xa7bb'# -> unI64 42938- '\xa7bd'# -> unI64 42940- '\xa7bf'# -> unI64 42942- '\xa7c1'# -> unI64 42944- '\xa7c3'# -> unI64 42946- '\xa7c8'# -> unI64 42951- '\xa7ca'# -> unI64 42953- '\xa7d1'# -> unI64 42960- '\xa7d7'# -> unI64 42966- '\xa7d9'# -> unI64 42968- '\xa7f6'# -> unI64 42997- '\xab53'# -> unI64 42931- '\xab70'# -> unI64 5024- '\xab71'# -> unI64 5025- '\xab72'# -> unI64 5026- '\xab73'# -> unI64 5027- '\xab74'# -> unI64 5028- '\xab75'# -> unI64 5029- '\xab76'# -> unI64 5030- '\xab77'# -> unI64 5031- '\xab78'# -> unI64 5032- '\xab79'# -> unI64 5033- '\xab7a'# -> unI64 5034- '\xab7b'# -> unI64 5035- '\xab7c'# -> unI64 5036- '\xab7d'# -> unI64 5037- '\xab7e'# -> unI64 5038- '\xab7f'# -> unI64 5039- '\xab80'# -> unI64 5040- '\xab81'# -> unI64 5041- '\xab82'# -> unI64 5042- '\xab83'# -> unI64 5043- '\xab84'# -> unI64 5044- '\xab85'# -> unI64 5045- '\xab86'# -> unI64 5046- '\xab87'# -> unI64 5047- '\xab88'# -> unI64 5048- '\xab89'# -> unI64 5049- '\xab8a'# -> unI64 5050- '\xab8b'# -> unI64 5051- '\xab8c'# -> unI64 5052- '\xab8d'# -> unI64 5053- '\xab8e'# -> unI64 5054- '\xab8f'# -> unI64 5055- '\xab90'# -> unI64 5056- '\xab91'# -> unI64 5057- '\xab92'# -> unI64 5058- '\xab93'# -> unI64 5059- '\xab94'# -> unI64 5060- '\xab95'# -> unI64 5061- '\xab96'# -> unI64 5062- '\xab97'# -> unI64 5063- '\xab98'# -> unI64 5064- '\xab99'# -> unI64 5065- '\xab9a'# -> unI64 5066- '\xab9b'# -> unI64 5067- '\xab9c'# -> unI64 5068- '\xab9d'# -> unI64 5069- '\xab9e'# -> unI64 5070- '\xab9f'# -> unI64 5071- '\xaba0'# -> unI64 5072- '\xaba1'# -> unI64 5073- '\xaba2'# -> unI64 5074- '\xaba3'# -> unI64 5075- '\xaba4'# -> unI64 5076- '\xaba5'# -> unI64 5077- '\xaba6'# -> unI64 5078- '\xaba7'# -> unI64 5079- '\xaba8'# -> unI64 5080- '\xaba9'# -> unI64 5081- '\xabaa'# -> unI64 5082- '\xabab'# -> unI64 5083- '\xabac'# -> unI64 5084- '\xabad'# -> unI64 5085- '\xabae'# -> unI64 5086- '\xabaf'# -> unI64 5087- '\xabb0'# -> unI64 5088- '\xabb1'# -> unI64 5089- '\xabb2'# -> unI64 5090- '\xabb3'# -> unI64 5091- '\xabb4'# -> unI64 5092- '\xabb5'# -> unI64 5093- '\xabb6'# -> unI64 5094- '\xabb7'# -> unI64 5095- '\xabb8'# -> unI64 5096- '\xabb9'# -> unI64 5097- '\xabba'# -> unI64 5098- '\xabbb'# -> unI64 5099- '\xabbc'# -> unI64 5100- '\xabbd'# -> unI64 5101- '\xabbe'# -> unI64 5102- '\xabbf'# -> unI64 5103- '\xff41'# -> unI64 65313- '\xff42'# -> unI64 65314- '\xff43'# -> unI64 65315- '\xff44'# -> unI64 65316- '\xff45'# -> unI64 65317- '\xff46'# -> unI64 65318- '\xff47'# -> unI64 65319- '\xff48'# -> unI64 65320- '\xff49'# -> unI64 65321- '\xff4a'# -> unI64 65322- '\xff4b'# -> unI64 65323- '\xff4c'# -> unI64 65324- '\xff4d'# -> unI64 65325- '\xff4e'# -> unI64 65326- '\xff4f'# -> unI64 65327- '\xff50'# -> unI64 65328- '\xff51'# -> unI64 65329- '\xff52'# -> unI64 65330- '\xff53'# -> unI64 65331- '\xff54'# -> unI64 65332- '\xff55'# -> unI64 65333- '\xff56'# -> unI64 65334- '\xff57'# -> unI64 65335- '\xff58'# -> unI64 65336- '\xff59'# -> unI64 65337- '\xff5a'# -> unI64 65338- '\x10428'# -> unI64 66560- '\x10429'# -> unI64 66561- '\x1042a'# -> unI64 66562- '\x1042b'# -> unI64 66563- '\x1042c'# -> unI64 66564- '\x1042d'# -> unI64 66565- '\x1042e'# -> unI64 66566- '\x1042f'# -> unI64 66567- '\x10430'# -> unI64 66568- '\x10431'# -> unI64 66569- '\x10432'# -> unI64 66570- '\x10433'# -> unI64 66571- '\x10434'# -> unI64 66572- '\x10435'# -> unI64 66573- '\x10436'# -> unI64 66574- '\x10437'# -> unI64 66575- '\x10438'# -> unI64 66576- '\x10439'# -> unI64 66577- '\x1043a'# -> unI64 66578- '\x1043b'# -> unI64 66579- '\x1043c'# -> unI64 66580- '\x1043d'# -> unI64 66581- '\x1043e'# -> unI64 66582- '\x1043f'# -> unI64 66583- '\x10440'# -> unI64 66584- '\x10441'# -> unI64 66585- '\x10442'# -> unI64 66586- '\x10443'# -> unI64 66587- '\x10444'# -> unI64 66588- '\x10445'# -> unI64 66589- '\x10446'# -> unI64 66590- '\x10447'# -> unI64 66591- '\x10448'# -> unI64 66592- '\x10449'# -> unI64 66593- '\x1044a'# -> unI64 66594- '\x1044b'# -> unI64 66595- '\x1044c'# -> unI64 66596- '\x1044d'# -> unI64 66597- '\x1044e'# -> unI64 66598- '\x1044f'# -> unI64 66599- '\x104d8'# -> unI64 66736- '\x104d9'# -> unI64 66737- '\x104da'# -> unI64 66738- '\x104db'# -> unI64 66739- '\x104dc'# -> unI64 66740- '\x104dd'# -> unI64 66741- '\x104de'# -> unI64 66742- '\x104df'# -> unI64 66743- '\x104e0'# -> unI64 66744- '\x104e1'# -> unI64 66745- '\x104e2'# -> unI64 66746- '\x104e3'# -> unI64 66747- '\x104e4'# -> unI64 66748- '\x104e5'# -> unI64 66749- '\x104e6'# -> unI64 66750- '\x104e7'# -> unI64 66751- '\x104e8'# -> unI64 66752- '\x104e9'# -> unI64 66753- '\x104ea'# -> unI64 66754- '\x104eb'# -> unI64 66755- '\x104ec'# -> unI64 66756- '\x104ed'# -> unI64 66757- '\x104ee'# -> unI64 66758- '\x104ef'# -> unI64 66759- '\x104f0'# -> unI64 66760- '\x104f1'# -> unI64 66761- '\x104f2'# -> unI64 66762- '\x104f3'# -> unI64 66763- '\x104f4'# -> unI64 66764- '\x104f5'# -> unI64 66765- '\x104f6'# -> unI64 66766- '\x104f7'# -> unI64 66767- '\x104f8'# -> unI64 66768- '\x104f9'# -> unI64 66769- '\x104fa'# -> unI64 66770- '\x104fb'# -> unI64 66771- '\x10597'# -> unI64 66928- '\x10598'# -> unI64 66929- '\x10599'# -> unI64 66930- '\x1059a'# -> unI64 66931- '\x1059b'# -> unI64 66932- '\x1059c'# -> unI64 66933- '\x1059d'# -> unI64 66934- '\x1059e'# -> unI64 66935- '\x1059f'# -> unI64 66936- '\x105a0'# -> unI64 66937- '\x105a1'# -> unI64 66938- '\x105a3'# -> unI64 66940- '\x105a4'# -> unI64 66941- '\x105a5'# -> unI64 66942- '\x105a6'# -> unI64 66943- '\x105a7'# -> unI64 66944- '\x105a8'# -> unI64 66945- '\x105a9'# -> unI64 66946- '\x105aa'# -> unI64 66947- '\x105ab'# -> unI64 66948- '\x105ac'# -> unI64 66949- '\x105ad'# -> unI64 66950- '\x105ae'# -> unI64 66951- '\x105af'# -> unI64 66952- '\x105b0'# -> unI64 66953- '\x105b1'# -> unI64 66954- '\x105b3'# -> unI64 66956- '\x105b4'# -> unI64 66957- '\x105b5'# -> unI64 66958- '\x105b6'# -> unI64 66959- '\x105b7'# -> unI64 66960- '\x105b8'# -> unI64 66961- '\x105b9'# -> unI64 66962- '\x105bb'# -> unI64 66964- '\x105bc'# -> unI64 66965- '\x10cc0'# -> unI64 68736- '\x10cc1'# -> unI64 68737- '\x10cc2'# -> unI64 68738- '\x10cc3'# -> unI64 68739- '\x10cc4'# -> unI64 68740- '\x10cc5'# -> unI64 68741- '\x10cc6'# -> unI64 68742- '\x10cc7'# -> unI64 68743- '\x10cc8'# -> unI64 68744- '\x10cc9'# -> unI64 68745- '\x10cca'# -> unI64 68746- '\x10ccb'# -> unI64 68747- '\x10ccc'# -> unI64 68748- '\x10ccd'# -> unI64 68749- '\x10cce'# -> unI64 68750- '\x10ccf'# -> unI64 68751- '\x10cd0'# -> unI64 68752- '\x10cd1'# -> unI64 68753- '\x10cd2'# -> unI64 68754- '\x10cd3'# -> unI64 68755- '\x10cd4'# -> unI64 68756- '\x10cd5'# -> unI64 68757- '\x10cd6'# -> unI64 68758- '\x10cd7'# -> unI64 68759- '\x10cd8'# -> unI64 68760- '\x10cd9'# -> unI64 68761- '\x10cda'# -> unI64 68762- '\x10cdb'# -> unI64 68763- '\x10cdc'# -> unI64 68764- '\x10cdd'# -> unI64 68765- '\x10cde'# -> unI64 68766- '\x10cdf'# -> unI64 68767- '\x10ce0'# -> unI64 68768- '\x10ce1'# -> unI64 68769- '\x10ce2'# -> unI64 68770- '\x10ce3'# -> unI64 68771- '\x10ce4'# -> unI64 68772- '\x10ce5'# -> unI64 68773- '\x10ce6'# -> unI64 68774- '\x10ce7'# -> unI64 68775- '\x10ce8'# -> unI64 68776- '\x10ce9'# -> unI64 68777- '\x10cea'# -> unI64 68778- '\x10ceb'# -> unI64 68779- '\x10cec'# -> unI64 68780- '\x10ced'# -> unI64 68781- '\x10cee'# -> unI64 68782- '\x10cef'# -> unI64 68783- '\x10cf0'# -> unI64 68784- '\x10cf1'# -> unI64 68785- '\x10cf2'# -> unI64 68786- '\x118c0'# -> unI64 71840- '\x118c1'# -> unI64 71841- '\x118c2'# -> unI64 71842- '\x118c3'# -> unI64 71843- '\x118c4'# -> unI64 71844- '\x118c5'# -> unI64 71845- '\x118c6'# -> unI64 71846- '\x118c7'# -> unI64 71847- '\x118c8'# -> unI64 71848- '\x118c9'# -> unI64 71849- '\x118ca'# -> unI64 71850- '\x118cb'# -> unI64 71851- '\x118cc'# -> unI64 71852- '\x118cd'# -> unI64 71853- '\x118ce'# -> unI64 71854- '\x118cf'# -> unI64 71855- '\x118d0'# -> unI64 71856- '\x118d1'# -> unI64 71857- '\x118d2'# -> unI64 71858- '\x118d3'# -> unI64 71859- '\x118d4'# -> unI64 71860- '\x118d5'# -> unI64 71861- '\x118d6'# -> unI64 71862- '\x118d7'# -> unI64 71863- '\x118d8'# -> unI64 71864- '\x118d9'# -> unI64 71865- '\x118da'# -> unI64 71866- '\x118db'# -> unI64 71867- '\x118dc'# -> unI64 71868- '\x118dd'# -> unI64 71869- '\x118de'# -> unI64 71870- '\x118df'# -> unI64 71871- '\x16e60'# -> unI64 93760- '\x16e61'# -> unI64 93761- '\x16e62'# -> unI64 93762- '\x16e63'# -> unI64 93763- '\x16e64'# -> unI64 93764- '\x16e65'# -> unI64 93765- '\x16e66'# -> unI64 93766- '\x16e67'# -> unI64 93767- '\x16e68'# -> unI64 93768- '\x16e69'# -> unI64 93769- '\x16e6a'# -> unI64 93770- '\x16e6b'# -> unI64 93771- '\x16e6c'# -> unI64 93772- '\x16e6d'# -> unI64 93773- '\x16e6e'# -> unI64 93774- '\x16e6f'# -> unI64 93775- '\x16e70'# -> unI64 93776- '\x16e71'# -> unI64 93777- '\x16e72'# -> unI64 93778- '\x16e73'# -> unI64 93779- '\x16e74'# -> unI64 93780- '\x16e75'# -> unI64 93781- '\x16e76'# -> unI64 93782- '\x16e77'# -> unI64 93783- '\x16e78'# -> unI64 93784- '\x16e79'# -> unI64 93785- '\x16e7a'# -> unI64 93786- '\x16e7b'# -> unI64 93787- '\x16e7c'# -> unI64 93788- '\x16e7d'# -> unI64 93789- '\x16e7e'# -> unI64 93790- '\x16e7f'# -> unI64 93791- '\x1e922'# -> unI64 125184- '\x1e923'# -> unI64 125185- '\x1e924'# -> unI64 125186- '\x1e925'# -> unI64 125187- '\x1e926'# -> unI64 125188- '\x1e927'# -> unI64 125189- '\x1e928'# -> unI64 125190- '\x1e929'# -> unI64 125191- '\x1e92a'# -> unI64 125192- '\x1e92b'# -> unI64 125193- '\x1e92c'# -> unI64 125194- '\x1e92d'# -> unI64 125195- '\x1e92e'# -> unI64 125196- '\x1e92f'# -> unI64 125197- '\x1e930'# -> unI64 125198- '\x1e931'# -> unI64 125199- '\x1e932'# -> unI64 125200- '\x1e933'# -> unI64 125201- '\x1e934'# -> unI64 125202- '\x1e935'# -> unI64 125203- '\x1e936'# -> unI64 125204- '\x1e937'# -> unI64 125205- '\x1e938'# -> unI64 125206- '\x1e939'# -> unI64 125207- '\x1e93a'# -> unI64 125208- '\x1e93b'# -> unI64 125209- '\x1e93c'# -> unI64 125210- '\x1e93d'# -> unI64 125211- '\x1e93e'# -> unI64 125212- '\x1e93f'# -> unI64 125213- '\x1e940'# -> unI64 125214- '\x1e941'# -> unI64 125215- '\x1e942'# -> unI64 125216- '\x1e943'# -> unI64 125217- _ -> unI64 0-foldMapping :: Char# -> _ {- unboxed Int64 -}-{-# NOINLINE foldMapping #-}-foldMapping = \case- -- LATIN CAPITAL LETTER A- '\x0041'# -> unI64 97- -- LATIN CAPITAL LETTER B- '\x0042'# -> unI64 98- -- LATIN CAPITAL LETTER C- '\x0043'# -> unI64 99- -- LATIN CAPITAL LETTER D- '\x0044'# -> unI64 100- -- LATIN CAPITAL LETTER E- '\x0045'# -> unI64 101- -- LATIN CAPITAL LETTER F- '\x0046'# -> unI64 102- -- LATIN CAPITAL LETTER G- '\x0047'# -> unI64 103- -- LATIN CAPITAL LETTER H- '\x0048'# -> unI64 104- -- LATIN CAPITAL LETTER I- '\x0049'# -> unI64 105- -- LATIN CAPITAL LETTER J- '\x004a'# -> unI64 106- -- LATIN CAPITAL LETTER K- '\x004b'# -> unI64 107- -- LATIN CAPITAL LETTER L- '\x004c'# -> unI64 108- -- LATIN CAPITAL LETTER M- '\x004d'# -> unI64 109- -- LATIN CAPITAL LETTER N- '\x004e'# -> unI64 110- -- LATIN CAPITAL LETTER O- '\x004f'# -> unI64 111- -- LATIN CAPITAL LETTER P- '\x0050'# -> unI64 112- -- LATIN CAPITAL LETTER Q- '\x0051'# -> unI64 113- -- LATIN CAPITAL LETTER R- '\x0052'# -> unI64 114- -- LATIN CAPITAL LETTER S- '\x0053'# -> unI64 115- -- LATIN CAPITAL LETTER T- '\x0054'# -> unI64 116- -- LATIN CAPITAL LETTER U- '\x0055'# -> unI64 117- -- LATIN CAPITAL LETTER V- '\x0056'# -> unI64 118- -- LATIN CAPITAL LETTER W- '\x0057'# -> unI64 119- -- LATIN CAPITAL LETTER X- '\x0058'# -> unI64 120- -- LATIN CAPITAL LETTER Y- '\x0059'# -> unI64 121- -- LATIN CAPITAL LETTER Z- '\x005a'# -> unI64 122- -- MICRO SIGN- '\x00b5'# -> unI64 956- -- LATIN CAPITAL LETTER A WITH GRAVE- '\x00c0'# -> unI64 224- -- LATIN CAPITAL LETTER A WITH ACUTE- '\x00c1'# -> unI64 225- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX- '\x00c2'# -> unI64 226- -- LATIN CAPITAL LETTER A WITH TILDE- '\x00c3'# -> unI64 227- -- LATIN CAPITAL LETTER A WITH DIAERESIS- '\x00c4'# -> unI64 228- -- LATIN CAPITAL LETTER A WITH RING ABOVE- '\x00c5'# -> unI64 229- -- LATIN CAPITAL LETTER AE- '\x00c6'# -> unI64 230- -- LATIN CAPITAL LETTER C WITH CEDILLA- '\x00c7'# -> unI64 231- -- LATIN CAPITAL LETTER E WITH GRAVE- '\x00c8'# -> unI64 232- -- LATIN CAPITAL LETTER E WITH ACUTE- '\x00c9'# -> unI64 233- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX- '\x00ca'# -> unI64 234- -- LATIN CAPITAL LETTER E WITH DIAERESIS- '\x00cb'# -> unI64 235- -- LATIN CAPITAL LETTER I WITH GRAVE- '\x00cc'# -> unI64 236- -- LATIN CAPITAL LETTER I WITH ACUTE- '\x00cd'# -> unI64 237- -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX- '\x00ce'# -> unI64 238- -- LATIN CAPITAL LETTER I WITH DIAERESIS- '\x00cf'# -> unI64 239- -- LATIN CAPITAL LETTER ETH- '\x00d0'# -> unI64 240- -- LATIN CAPITAL LETTER N WITH TILDE- '\x00d1'# -> unI64 241- -- LATIN CAPITAL LETTER O WITH GRAVE- '\x00d2'# -> unI64 242- -- LATIN CAPITAL LETTER O WITH ACUTE- '\x00d3'# -> unI64 243- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX- '\x00d4'# -> unI64 244- -- LATIN CAPITAL LETTER O WITH TILDE- '\x00d5'# -> unI64 245- -- LATIN CAPITAL LETTER O WITH DIAERESIS- '\x00d6'# -> unI64 246- -- LATIN CAPITAL LETTER O WITH STROKE- '\x00d8'# -> unI64 248- -- LATIN CAPITAL LETTER U WITH GRAVE- '\x00d9'# -> unI64 249- -- LATIN CAPITAL LETTER U WITH ACUTE- '\x00da'# -> unI64 250- -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX- '\x00db'# -> unI64 251- -- LATIN CAPITAL LETTER U WITH DIAERESIS- '\x00dc'# -> unI64 252- -- LATIN CAPITAL LETTER Y WITH ACUTE- '\x00dd'# -> unI64 253- -- LATIN CAPITAL LETTER THORN- '\x00de'# -> unI64 254- -- LATIN SMALL LETTER SHARP S- '\x00df'# -> unI64 241172595- -- LATIN CAPITAL LETTER A WITH MACRON- '\x0100'# -> unI64 257- -- LATIN CAPITAL LETTER A WITH BREVE- '\x0102'# -> unI64 259- -- LATIN CAPITAL LETTER A WITH OGONEK- '\x0104'# -> unI64 261- -- LATIN CAPITAL LETTER C WITH ACUTE- '\x0106'# -> unI64 263- -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX- '\x0108'# -> unI64 265- -- LATIN CAPITAL LETTER C WITH DOT ABOVE- '\x010a'# -> unI64 267- -- LATIN CAPITAL LETTER C WITH CARON- '\x010c'# -> unI64 269- -- LATIN CAPITAL LETTER D WITH CARON- '\x010e'# -> unI64 271- -- LATIN CAPITAL LETTER D WITH STROKE- '\x0110'# -> unI64 273- -- LATIN CAPITAL LETTER E WITH MACRON- '\x0112'# -> unI64 275- -- LATIN CAPITAL LETTER E WITH BREVE- '\x0114'# -> unI64 277- -- LATIN CAPITAL LETTER E WITH DOT ABOVE- '\x0116'# -> unI64 279- -- LATIN CAPITAL LETTER E WITH OGONEK- '\x0118'# -> unI64 281- -- LATIN CAPITAL LETTER E WITH CARON- '\x011a'# -> unI64 283- -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX- '\x011c'# -> unI64 285- -- LATIN CAPITAL LETTER G WITH BREVE- '\x011e'# -> unI64 287- -- LATIN CAPITAL LETTER G WITH DOT ABOVE- '\x0120'# -> unI64 289- -- LATIN CAPITAL LETTER G WITH CEDILLA- '\x0122'# -> unI64 291- -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX- '\x0124'# -> unI64 293- -- LATIN CAPITAL LETTER H WITH STROKE- '\x0126'# -> unI64 295- -- LATIN CAPITAL LETTER I WITH TILDE- '\x0128'# -> unI64 297- -- LATIN CAPITAL LETTER I WITH MACRON- '\x012a'# -> unI64 299- -- LATIN CAPITAL LETTER I WITH BREVE- '\x012c'# -> unI64 301- -- LATIN CAPITAL LETTER I WITH OGONEK- '\x012e'# -> unI64 303- -- LATIN CAPITAL LETTER I WITH DOT ABOVE- '\x0130'# -> unI64 1625292905- -- LATIN CAPITAL LIGATURE IJ- '\x0132'# -> unI64 307- -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX- '\x0134'# -> unI64 309- -- LATIN CAPITAL LETTER K WITH CEDILLA- '\x0136'# -> unI64 311- -- LATIN CAPITAL LETTER L WITH ACUTE- '\x0139'# -> unI64 314- -- LATIN CAPITAL LETTER L WITH CEDILLA- '\x013b'# -> unI64 316- -- LATIN CAPITAL LETTER L WITH CARON- '\x013d'# -> unI64 318- -- LATIN CAPITAL LETTER L WITH MIDDLE DOT- '\x013f'# -> unI64 320- -- LATIN CAPITAL LETTER L WITH STROKE- '\x0141'# -> unI64 322- -- LATIN CAPITAL LETTER N WITH ACUTE- '\x0143'# -> unI64 324- -- LATIN CAPITAL LETTER N WITH CEDILLA- '\x0145'# -> unI64 326- -- LATIN CAPITAL LETTER N WITH CARON- '\x0147'# -> unI64 328- -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE- '\x0149'# -> unI64 230687420- -- LATIN CAPITAL LETTER ENG- '\x014a'# -> unI64 331- -- LATIN CAPITAL LETTER O WITH MACRON- '\x014c'# -> unI64 333- -- LATIN CAPITAL LETTER O WITH BREVE- '\x014e'# -> unI64 335- -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE- '\x0150'# -> unI64 337- -- LATIN CAPITAL LIGATURE OE- '\x0152'# -> unI64 339- -- LATIN CAPITAL LETTER R WITH ACUTE- '\x0154'# -> unI64 341- -- LATIN CAPITAL LETTER R WITH CEDILLA- '\x0156'# -> unI64 343- -- LATIN CAPITAL LETTER R WITH CARON- '\x0158'# -> unI64 345- -- LATIN CAPITAL LETTER S WITH ACUTE- '\x015a'# -> unI64 347- -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX- '\x015c'# -> unI64 349- -- LATIN CAPITAL LETTER S WITH CEDILLA- '\x015e'# -> unI64 351- -- LATIN CAPITAL LETTER S WITH CARON- '\x0160'# -> unI64 353- -- LATIN CAPITAL LETTER T WITH CEDILLA- '\x0162'# -> unI64 355- -- LATIN CAPITAL LETTER T WITH CARON- '\x0164'# -> unI64 357- -- LATIN CAPITAL LETTER T WITH STROKE- '\x0166'# -> unI64 359- -- LATIN CAPITAL LETTER U WITH TILDE- '\x0168'# -> unI64 361- -- LATIN CAPITAL LETTER U WITH MACRON- '\x016a'# -> unI64 363- -- LATIN CAPITAL LETTER U WITH BREVE- '\x016c'# -> unI64 365- -- LATIN CAPITAL LETTER U WITH RING ABOVE- '\x016e'# -> unI64 367- -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE- '\x0170'# -> unI64 369- -- LATIN CAPITAL LETTER U WITH OGONEK- '\x0172'# -> unI64 371- -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX- '\x0174'# -> unI64 373- -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX- '\x0176'# -> unI64 375- -- LATIN CAPITAL LETTER Y WITH DIAERESIS- '\x0178'# -> unI64 255- -- LATIN CAPITAL LETTER Z WITH ACUTE- '\x0179'# -> unI64 378- -- LATIN CAPITAL LETTER Z WITH DOT ABOVE- '\x017b'# -> unI64 380- -- LATIN CAPITAL LETTER Z WITH CARON- '\x017d'# -> unI64 382- -- LATIN SMALL LETTER LONG S- '\x017f'# -> unI64 115- -- LATIN CAPITAL LETTER B WITH HOOK- '\x0181'# -> unI64 595- -- LATIN CAPITAL LETTER B WITH TOPBAR- '\x0182'# -> unI64 387- -- LATIN CAPITAL LETTER TONE SIX- '\x0184'# -> unI64 389- -- LATIN CAPITAL LETTER OPEN O- '\x0186'# -> unI64 596- -- LATIN CAPITAL LETTER C WITH HOOK- '\x0187'# -> unI64 392- -- LATIN CAPITAL LETTER AFRICAN D- '\x0189'# -> unI64 598- -- LATIN CAPITAL LETTER D WITH HOOK- '\x018a'# -> unI64 599- -- LATIN CAPITAL LETTER D WITH TOPBAR- '\x018b'# -> unI64 396- -- LATIN CAPITAL LETTER REVERSED E- '\x018e'# -> unI64 477- -- LATIN CAPITAL LETTER SCHWA- '\x018f'# -> unI64 601- -- LATIN CAPITAL LETTER OPEN E- '\x0190'# -> unI64 603- -- LATIN CAPITAL LETTER F WITH HOOK- '\x0191'# -> unI64 402- -- LATIN CAPITAL LETTER G WITH HOOK- '\x0193'# -> unI64 608- -- LATIN CAPITAL LETTER GAMMA- '\x0194'# -> unI64 611- -- LATIN CAPITAL LETTER IOTA- '\x0196'# -> unI64 617- -- LATIN CAPITAL LETTER I WITH STROKE- '\x0197'# -> unI64 616- -- LATIN CAPITAL LETTER K WITH HOOK- '\x0198'# -> unI64 409- -- LATIN CAPITAL LETTER TURNED M- '\x019c'# -> unI64 623- -- LATIN CAPITAL LETTER N WITH LEFT HOOK- '\x019d'# -> unI64 626- -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE- '\x019f'# -> unI64 629- -- LATIN CAPITAL LETTER O WITH HORN- '\x01a0'# -> unI64 417- -- LATIN CAPITAL LETTER OI- '\x01a2'# -> unI64 419- -- LATIN CAPITAL LETTER P WITH HOOK- '\x01a4'# -> unI64 421- -- LATIN LETTER YR- '\x01a6'# -> unI64 640- -- LATIN CAPITAL LETTER TONE TWO- '\x01a7'# -> unI64 424- -- LATIN CAPITAL LETTER ESH- '\x01a9'# -> unI64 643- -- LATIN CAPITAL LETTER T WITH HOOK- '\x01ac'# -> unI64 429- -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK- '\x01ae'# -> unI64 648- -- LATIN CAPITAL LETTER U WITH HORN- '\x01af'# -> unI64 432- -- LATIN CAPITAL LETTER UPSILON- '\x01b1'# -> unI64 650- -- LATIN CAPITAL LETTER V WITH HOOK- '\x01b2'# -> unI64 651- -- LATIN CAPITAL LETTER Y WITH HOOK- '\x01b3'# -> unI64 436- -- LATIN CAPITAL LETTER Z WITH STROKE- '\x01b5'# -> unI64 438- -- LATIN CAPITAL LETTER EZH- '\x01b7'# -> unI64 658- -- LATIN CAPITAL LETTER EZH REVERSED- '\x01b8'# -> unI64 441- -- LATIN CAPITAL LETTER TONE FIVE- '\x01bc'# -> unI64 445- -- LATIN CAPITAL LETTER DZ WITH CARON- '\x01c4'# -> unI64 454- -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON- '\x01c5'# -> unI64 454- -- LATIN CAPITAL LETTER LJ- '\x01c7'# -> unI64 457- -- LATIN CAPITAL LETTER L WITH SMALL LETTER J- '\x01c8'# -> unI64 457- -- LATIN CAPITAL LETTER NJ- '\x01ca'# -> unI64 460- -- LATIN CAPITAL LETTER N WITH SMALL LETTER J- '\x01cb'# -> unI64 460- -- LATIN CAPITAL LETTER A WITH CARON- '\x01cd'# -> unI64 462- -- LATIN CAPITAL LETTER I WITH CARON- '\x01cf'# -> unI64 464- -- LATIN CAPITAL LETTER O WITH CARON- '\x01d1'# -> unI64 466- -- LATIN CAPITAL LETTER U WITH CARON- '\x01d3'# -> unI64 468- -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON- '\x01d5'# -> unI64 470- -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE- '\x01d7'# -> unI64 472- -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON- '\x01d9'# -> unI64 474- -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE- '\x01db'# -> unI64 476- -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON- '\x01de'# -> unI64 479- -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON- '\x01e0'# -> unI64 481- -- LATIN CAPITAL LETTER AE WITH MACRON- '\x01e2'# -> unI64 483- -- LATIN CAPITAL LETTER G WITH STROKE- '\x01e4'# -> unI64 485- -- LATIN CAPITAL LETTER G WITH CARON- '\x01e6'# -> unI64 487- -- LATIN CAPITAL LETTER K WITH CARON- '\x01e8'# -> unI64 489- -- LATIN CAPITAL LETTER O WITH OGONEK- '\x01ea'# -> unI64 491- -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON- '\x01ec'# -> unI64 493- -- LATIN CAPITAL LETTER EZH WITH CARON- '\x01ee'# -> unI64 495- -- LATIN SMALL LETTER J WITH CARON- '\x01f0'# -> unI64 1635778666- -- LATIN CAPITAL LETTER DZ- '\x01f1'# -> unI64 499- -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z- '\x01f2'# -> unI64 499- -- LATIN CAPITAL LETTER G WITH ACUTE- '\x01f4'# -> unI64 501- -- LATIN CAPITAL LETTER HWAIR- '\x01f6'# -> unI64 405- -- LATIN CAPITAL LETTER WYNN- '\x01f7'# -> unI64 447- -- LATIN CAPITAL LETTER N WITH GRAVE- '\x01f8'# -> unI64 505- -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE- '\x01fa'# -> unI64 507- -- LATIN CAPITAL LETTER AE WITH ACUTE- '\x01fc'# -> unI64 509- -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE- '\x01fe'# -> unI64 511- -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE- '\x0200'# -> unI64 513- -- LATIN CAPITAL LETTER A WITH INVERTED BREVE- '\x0202'# -> unI64 515- -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE- '\x0204'# -> unI64 517- -- LATIN CAPITAL LETTER E WITH INVERTED BREVE- '\x0206'# -> unI64 519- -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE- '\x0208'# -> unI64 521- -- LATIN CAPITAL LETTER I WITH INVERTED BREVE- '\x020a'# -> unI64 523- -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE- '\x020c'# -> unI64 525- -- LATIN CAPITAL LETTER O WITH INVERTED BREVE- '\x020e'# -> unI64 527- -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE- '\x0210'# -> unI64 529- -- LATIN CAPITAL LETTER R WITH INVERTED BREVE- '\x0212'# -> unI64 531- -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE- '\x0214'# -> unI64 533- -- LATIN CAPITAL LETTER U WITH INVERTED BREVE- '\x0216'# -> unI64 535- -- LATIN CAPITAL LETTER S WITH COMMA BELOW- '\x0218'# -> unI64 537- -- LATIN CAPITAL LETTER T WITH COMMA BELOW- '\x021a'# -> unI64 539- -- LATIN CAPITAL LETTER YOGH- '\x021c'# -> unI64 541- -- LATIN CAPITAL LETTER H WITH CARON- '\x021e'# -> unI64 543- -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG- '\x0220'# -> unI64 414- -- LATIN CAPITAL LETTER OU- '\x0222'# -> unI64 547- -- LATIN CAPITAL LETTER Z WITH HOOK- '\x0224'# -> unI64 549- -- LATIN CAPITAL LETTER A WITH DOT ABOVE- '\x0226'# -> unI64 551- -- LATIN CAPITAL LETTER E WITH CEDILLA- '\x0228'# -> unI64 553- -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON- '\x022a'# -> unI64 555- -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON- '\x022c'# -> unI64 557- -- LATIN CAPITAL LETTER O WITH DOT ABOVE- '\x022e'# -> unI64 559- -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON- '\x0230'# -> unI64 561- -- LATIN CAPITAL LETTER Y WITH MACRON- '\x0232'# -> unI64 563- -- LATIN CAPITAL LETTER A WITH STROKE- '\x023a'# -> unI64 11365- -- LATIN CAPITAL LETTER C WITH STROKE- '\x023b'# -> unI64 572- -- LATIN CAPITAL LETTER L WITH BAR- '\x023d'# -> unI64 410- -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE- '\x023e'# -> unI64 11366- -- LATIN CAPITAL LETTER GLOTTAL STOP- '\x0241'# -> unI64 578- -- LATIN CAPITAL LETTER B WITH STROKE- '\x0243'# -> unI64 384- -- LATIN CAPITAL LETTER U BAR- '\x0244'# -> unI64 649- -- LATIN CAPITAL LETTER TURNED V- '\x0245'# -> unI64 652- -- LATIN CAPITAL LETTER E WITH STROKE- '\x0246'# -> unI64 583- -- LATIN CAPITAL LETTER J WITH STROKE- '\x0248'# -> unI64 585- -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL- '\x024a'# -> unI64 587- -- LATIN CAPITAL LETTER R WITH STROKE- '\x024c'# -> unI64 589- -- LATIN CAPITAL LETTER Y WITH STROKE- '\x024e'# -> unI64 591- -- COMBINING GREEK YPOGEGRAMMENI- '\x0345'# -> unI64 953- -- GREEK CAPITAL LETTER HETA- '\x0370'# -> unI64 881- -- GREEK CAPITAL LETTER ARCHAIC SAMPI- '\x0372'# -> unI64 883- -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA- '\x0376'# -> unI64 887- -- GREEK CAPITAL LETTER YOT- '\x037f'# -> unI64 1011- -- GREEK CAPITAL LETTER ALPHA WITH TONOS- '\x0386'# -> unI64 940- -- GREEK CAPITAL LETTER EPSILON WITH TONOS- '\x0388'# -> unI64 941- -- GREEK CAPITAL LETTER ETA WITH TONOS- '\x0389'# -> unI64 942- -- GREEK CAPITAL LETTER IOTA WITH TONOS- '\x038a'# -> unI64 943- -- GREEK CAPITAL LETTER OMICRON WITH TONOS- '\x038c'# -> unI64 972- -- GREEK CAPITAL LETTER UPSILON WITH TONOS- '\x038e'# -> unI64 973- -- GREEK CAPITAL LETTER OMEGA WITH TONOS- '\x038f'# -> unI64 974- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS- '\x0390'# -> unI64 3382099394429881- -- GREEK CAPITAL LETTER ALPHA- '\x0391'# -> unI64 945- -- GREEK CAPITAL LETTER BETA- '\x0392'# -> unI64 946- -- GREEK CAPITAL LETTER GAMMA- '\x0393'# -> unI64 947- -- GREEK CAPITAL LETTER DELTA- '\x0394'# -> unI64 948- -- GREEK CAPITAL LETTER EPSILON- '\x0395'# -> unI64 949- -- GREEK CAPITAL LETTER ZETA- '\x0396'# -> unI64 950- -- GREEK CAPITAL LETTER ETA- '\x0397'# -> unI64 951- -- GREEK CAPITAL LETTER THETA- '\x0398'# -> unI64 952- -- GREEK CAPITAL LETTER IOTA- '\x0399'# -> unI64 953- -- GREEK CAPITAL LETTER KAPPA- '\x039a'# -> unI64 954- -- GREEK CAPITAL LETTER LAMDA- '\x039b'# -> unI64 955- -- GREEK CAPITAL LETTER MU- '\x039c'# -> unI64 956- -- GREEK CAPITAL LETTER NU- '\x039d'# -> unI64 957- -- GREEK CAPITAL LETTER XI- '\x039e'# -> unI64 958- -- GREEK CAPITAL LETTER OMICRON- '\x039f'# -> unI64 959- -- GREEK CAPITAL LETTER PI- '\x03a0'# -> unI64 960- -- GREEK CAPITAL LETTER RHO- '\x03a1'# -> unI64 961- -- GREEK CAPITAL LETTER SIGMA- '\x03a3'# -> unI64 963- -- GREEK CAPITAL LETTER TAU- '\x03a4'# -> unI64 964- -- GREEK CAPITAL LETTER UPSILON- '\x03a5'# -> unI64 965- -- GREEK CAPITAL LETTER PHI- '\x03a6'# -> unI64 966- -- GREEK CAPITAL LETTER CHI- '\x03a7'# -> unI64 967- -- GREEK CAPITAL LETTER PSI- '\x03a8'# -> unI64 968- -- GREEK CAPITAL LETTER OMEGA- '\x03a9'# -> unI64 969- -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA- '\x03aa'# -> unI64 970- -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA- '\x03ab'# -> unI64 971- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS- '\x03b0'# -> unI64 3382099394429893- -- GREEK SMALL LETTER FINAL SIGMA- '\x03c2'# -> unI64 963- -- GREEK CAPITAL KAI SYMBOL- '\x03cf'# -> unI64 983- -- GREEK BETA SYMBOL- '\x03d0'# -> unI64 946- -- GREEK THETA SYMBOL- '\x03d1'# -> unI64 952- -- GREEK PHI SYMBOL- '\x03d5'# -> unI64 966- -- GREEK PI SYMBOL- '\x03d6'# -> unI64 960- -- GREEK LETTER ARCHAIC KOPPA- '\x03d8'# -> unI64 985- -- GREEK LETTER STIGMA- '\x03da'# -> unI64 987- -- GREEK LETTER DIGAMMA- '\x03dc'# -> unI64 989- -- GREEK LETTER KOPPA- '\x03de'# -> unI64 991- -- GREEK LETTER SAMPI- '\x03e0'# -> unI64 993- -- COPTIC CAPITAL LETTER SHEI- '\x03e2'# -> unI64 995- -- COPTIC CAPITAL LETTER FEI- '\x03e4'# -> unI64 997- -- COPTIC CAPITAL LETTER KHEI- '\x03e6'# -> unI64 999- -- COPTIC CAPITAL LETTER HORI- '\x03e8'# -> unI64 1001- -- COPTIC CAPITAL LETTER GANGIA- '\x03ea'# -> unI64 1003- -- COPTIC CAPITAL LETTER SHIMA- '\x03ec'# -> unI64 1005- -- COPTIC CAPITAL LETTER DEI- '\x03ee'# -> unI64 1007- -- GREEK KAPPA SYMBOL- '\x03f0'# -> unI64 954- -- GREEK RHO SYMBOL- '\x03f1'# -> unI64 961- -- GREEK CAPITAL THETA SYMBOL- '\x03f4'# -> unI64 952- -- GREEK LUNATE EPSILON SYMBOL- '\x03f5'# -> unI64 949- -- GREEK CAPITAL LETTER SHO- '\x03f7'# -> unI64 1016- -- GREEK CAPITAL LUNATE SIGMA SYMBOL- '\x03f9'# -> unI64 1010- -- GREEK CAPITAL LETTER SAN- '\x03fa'# -> unI64 1019- -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL- '\x03fd'# -> unI64 891- -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL- '\x03fe'# -> unI64 892- -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL- '\x03ff'# -> unI64 893- -- CYRILLIC CAPITAL LETTER IE WITH GRAVE- '\x0400'# -> unI64 1104- -- CYRILLIC CAPITAL LETTER IO- '\x0401'# -> unI64 1105- -- CYRILLIC CAPITAL LETTER DJE- '\x0402'# -> unI64 1106- -- CYRILLIC CAPITAL LETTER GJE- '\x0403'# -> unI64 1107- -- CYRILLIC CAPITAL LETTER UKRAINIAN IE- '\x0404'# -> unI64 1108- -- CYRILLIC CAPITAL LETTER DZE- '\x0405'# -> unI64 1109- -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I- '\x0406'# -> unI64 1110- -- CYRILLIC CAPITAL LETTER YI- '\x0407'# -> unI64 1111- -- CYRILLIC CAPITAL LETTER JE- '\x0408'# -> unI64 1112- -- CYRILLIC CAPITAL LETTER LJE- '\x0409'# -> unI64 1113- -- CYRILLIC CAPITAL LETTER NJE- '\x040a'# -> unI64 1114- -- CYRILLIC CAPITAL LETTER TSHE- '\x040b'# -> unI64 1115- -- CYRILLIC CAPITAL LETTER KJE- '\x040c'# -> unI64 1116- -- CYRILLIC CAPITAL LETTER I WITH GRAVE- '\x040d'# -> unI64 1117- -- CYRILLIC CAPITAL LETTER SHORT U- '\x040e'# -> unI64 1118- -- CYRILLIC CAPITAL LETTER DZHE- '\x040f'# -> unI64 1119- -- CYRILLIC CAPITAL LETTER A- '\x0410'# -> unI64 1072- -- CYRILLIC CAPITAL LETTER BE- '\x0411'# -> unI64 1073- -- CYRILLIC CAPITAL LETTER VE- '\x0412'# -> unI64 1074- -- CYRILLIC CAPITAL LETTER GHE- '\x0413'# -> unI64 1075- -- CYRILLIC CAPITAL LETTER DE- '\x0414'# -> unI64 1076- -- CYRILLIC CAPITAL LETTER IE- '\x0415'# -> unI64 1077- -- CYRILLIC CAPITAL LETTER ZHE- '\x0416'# -> unI64 1078- -- CYRILLIC CAPITAL LETTER ZE- '\x0417'# -> unI64 1079- -- CYRILLIC CAPITAL LETTER I- '\x0418'# -> unI64 1080- -- CYRILLIC CAPITAL LETTER SHORT I- '\x0419'# -> unI64 1081- -- CYRILLIC CAPITAL LETTER KA- '\x041a'# -> unI64 1082- -- CYRILLIC CAPITAL LETTER EL- '\x041b'# -> unI64 1083- -- CYRILLIC CAPITAL LETTER EM- '\x041c'# -> unI64 1084- -- CYRILLIC CAPITAL LETTER EN- '\x041d'# -> unI64 1085- -- CYRILLIC CAPITAL LETTER O- '\x041e'# -> unI64 1086- -- CYRILLIC CAPITAL LETTER PE- '\x041f'# -> unI64 1087- -- CYRILLIC CAPITAL LETTER ER- '\x0420'# -> unI64 1088- -- CYRILLIC CAPITAL LETTER ES- '\x0421'# -> unI64 1089- -- CYRILLIC CAPITAL LETTER TE- '\x0422'# -> unI64 1090- -- CYRILLIC CAPITAL LETTER U- '\x0423'# -> unI64 1091- -- CYRILLIC CAPITAL LETTER EF- '\x0424'# -> unI64 1092- -- CYRILLIC CAPITAL LETTER HA- '\x0425'# -> unI64 1093- -- CYRILLIC CAPITAL LETTER TSE- '\x0426'# -> unI64 1094- -- CYRILLIC CAPITAL LETTER CHE- '\x0427'# -> unI64 1095- -- CYRILLIC CAPITAL LETTER SHA- '\x0428'# -> unI64 1096- -- CYRILLIC CAPITAL LETTER SHCHA- '\x0429'# -> unI64 1097- -- CYRILLIC CAPITAL LETTER HARD SIGN- '\x042a'# -> unI64 1098- -- CYRILLIC CAPITAL LETTER YERU- '\x042b'# -> unI64 1099- -- CYRILLIC CAPITAL LETTER SOFT SIGN- '\x042c'# -> unI64 1100- -- CYRILLIC CAPITAL LETTER E- '\x042d'# -> unI64 1101- -- CYRILLIC CAPITAL LETTER YU- '\x042e'# -> unI64 1102- -- CYRILLIC CAPITAL LETTER YA- '\x042f'# -> unI64 1103- -- CYRILLIC CAPITAL LETTER OMEGA- '\x0460'# -> unI64 1121- -- CYRILLIC CAPITAL LETTER YAT- '\x0462'# -> unI64 1123- -- CYRILLIC CAPITAL LETTER IOTIFIED E- '\x0464'# -> unI64 1125- -- CYRILLIC CAPITAL LETTER LITTLE YUS- '\x0466'# -> unI64 1127- -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS- '\x0468'# -> unI64 1129- -- CYRILLIC CAPITAL LETTER BIG YUS- '\x046a'# -> unI64 1131- -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS- '\x046c'# -> unI64 1133- -- CYRILLIC CAPITAL LETTER KSI- '\x046e'# -> unI64 1135- -- CYRILLIC CAPITAL LETTER PSI- '\x0470'# -> unI64 1137- -- CYRILLIC CAPITAL LETTER FITA- '\x0472'# -> unI64 1139- -- CYRILLIC CAPITAL LETTER IZHITSA- '\x0474'# -> unI64 1141- -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT- '\x0476'# -> unI64 1143- -- CYRILLIC CAPITAL LETTER UK- '\x0478'# -> unI64 1145- -- CYRILLIC CAPITAL LETTER ROUND OMEGA- '\x047a'# -> unI64 1147- -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO- '\x047c'# -> unI64 1149- -- CYRILLIC CAPITAL LETTER OT- '\x047e'# -> unI64 1151- -- CYRILLIC CAPITAL LETTER KOPPA- '\x0480'# -> unI64 1153- -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL- '\x048a'# -> unI64 1163- -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN- '\x048c'# -> unI64 1165- -- CYRILLIC CAPITAL LETTER ER WITH TICK- '\x048e'# -> unI64 1167- -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN- '\x0490'# -> unI64 1169- -- CYRILLIC CAPITAL LETTER GHE WITH STROKE- '\x0492'# -> unI64 1171- -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK- '\x0494'# -> unI64 1173- -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER- '\x0496'# -> unI64 1175- -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER- '\x0498'# -> unI64 1177- -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER- '\x049a'# -> unI64 1179- -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE- '\x049c'# -> unI64 1181- -- CYRILLIC CAPITAL LETTER KA WITH STROKE- '\x049e'# -> unI64 1183- -- CYRILLIC CAPITAL LETTER BASHKIR KA- '\x04a0'# -> unI64 1185- -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER- '\x04a2'# -> unI64 1187- -- CYRILLIC CAPITAL LIGATURE EN GHE- '\x04a4'# -> unI64 1189- -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK- '\x04a6'# -> unI64 1191- -- CYRILLIC CAPITAL LETTER ABKHASIAN HA- '\x04a8'# -> unI64 1193- -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER- '\x04aa'# -> unI64 1195- -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER- '\x04ac'# -> unI64 1197- -- CYRILLIC CAPITAL LETTER STRAIGHT U- '\x04ae'# -> unI64 1199- -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE- '\x04b0'# -> unI64 1201- -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER- '\x04b2'# -> unI64 1203- -- CYRILLIC CAPITAL LIGATURE TE TSE- '\x04b4'# -> unI64 1205- -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER- '\x04b6'# -> unI64 1207- -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE- '\x04b8'# -> unI64 1209- -- CYRILLIC CAPITAL LETTER SHHA- '\x04ba'# -> unI64 1211- -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE- '\x04bc'# -> unI64 1213- -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER- '\x04be'# -> unI64 1215- -- CYRILLIC LETTER PALOCHKA- '\x04c0'# -> unI64 1231- -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE- '\x04c1'# -> unI64 1218- -- CYRILLIC CAPITAL LETTER KA WITH HOOK- '\x04c3'# -> unI64 1220- -- CYRILLIC CAPITAL LETTER EL WITH TAIL- '\x04c5'# -> unI64 1222- -- CYRILLIC CAPITAL LETTER EN WITH HOOK- '\x04c7'# -> unI64 1224- -- CYRILLIC CAPITAL LETTER EN WITH TAIL- '\x04c9'# -> unI64 1226- -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE- '\x04cb'# -> unI64 1228- -- CYRILLIC CAPITAL LETTER EM WITH TAIL- '\x04cd'# -> unI64 1230- -- CYRILLIC CAPITAL LETTER A WITH BREVE- '\x04d0'# -> unI64 1233- -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS- '\x04d2'# -> unI64 1235- -- CYRILLIC CAPITAL LIGATURE A IE- '\x04d4'# -> unI64 1237- -- CYRILLIC CAPITAL LETTER IE WITH BREVE- '\x04d6'# -> unI64 1239- -- CYRILLIC CAPITAL LETTER SCHWA- '\x04d8'# -> unI64 1241- -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS- '\x04da'# -> unI64 1243- -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS- '\x04dc'# -> unI64 1245- -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS- '\x04de'# -> unI64 1247- -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE- '\x04e0'# -> unI64 1249- -- CYRILLIC CAPITAL LETTER I WITH MACRON- '\x04e2'# -> unI64 1251- -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS- '\x04e4'# -> unI64 1253- -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS- '\x04e6'# -> unI64 1255- -- CYRILLIC CAPITAL LETTER BARRED O- '\x04e8'# -> unI64 1257- -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS- '\x04ea'# -> unI64 1259- -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS- '\x04ec'# -> unI64 1261- -- CYRILLIC CAPITAL LETTER U WITH MACRON- '\x04ee'# -> unI64 1263- -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS- '\x04f0'# -> unI64 1265- -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE- '\x04f2'# -> unI64 1267- -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS- '\x04f4'# -> unI64 1269- -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER- '\x04f6'# -> unI64 1271- -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS- '\x04f8'# -> unI64 1273- -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK- '\x04fa'# -> unI64 1275- -- CYRILLIC CAPITAL LETTER HA WITH HOOK- '\x04fc'# -> unI64 1277- -- CYRILLIC CAPITAL LETTER HA WITH STROKE- '\x04fe'# -> unI64 1279- -- CYRILLIC CAPITAL LETTER KOMI DE- '\x0500'# -> unI64 1281- -- CYRILLIC CAPITAL LETTER KOMI DJE- '\x0502'# -> unI64 1283- -- CYRILLIC CAPITAL LETTER KOMI ZJE- '\x0504'# -> unI64 1285- -- CYRILLIC CAPITAL LETTER KOMI DZJE- '\x0506'# -> unI64 1287- -- CYRILLIC CAPITAL LETTER KOMI LJE- '\x0508'# -> unI64 1289- -- CYRILLIC CAPITAL LETTER KOMI NJE- '\x050a'# -> unI64 1291- -- CYRILLIC CAPITAL LETTER KOMI SJE- '\x050c'# -> unI64 1293- -- CYRILLIC CAPITAL LETTER KOMI TJE- '\x050e'# -> unI64 1295- -- CYRILLIC CAPITAL LETTER REVERSED ZE- '\x0510'# -> unI64 1297- -- CYRILLIC CAPITAL LETTER EL WITH HOOK- '\x0512'# -> unI64 1299- -- CYRILLIC CAPITAL LETTER LHA- '\x0514'# -> unI64 1301- -- CYRILLIC CAPITAL LETTER RHA- '\x0516'# -> unI64 1303- -- CYRILLIC CAPITAL LETTER YAE- '\x0518'# -> unI64 1305- -- CYRILLIC CAPITAL LETTER QA- '\x051a'# -> unI64 1307- -- CYRILLIC CAPITAL LETTER WE- '\x051c'# -> unI64 1309- -- CYRILLIC CAPITAL LETTER ALEUT KA- '\x051e'# -> unI64 1311- -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK- '\x0520'# -> unI64 1313- -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK- '\x0522'# -> unI64 1315- -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER- '\x0524'# -> unI64 1317- -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER- '\x0526'# -> unI64 1319- -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK- '\x0528'# -> unI64 1321- -- CYRILLIC CAPITAL LETTER DZZHE- '\x052a'# -> unI64 1323- -- CYRILLIC CAPITAL LETTER DCHE- '\x052c'# -> unI64 1325- -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER- '\x052e'# -> unI64 1327- -- ARMENIAN CAPITAL LETTER AYB- '\x0531'# -> unI64 1377- -- ARMENIAN CAPITAL LETTER BEN- '\x0532'# -> unI64 1378- -- ARMENIAN CAPITAL LETTER GIM- '\x0533'# -> unI64 1379- -- ARMENIAN CAPITAL LETTER DA- '\x0534'# -> unI64 1380- -- ARMENIAN CAPITAL LETTER ECH- '\x0535'# -> unI64 1381- -- ARMENIAN CAPITAL LETTER ZA- '\x0536'# -> unI64 1382- -- ARMENIAN CAPITAL LETTER EH- '\x0537'# -> unI64 1383- -- ARMENIAN CAPITAL LETTER ET- '\x0538'# -> unI64 1384- -- ARMENIAN CAPITAL LETTER TO- '\x0539'# -> unI64 1385- -- ARMENIAN CAPITAL LETTER ZHE- '\x053a'# -> unI64 1386- -- ARMENIAN CAPITAL LETTER INI- '\x053b'# -> unI64 1387- -- ARMENIAN CAPITAL LETTER LIWN- '\x053c'# -> unI64 1388- -- ARMENIAN CAPITAL LETTER XEH- '\x053d'# -> unI64 1389- -- ARMENIAN CAPITAL LETTER CA- '\x053e'# -> unI64 1390- -- ARMENIAN CAPITAL LETTER KEN- '\x053f'# -> unI64 1391- -- ARMENIAN CAPITAL LETTER HO- '\x0540'# -> unI64 1392- -- ARMENIAN CAPITAL LETTER JA- '\x0541'# -> unI64 1393- -- ARMENIAN CAPITAL LETTER GHAD- '\x0542'# -> unI64 1394- -- ARMENIAN CAPITAL LETTER CHEH- '\x0543'# -> unI64 1395- -- ARMENIAN CAPITAL LETTER MEN- '\x0544'# -> unI64 1396- -- ARMENIAN CAPITAL LETTER YI- '\x0545'# -> unI64 1397- -- ARMENIAN CAPITAL LETTER NOW- '\x0546'# -> unI64 1398- -- ARMENIAN CAPITAL LETTER SHA- '\x0547'# -> unI64 1399- -- ARMENIAN CAPITAL LETTER VO- '\x0548'# -> unI64 1400- -- ARMENIAN CAPITAL LETTER CHA- '\x0549'# -> unI64 1401- -- ARMENIAN CAPITAL LETTER PEH- '\x054a'# -> unI64 1402- -- ARMENIAN CAPITAL LETTER JHEH- '\x054b'# -> unI64 1403- -- ARMENIAN CAPITAL LETTER RA- '\x054c'# -> unI64 1404- -- ARMENIAN CAPITAL LETTER SEH- '\x054d'# -> unI64 1405- -- ARMENIAN CAPITAL LETTER VEW- '\x054e'# -> unI64 1406- -- ARMENIAN CAPITAL LETTER TIWN- '\x054f'# -> unI64 1407- -- ARMENIAN CAPITAL LETTER REH- '\x0550'# -> unI64 1408- -- ARMENIAN CAPITAL LETTER CO- '\x0551'# -> unI64 1409- -- ARMENIAN CAPITAL LETTER YIWN- '\x0552'# -> unI64 1410- -- ARMENIAN CAPITAL LETTER PIWR- '\x0553'# -> unI64 1411- -- ARMENIAN CAPITAL LETTER KEH- '\x0554'# -> unI64 1412- -- ARMENIAN CAPITAL LETTER OH- '\x0555'# -> unI64 1413- -- ARMENIAN CAPITAL LETTER FEH- '\x0556'# -> unI64 1414- -- ARMENIAN SMALL LIGATURE ECH YIWN- '\x0587'# -> unI64 2956985701- -- GEORGIAN CAPITAL LETTER AN- '\x10a0'# -> unI64 11520- -- GEORGIAN CAPITAL LETTER BAN- '\x10a1'# -> unI64 11521- -- GEORGIAN CAPITAL LETTER GAN- '\x10a2'# -> unI64 11522- -- GEORGIAN CAPITAL LETTER DON- '\x10a3'# -> unI64 11523- -- GEORGIAN CAPITAL LETTER EN- '\x10a4'# -> unI64 11524- -- GEORGIAN CAPITAL LETTER VIN- '\x10a5'# -> unI64 11525- -- GEORGIAN CAPITAL LETTER ZEN- '\x10a6'# -> unI64 11526- -- GEORGIAN CAPITAL LETTER TAN- '\x10a7'# -> unI64 11527- -- GEORGIAN CAPITAL LETTER IN- '\x10a8'# -> unI64 11528- -- GEORGIAN CAPITAL LETTER KAN- '\x10a9'# -> unI64 11529- -- GEORGIAN CAPITAL LETTER LAS- '\x10aa'# -> unI64 11530- -- GEORGIAN CAPITAL LETTER MAN- '\x10ab'# -> unI64 11531- -- GEORGIAN CAPITAL LETTER NAR- '\x10ac'# -> unI64 11532- -- GEORGIAN CAPITAL LETTER ON- '\x10ad'# -> unI64 11533- -- GEORGIAN CAPITAL LETTER PAR- '\x10ae'# -> unI64 11534- -- GEORGIAN CAPITAL LETTER ZHAR- '\x10af'# -> unI64 11535- -- GEORGIAN CAPITAL LETTER RAE- '\x10b0'# -> unI64 11536- -- GEORGIAN CAPITAL LETTER SAN- '\x10b1'# -> unI64 11537- -- GEORGIAN CAPITAL LETTER TAR- '\x10b2'# -> unI64 11538- -- GEORGIAN CAPITAL LETTER UN- '\x10b3'# -> unI64 11539- -- GEORGIAN CAPITAL LETTER PHAR- '\x10b4'# -> unI64 11540- -- GEORGIAN CAPITAL LETTER KHAR- '\x10b5'# -> unI64 11541- -- GEORGIAN CAPITAL LETTER GHAN- '\x10b6'# -> unI64 11542- -- GEORGIAN CAPITAL LETTER QAR- '\x10b7'# -> unI64 11543- -- GEORGIAN CAPITAL LETTER SHIN- '\x10b8'# -> unI64 11544- -- GEORGIAN CAPITAL LETTER CHIN- '\x10b9'# -> unI64 11545- -- GEORGIAN CAPITAL LETTER CAN- '\x10ba'# -> unI64 11546- -- GEORGIAN CAPITAL LETTER JIL- '\x10bb'# -> unI64 11547- -- GEORGIAN CAPITAL LETTER CIL- '\x10bc'# -> unI64 11548- -- GEORGIAN CAPITAL LETTER CHAR- '\x10bd'# -> unI64 11549- -- GEORGIAN CAPITAL LETTER XAN- '\x10be'# -> unI64 11550- -- GEORGIAN CAPITAL LETTER JHAN- '\x10bf'# -> unI64 11551- -- GEORGIAN CAPITAL LETTER HAE- '\x10c0'# -> unI64 11552- -- GEORGIAN CAPITAL LETTER HE- '\x10c1'# -> unI64 11553- -- GEORGIAN CAPITAL LETTER HIE- '\x10c2'# -> unI64 11554- -- GEORGIAN CAPITAL LETTER WE- '\x10c3'# -> unI64 11555- -- GEORGIAN CAPITAL LETTER HAR- '\x10c4'# -> unI64 11556- -- GEORGIAN CAPITAL LETTER HOE- '\x10c5'# -> unI64 11557- -- GEORGIAN CAPITAL LETTER YN- '\x10c7'# -> unI64 11559- -- GEORGIAN CAPITAL LETTER AEN- '\x10cd'# -> unI64 11565- -- CHEROKEE SMALL LETTER YE- '\x13f8'# -> unI64 5104- -- CHEROKEE SMALL LETTER YI- '\x13f9'# -> unI64 5105- -- CHEROKEE SMALL LETTER YO- '\x13fa'# -> unI64 5106- -- CHEROKEE SMALL LETTER YU- '\x13fb'# -> unI64 5107- -- CHEROKEE SMALL LETTER YV- '\x13fc'# -> unI64 5108- -- CHEROKEE SMALL LETTER MV- '\x13fd'# -> unI64 5109- -- CYRILLIC SMALL LETTER ROUNDED VE- '\x1c80'# -> unI64 1074- -- CYRILLIC SMALL LETTER LONG-LEGGED DE- '\x1c81'# -> unI64 1076- -- CYRILLIC SMALL LETTER NARROW O- '\x1c82'# -> unI64 1086- -- CYRILLIC SMALL LETTER WIDE ES- '\x1c83'# -> unI64 1089- -- CYRILLIC SMALL LETTER TALL TE- '\x1c84'# -> unI64 1090- -- CYRILLIC SMALL LETTER THREE-LEGGED TE- '\x1c85'# -> unI64 1090- -- CYRILLIC SMALL LETTER TALL HARD SIGN- '\x1c86'# -> unI64 1098- -- CYRILLIC SMALL LETTER TALL YAT- '\x1c87'# -> unI64 1123- -- CYRILLIC SMALL LETTER UNBLENDED UK- '\x1c88'# -> unI64 42571- -- CYRILLIC CAPITAL LETTER TJE- '\x1c89'# -> unI64 7306- -- GEORGIAN MTAVRULI CAPITAL LETTER AN- '\x1c90'# -> unI64 4304- -- GEORGIAN MTAVRULI CAPITAL LETTER BAN- '\x1c91'# -> unI64 4305- -- GEORGIAN MTAVRULI CAPITAL LETTER GAN- '\x1c92'# -> unI64 4306- -- GEORGIAN MTAVRULI CAPITAL LETTER DON- '\x1c93'# -> unI64 4307- -- GEORGIAN MTAVRULI CAPITAL LETTER EN- '\x1c94'# -> unI64 4308- -- GEORGIAN MTAVRULI CAPITAL LETTER VIN- '\x1c95'# -> unI64 4309- -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN- '\x1c96'# -> unI64 4310- -- GEORGIAN MTAVRULI CAPITAL LETTER TAN- '\x1c97'# -> unI64 4311- -- GEORGIAN MTAVRULI CAPITAL LETTER IN- '\x1c98'# -> unI64 4312- -- GEORGIAN MTAVRULI CAPITAL LETTER KAN- '\x1c99'# -> unI64 4313- -- GEORGIAN MTAVRULI CAPITAL LETTER LAS- '\x1c9a'# -> unI64 4314- -- GEORGIAN MTAVRULI CAPITAL LETTER MAN- '\x1c9b'# -> unI64 4315- -- GEORGIAN MTAVRULI CAPITAL LETTER NAR- '\x1c9c'# -> unI64 4316- -- GEORGIAN MTAVRULI CAPITAL LETTER ON- '\x1c9d'# -> unI64 4317- -- GEORGIAN MTAVRULI CAPITAL LETTER PAR- '\x1c9e'# -> unI64 4318- -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR- '\x1c9f'# -> unI64 4319- -- GEORGIAN MTAVRULI CAPITAL LETTER RAE- '\x1ca0'# -> unI64 4320- -- GEORGIAN MTAVRULI CAPITAL LETTER SAN- '\x1ca1'# -> unI64 4321- -- GEORGIAN MTAVRULI CAPITAL LETTER TAR- '\x1ca2'# -> unI64 4322- -- GEORGIAN MTAVRULI CAPITAL LETTER UN- '\x1ca3'# -> unI64 4323- -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR- '\x1ca4'# -> unI64 4324- -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR- '\x1ca5'# -> unI64 4325- -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN- '\x1ca6'# -> unI64 4326- -- GEORGIAN MTAVRULI CAPITAL LETTER QAR- '\x1ca7'# -> unI64 4327- -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN- '\x1ca8'# -> unI64 4328- -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN- '\x1ca9'# -> unI64 4329- -- GEORGIAN MTAVRULI CAPITAL LETTER CAN- '\x1caa'# -> unI64 4330- -- GEORGIAN MTAVRULI CAPITAL LETTER JIL- '\x1cab'# -> unI64 4331- -- GEORGIAN MTAVRULI CAPITAL LETTER CIL- '\x1cac'# -> unI64 4332- -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR- '\x1cad'# -> unI64 4333- -- GEORGIAN MTAVRULI CAPITAL LETTER XAN- '\x1cae'# -> unI64 4334- -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN- '\x1caf'# -> unI64 4335- -- GEORGIAN MTAVRULI CAPITAL LETTER HAE- '\x1cb0'# -> unI64 4336- -- GEORGIAN MTAVRULI CAPITAL LETTER HE- '\x1cb1'# -> unI64 4337- -- GEORGIAN MTAVRULI CAPITAL LETTER HIE- '\x1cb2'# -> unI64 4338- -- GEORGIAN MTAVRULI CAPITAL LETTER WE- '\x1cb3'# -> unI64 4339- -- GEORGIAN MTAVRULI CAPITAL LETTER HAR- '\x1cb4'# -> unI64 4340- -- GEORGIAN MTAVRULI CAPITAL LETTER HOE- '\x1cb5'# -> unI64 4341- -- GEORGIAN MTAVRULI CAPITAL LETTER FI- '\x1cb6'# -> unI64 4342- -- GEORGIAN MTAVRULI CAPITAL LETTER YN- '\x1cb7'# -> unI64 4343- -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI- '\x1cb8'# -> unI64 4344- -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN- '\x1cb9'# -> unI64 4345- -- GEORGIAN MTAVRULI CAPITAL LETTER AIN- '\x1cba'# -> unI64 4346- -- GEORGIAN MTAVRULI CAPITAL LETTER AEN- '\x1cbd'# -> unI64 4349- -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN- '\x1cbe'# -> unI64 4350- -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN- '\x1cbf'# -> unI64 4351- -- LATIN CAPITAL LETTER A WITH RING BELOW- '\x1e00'# -> unI64 7681- -- LATIN CAPITAL LETTER B WITH DOT ABOVE- '\x1e02'# -> unI64 7683- -- LATIN CAPITAL LETTER B WITH DOT BELOW- '\x1e04'# -> unI64 7685- -- LATIN CAPITAL LETTER B WITH LINE BELOW- '\x1e06'# -> unI64 7687- -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE- '\x1e08'# -> unI64 7689- -- LATIN CAPITAL LETTER D WITH DOT ABOVE- '\x1e0a'# -> unI64 7691- -- LATIN CAPITAL LETTER D WITH DOT BELOW- '\x1e0c'# -> unI64 7693- -- LATIN CAPITAL LETTER D WITH LINE BELOW- '\x1e0e'# -> unI64 7695- -- LATIN CAPITAL LETTER D WITH CEDILLA- '\x1e10'# -> unI64 7697- -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW- '\x1e12'# -> unI64 7699- -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE- '\x1e14'# -> unI64 7701- -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE- '\x1e16'# -> unI64 7703- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW- '\x1e18'# -> unI64 7705- -- LATIN CAPITAL LETTER E WITH TILDE BELOW- '\x1e1a'# -> unI64 7707- -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE- '\x1e1c'# -> unI64 7709- -- LATIN CAPITAL LETTER F WITH DOT ABOVE- '\x1e1e'# -> unI64 7711- -- LATIN CAPITAL LETTER G WITH MACRON- '\x1e20'# -> unI64 7713- -- LATIN CAPITAL LETTER H WITH DOT ABOVE- '\x1e22'# -> unI64 7715- -- LATIN CAPITAL LETTER H WITH DOT BELOW- '\x1e24'# -> unI64 7717- -- LATIN CAPITAL LETTER H WITH DIAERESIS- '\x1e26'# -> unI64 7719- -- LATIN CAPITAL LETTER H WITH CEDILLA- '\x1e28'# -> unI64 7721- -- LATIN CAPITAL LETTER H WITH BREVE BELOW- '\x1e2a'# -> unI64 7723- -- LATIN CAPITAL LETTER I WITH TILDE BELOW- '\x1e2c'# -> unI64 7725- -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE- '\x1e2e'# -> unI64 7727- -- LATIN CAPITAL LETTER K WITH ACUTE- '\x1e30'# -> unI64 7729- -- LATIN CAPITAL LETTER K WITH DOT BELOW- '\x1e32'# -> unI64 7731- -- LATIN CAPITAL LETTER K WITH LINE BELOW- '\x1e34'# -> unI64 7733- -- LATIN CAPITAL LETTER L WITH DOT BELOW- '\x1e36'# -> unI64 7735- -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON- '\x1e38'# -> unI64 7737- -- LATIN CAPITAL LETTER L WITH LINE BELOW- '\x1e3a'# -> unI64 7739- -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW- '\x1e3c'# -> unI64 7741- -- LATIN CAPITAL LETTER M WITH ACUTE- '\x1e3e'# -> unI64 7743- -- LATIN CAPITAL LETTER M WITH DOT ABOVE- '\x1e40'# -> unI64 7745- -- LATIN CAPITAL LETTER M WITH DOT BELOW- '\x1e42'# -> unI64 7747- -- LATIN CAPITAL LETTER N WITH DOT ABOVE- '\x1e44'# -> unI64 7749- -- LATIN CAPITAL LETTER N WITH DOT BELOW- '\x1e46'# -> unI64 7751- -- LATIN CAPITAL LETTER N WITH LINE BELOW- '\x1e48'# -> unI64 7753- -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW- '\x1e4a'# -> unI64 7755- -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE- '\x1e4c'# -> unI64 7757- -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS- '\x1e4e'# -> unI64 7759- -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE- '\x1e50'# -> unI64 7761- -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE- '\x1e52'# -> unI64 7763- -- LATIN CAPITAL LETTER P WITH ACUTE- '\x1e54'# -> unI64 7765- -- LATIN CAPITAL LETTER P WITH DOT ABOVE- '\x1e56'# -> unI64 7767- -- LATIN CAPITAL LETTER R WITH DOT ABOVE- '\x1e58'# -> unI64 7769- -- LATIN CAPITAL LETTER R WITH DOT BELOW- '\x1e5a'# -> unI64 7771- -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON- '\x1e5c'# -> unI64 7773- -- LATIN CAPITAL LETTER R WITH LINE BELOW- '\x1e5e'# -> unI64 7775- -- LATIN CAPITAL LETTER S WITH DOT ABOVE- '\x1e60'# -> unI64 7777- -- LATIN CAPITAL LETTER S WITH DOT BELOW- '\x1e62'# -> unI64 7779- -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE- '\x1e64'# -> unI64 7781- -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE- '\x1e66'# -> unI64 7783- -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE- '\x1e68'# -> unI64 7785- -- LATIN CAPITAL LETTER T WITH DOT ABOVE- '\x1e6a'# -> unI64 7787- -- LATIN CAPITAL LETTER T WITH DOT BELOW- '\x1e6c'# -> unI64 7789- -- LATIN CAPITAL LETTER T WITH LINE BELOW- '\x1e6e'# -> unI64 7791- -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW- '\x1e70'# -> unI64 7793- -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW- '\x1e72'# -> unI64 7795- -- LATIN CAPITAL LETTER U WITH TILDE BELOW- '\x1e74'# -> unI64 7797- -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW- '\x1e76'# -> unI64 7799- -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE- '\x1e78'# -> unI64 7801- -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS- '\x1e7a'# -> unI64 7803- -- LATIN CAPITAL LETTER V WITH TILDE- '\x1e7c'# -> unI64 7805- -- LATIN CAPITAL LETTER V WITH DOT BELOW- '\x1e7e'# -> unI64 7807- -- LATIN CAPITAL LETTER W WITH GRAVE- '\x1e80'# -> unI64 7809- -- LATIN CAPITAL LETTER W WITH ACUTE- '\x1e82'# -> unI64 7811- -- LATIN CAPITAL LETTER W WITH DIAERESIS- '\x1e84'# -> unI64 7813- -- LATIN CAPITAL LETTER W WITH DOT ABOVE- '\x1e86'# -> unI64 7815- -- LATIN CAPITAL LETTER W WITH DOT BELOW- '\x1e88'# -> unI64 7817- -- LATIN CAPITAL LETTER X WITH DOT ABOVE- '\x1e8a'# -> unI64 7819- -- LATIN CAPITAL LETTER X WITH DIAERESIS- '\x1e8c'# -> unI64 7821- -- LATIN CAPITAL LETTER Y WITH DOT ABOVE- '\x1e8e'# -> unI64 7823- -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX- '\x1e90'# -> unI64 7825- -- LATIN CAPITAL LETTER Z WITH DOT BELOW- '\x1e92'# -> unI64 7827- -- LATIN CAPITAL LETTER Z WITH LINE BELOW- '\x1e94'# -> unI64 7829- -- LATIN SMALL LETTER H WITH LINE BELOW- '\x1e96'# -> unI64 1713373288- -- LATIN SMALL LETTER T WITH DIAERESIS- '\x1e97'# -> unI64 1627390068- -- LATIN SMALL LETTER W WITH RING ABOVE- '\x1e98'# -> unI64 1631584375- -- LATIN SMALL LETTER Y WITH RING ABOVE- '\x1e99'# -> unI64 1631584377- -- LATIN SMALL LETTER A WITH RIGHT HALF RING- '\x1e9a'# -> unI64 1472200801- -- LATIN SMALL LETTER LONG S WITH DOT ABOVE- '\x1e9b'# -> unI64 7777- -- LATIN CAPITAL LETTER SHARP S- '\x1e9e'# -> unI64 241172595- -- LATIN CAPITAL LETTER A WITH DOT BELOW- '\x1ea0'# -> unI64 7841- -- LATIN CAPITAL LETTER A WITH HOOK ABOVE- '\x1ea2'# -> unI64 7843- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE- '\x1ea4'# -> unI64 7845- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE- '\x1ea6'# -> unI64 7847- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE- '\x1ea8'# -> unI64 7849- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE- '\x1eaa'# -> unI64 7851- -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW- '\x1eac'# -> unI64 7853- -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE- '\x1eae'# -> unI64 7855- -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE- '\x1eb0'# -> unI64 7857- -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE- '\x1eb2'# -> unI64 7859- -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE- '\x1eb4'# -> unI64 7861- -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW- '\x1eb6'# -> unI64 7863- -- LATIN CAPITAL LETTER E WITH DOT BELOW- '\x1eb8'# -> unI64 7865- -- LATIN CAPITAL LETTER E WITH HOOK ABOVE- '\x1eba'# -> unI64 7867- -- LATIN CAPITAL LETTER E WITH TILDE- '\x1ebc'# -> unI64 7869- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE- '\x1ebe'# -> unI64 7871- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE- '\x1ec0'# -> unI64 7873- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE- '\x1ec2'# -> unI64 7875- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE- '\x1ec4'# -> unI64 7877- -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW- '\x1ec6'# -> unI64 7879- -- LATIN CAPITAL LETTER I WITH HOOK ABOVE- '\x1ec8'# -> unI64 7881- -- LATIN CAPITAL LETTER I WITH DOT BELOW- '\x1eca'# -> unI64 7883- -- LATIN CAPITAL LETTER O WITH DOT BELOW- '\x1ecc'# -> unI64 7885- -- LATIN CAPITAL LETTER O WITH HOOK ABOVE- '\x1ece'# -> unI64 7887- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE- '\x1ed0'# -> unI64 7889- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE- '\x1ed2'# -> unI64 7891- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE- '\x1ed4'# -> unI64 7893- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE- '\x1ed6'# -> unI64 7895- -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW- '\x1ed8'# -> unI64 7897- -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE- '\x1eda'# -> unI64 7899- -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE- '\x1edc'# -> unI64 7901- -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE- '\x1ede'# -> unI64 7903- -- LATIN CAPITAL LETTER O WITH HORN AND TILDE- '\x1ee0'# -> unI64 7905- -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW- '\x1ee2'# -> unI64 7907- -- LATIN CAPITAL LETTER U WITH DOT BELOW- '\x1ee4'# -> unI64 7909- -- LATIN CAPITAL LETTER U WITH HOOK ABOVE- '\x1ee6'# -> unI64 7911- -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE- '\x1ee8'# -> unI64 7913- -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE- '\x1eea'# -> unI64 7915- -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE- '\x1eec'# -> unI64 7917- -- LATIN CAPITAL LETTER U WITH HORN AND TILDE- '\x1eee'# -> unI64 7919- -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW- '\x1ef0'# -> unI64 7921- -- LATIN CAPITAL LETTER Y WITH GRAVE- '\x1ef2'# -> unI64 7923- -- LATIN CAPITAL LETTER Y WITH DOT BELOW- '\x1ef4'# -> unI64 7925- -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE- '\x1ef6'# -> unI64 7927- -- LATIN CAPITAL LETTER Y WITH TILDE- '\x1ef8'# -> unI64 7929- -- LATIN CAPITAL LETTER MIDDLE-WELSH LL- '\x1efa'# -> unI64 7931- -- LATIN CAPITAL LETTER MIDDLE-WELSH V- '\x1efc'# -> unI64 7933- -- LATIN CAPITAL LETTER Y WITH LOOP- '\x1efe'# -> unI64 7935- -- GREEK CAPITAL LETTER ALPHA WITH PSILI- '\x1f08'# -> unI64 7936- -- GREEK CAPITAL LETTER ALPHA WITH DASIA- '\x1f09'# -> unI64 7937- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA- '\x1f0a'# -> unI64 7938- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA- '\x1f0b'# -> unI64 7939- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA- '\x1f0c'# -> unI64 7940- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA- '\x1f0d'# -> unI64 7941- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI- '\x1f0e'# -> unI64 7942- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI- '\x1f0f'# -> unI64 7943- -- GREEK CAPITAL LETTER EPSILON WITH PSILI- '\x1f18'# -> unI64 7952- -- GREEK CAPITAL LETTER EPSILON WITH DASIA- '\x1f19'# -> unI64 7953- -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA- '\x1f1a'# -> unI64 7954- -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA- '\x1f1b'# -> unI64 7955- -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA- '\x1f1c'# -> unI64 7956- -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA- '\x1f1d'# -> unI64 7957- -- GREEK CAPITAL LETTER ETA WITH PSILI- '\x1f28'# -> unI64 7968- -- GREEK CAPITAL LETTER ETA WITH DASIA- '\x1f29'# -> unI64 7969- -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA- '\x1f2a'# -> unI64 7970- -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA- '\x1f2b'# -> unI64 7971- -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA- '\x1f2c'# -> unI64 7972- -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA- '\x1f2d'# -> unI64 7973- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI- '\x1f2e'# -> unI64 7974- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI- '\x1f2f'# -> unI64 7975- -- GREEK CAPITAL LETTER IOTA WITH PSILI- '\x1f38'# -> unI64 7984- -- GREEK CAPITAL LETTER IOTA WITH DASIA- '\x1f39'# -> unI64 7985- -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA- '\x1f3a'# -> unI64 7986- -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA- '\x1f3b'# -> unI64 7987- -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA- '\x1f3c'# -> unI64 7988- -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA- '\x1f3d'# -> unI64 7989- -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI- '\x1f3e'# -> unI64 7990- -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI- '\x1f3f'# -> unI64 7991- -- GREEK CAPITAL LETTER OMICRON WITH PSILI- '\x1f48'# -> unI64 8000- -- GREEK CAPITAL LETTER OMICRON WITH DASIA- '\x1f49'# -> unI64 8001- -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA- '\x1f4a'# -> unI64 8002- -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA- '\x1f4b'# -> unI64 8003- -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA- '\x1f4c'# -> unI64 8004- -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA- '\x1f4d'# -> unI64 8005- -- GREEK SMALL LETTER UPSILON WITH PSILI- '\x1f50'# -> unI64 1650459589- -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA- '\x1f52'# -> unI64 3377701370987461- -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA- '\x1f54'# -> unI64 3382099417498565- -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI- '\x1f56'# -> unI64 3667972440720325- -- GREEK CAPITAL LETTER UPSILON WITH DASIA- '\x1f59'# -> unI64 8017- -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA- '\x1f5b'# -> unI64 8019- -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA- '\x1f5d'# -> unI64 8021- -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI- '\x1f5f'# -> unI64 8023- -- GREEK CAPITAL LETTER OMEGA WITH PSILI- '\x1f68'# -> unI64 8032- -- GREEK CAPITAL LETTER OMEGA WITH DASIA- '\x1f69'# -> unI64 8033- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA- '\x1f6a'# -> unI64 8034- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA- '\x1f6b'# -> unI64 8035- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA- '\x1f6c'# -> unI64 8036- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA- '\x1f6d'# -> unI64 8037- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI- '\x1f6e'# -> unI64 8038- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI- '\x1f6f'# -> unI64 8039- -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI- '\x1f80'# -> unI64 1998593792- -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI- '\x1f81'# -> unI64 1998593793- -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f82'# -> unI64 1998593794- -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f83'# -> unI64 1998593795- -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f84'# -> unI64 1998593796- -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f85'# -> unI64 1998593797- -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f86'# -> unI64 1998593798- -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f87'# -> unI64 1998593799- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI- '\x1f88'# -> unI64 1998593792- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI- '\x1f89'# -> unI64 1998593793- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f8a'# -> unI64 1998593794- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f8b'# -> unI64 1998593795- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f8c'# -> unI64 1998593796- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f8d'# -> unI64 1998593797- -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8e'# -> unI64 1998593798- -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f8f'# -> unI64 1998593799- -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI- '\x1f90'# -> unI64 1998593824- -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI- '\x1f91'# -> unI64 1998593825- -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1f92'# -> unI64 1998593826- -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1f93'# -> unI64 1998593827- -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1f94'# -> unI64 1998593828- -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1f95'# -> unI64 1998593829- -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f96'# -> unI64 1998593830- -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1f97'# -> unI64 1998593831- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI- '\x1f98'# -> unI64 1998593824- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI- '\x1f99'# -> unI64 1998593825- -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1f9a'# -> unI64 1998593826- -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1f9b'# -> unI64 1998593827- -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1f9c'# -> unI64 1998593828- -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1f9d'# -> unI64 1998593829- -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9e'# -> unI64 1998593830- -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1f9f'# -> unI64 1998593831- -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI- '\x1fa0'# -> unI64 1998593888- -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI- '\x1fa1'# -> unI64 1998593889- -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI- '\x1fa2'# -> unI64 1998593890- -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI- '\x1fa3'# -> unI64 1998593891- -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI- '\x1fa4'# -> unI64 1998593892- -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI- '\x1fa5'# -> unI64 1998593893- -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa6'# -> unI64 1998593894- -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI- '\x1fa7'# -> unI64 1998593895- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI- '\x1fa8'# -> unI64 1998593888- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI- '\x1fa9'# -> unI64 1998593889- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI- '\x1faa'# -> unI64 1998593890- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI- '\x1fab'# -> unI64 1998593891- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI- '\x1fac'# -> unI64 1998593892- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI- '\x1fad'# -> unI64 1998593893- -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI- '\x1fae'# -> unI64 1998593894- -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI- '\x1faf'# -> unI64 1998593895- -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI- '\x1fb2'# -> unI64 1998593904- -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI- '\x1fb3'# -> unI64 1998586801- -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI- '\x1fb4'# -> unI64 1998586796- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI- '\x1fb6'# -> unI64 1749025713- -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fb7'# -> unI64 4191340074107825- -- GREEK CAPITAL LETTER ALPHA WITH VRACHY- '\x1fb8'# -> unI64 8112- -- GREEK CAPITAL LETTER ALPHA WITH MACRON- '\x1fb9'# -> unI64 8113- -- GREEK CAPITAL LETTER ALPHA WITH VARIA- '\x1fba'# -> unI64 8048- -- GREEK CAPITAL LETTER ALPHA WITH OXIA- '\x1fbb'# -> unI64 8049- -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI- '\x1fbc'# -> unI64 1998586801- -- GREEK PROSGEGRAMMENI- '\x1fbe'# -> unI64 953- -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI- '\x1fc2'# -> unI64 1998593908- -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI- '\x1fc3'# -> unI64 1998586807- -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI- '\x1fc4'# -> unI64 1998586798- -- GREEK SMALL LETTER ETA WITH PERISPOMENI- '\x1fc6'# -> unI64 1749025719- -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1fc7'# -> unI64 4191340074107831- -- GREEK CAPITAL LETTER EPSILON WITH VARIA- '\x1fc8'# -> unI64 8050- -- GREEK CAPITAL LETTER EPSILON WITH OXIA- '\x1fc9'# -> unI64 8051- -- GREEK CAPITAL LETTER ETA WITH VARIA- '\x1fca'# -> unI64 8052- -- GREEK CAPITAL LETTER ETA WITH OXIA- '\x1fcb'# -> unI64 8053- -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI- '\x1fcc'# -> unI64 1998586807- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA- '\x1fd2'# -> unI64 3377701347918777- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA- '\x1fd3'# -> unI64 3382099394429881- -- GREEK SMALL LETTER IOTA WITH PERISPOMENI- '\x1fd6'# -> unI64 1749025721- -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI- '\x1fd7'# -> unI64 3667972417651641- -- GREEK CAPITAL LETTER IOTA WITH VRACHY- '\x1fd8'# -> unI64 8144- -- GREEK CAPITAL LETTER IOTA WITH MACRON- '\x1fd9'# -> unI64 8145- -- GREEK CAPITAL LETTER IOTA WITH VARIA- '\x1fda'# -> unI64 8054- -- GREEK CAPITAL LETTER IOTA WITH OXIA- '\x1fdb'# -> unI64 8055- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA- '\x1fe2'# -> unI64 3377701347918789- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA- '\x1fe3'# -> unI64 3382099394429893- -- GREEK SMALL LETTER RHO WITH PSILI- '\x1fe4'# -> unI64 1650459585- -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI- '\x1fe6'# -> unI64 1749025733- -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI- '\x1fe7'# -> unI64 3667972417651653- -- GREEK CAPITAL LETTER UPSILON WITH VRACHY- '\x1fe8'# -> unI64 8160- -- GREEK CAPITAL LETTER UPSILON WITH MACRON- '\x1fe9'# -> unI64 8161- -- GREEK CAPITAL LETTER UPSILON WITH VARIA- '\x1fea'# -> unI64 8058- -- GREEK CAPITAL LETTER UPSILON WITH OXIA- '\x1feb'# -> unI64 8059- -- GREEK CAPITAL LETTER RHO WITH DASIA- '\x1fec'# -> unI64 8165- -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI- '\x1ff2'# -> unI64 1998593916- -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI- '\x1ff3'# -> unI64 1998586825- -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI- '\x1ff4'# -> unI64 1998586830- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI- '\x1ff6'# -> unI64 1749025737- -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI- '\x1ff7'# -> unI64 4191340074107849- -- GREEK CAPITAL LETTER OMICRON WITH VARIA- '\x1ff8'# -> unI64 8056- -- GREEK CAPITAL LETTER OMICRON WITH OXIA- '\x1ff9'# -> unI64 8057- -- GREEK CAPITAL LETTER OMEGA WITH VARIA- '\x1ffa'# -> unI64 8060- -- GREEK CAPITAL LETTER OMEGA WITH OXIA- '\x1ffb'# -> unI64 8061- -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI- '\x1ffc'# -> unI64 1998586825- -- OHM SIGN- '\x2126'# -> unI64 969- -- KELVIN SIGN- '\x212a'# -> unI64 107- -- ANGSTROM SIGN- '\x212b'# -> unI64 229- -- TURNED CAPITAL F- '\x2132'# -> unI64 8526- -- ROMAN NUMERAL ONE- '\x2160'# -> unI64 8560- -- ROMAN NUMERAL TWO- '\x2161'# -> unI64 8561- -- ROMAN NUMERAL THREE- '\x2162'# -> unI64 8562- -- ROMAN NUMERAL FOUR- '\x2163'# -> unI64 8563- -- ROMAN NUMERAL FIVE- '\x2164'# -> unI64 8564- -- ROMAN NUMERAL SIX- '\x2165'# -> unI64 8565- -- ROMAN NUMERAL SEVEN- '\x2166'# -> unI64 8566- -- ROMAN NUMERAL EIGHT- '\x2167'# -> unI64 8567- -- ROMAN NUMERAL NINE- '\x2168'# -> unI64 8568- -- ROMAN NUMERAL TEN- '\x2169'# -> unI64 8569- -- ROMAN NUMERAL ELEVEN- '\x216a'# -> unI64 8570- -- ROMAN NUMERAL TWELVE- '\x216b'# -> unI64 8571- -- ROMAN NUMERAL FIFTY- '\x216c'# -> unI64 8572- -- ROMAN NUMERAL ONE HUNDRED- '\x216d'# -> unI64 8573- -- ROMAN NUMERAL FIVE HUNDRED- '\x216e'# -> unI64 8574- -- ROMAN NUMERAL ONE THOUSAND- '\x216f'# -> unI64 8575- -- ROMAN NUMERAL REVERSED ONE HUNDRED- '\x2183'# -> unI64 8580- -- CIRCLED LATIN CAPITAL LETTER A- '\x24b6'# -> unI64 9424- -- CIRCLED LATIN CAPITAL LETTER B- '\x24b7'# -> unI64 9425- -- CIRCLED LATIN CAPITAL LETTER C- '\x24b8'# -> unI64 9426- -- CIRCLED LATIN CAPITAL LETTER D- '\x24b9'# -> unI64 9427- -- CIRCLED LATIN CAPITAL LETTER E- '\x24ba'# -> unI64 9428- -- CIRCLED LATIN CAPITAL LETTER F- '\x24bb'# -> unI64 9429- -- CIRCLED LATIN CAPITAL LETTER G- '\x24bc'# -> unI64 9430- -- CIRCLED LATIN CAPITAL LETTER H- '\x24bd'# -> unI64 9431- -- CIRCLED LATIN CAPITAL LETTER I- '\x24be'# -> unI64 9432- -- CIRCLED LATIN CAPITAL LETTER J- '\x24bf'# -> unI64 9433- -- CIRCLED LATIN CAPITAL LETTER K- '\x24c0'# -> unI64 9434- -- CIRCLED LATIN CAPITAL LETTER L- '\x24c1'# -> unI64 9435- -- CIRCLED LATIN CAPITAL LETTER M- '\x24c2'# -> unI64 9436- -- CIRCLED LATIN CAPITAL LETTER N- '\x24c3'# -> unI64 9437- -- CIRCLED LATIN CAPITAL LETTER O- '\x24c4'# -> unI64 9438- -- CIRCLED LATIN CAPITAL LETTER P- '\x24c5'# -> unI64 9439- -- CIRCLED LATIN CAPITAL LETTER Q- '\x24c6'# -> unI64 9440- -- CIRCLED LATIN CAPITAL LETTER R- '\x24c7'# -> unI64 9441- -- CIRCLED LATIN CAPITAL LETTER S- '\x24c8'# -> unI64 9442- -- CIRCLED LATIN CAPITAL LETTER T- '\x24c9'# -> unI64 9443- -- CIRCLED LATIN CAPITAL LETTER U- '\x24ca'# -> unI64 9444- -- CIRCLED LATIN CAPITAL LETTER V- '\x24cb'# -> unI64 9445- -- CIRCLED LATIN CAPITAL LETTER W- '\x24cc'# -> unI64 9446- -- CIRCLED LATIN CAPITAL LETTER X- '\x24cd'# -> unI64 9447- -- CIRCLED LATIN CAPITAL LETTER Y- '\x24ce'# -> unI64 9448- -- CIRCLED LATIN CAPITAL LETTER Z- '\x24cf'# -> unI64 9449- -- GLAGOLITIC CAPITAL LETTER AZU- '\x2c00'# -> unI64 11312- -- GLAGOLITIC CAPITAL LETTER BUKY- '\x2c01'# -> unI64 11313- -- GLAGOLITIC CAPITAL LETTER VEDE- '\x2c02'# -> unI64 11314- -- GLAGOLITIC CAPITAL LETTER GLAGOLI- '\x2c03'# -> unI64 11315- -- GLAGOLITIC CAPITAL LETTER DOBRO- '\x2c04'# -> unI64 11316- -- GLAGOLITIC CAPITAL LETTER YESTU- '\x2c05'# -> unI64 11317- -- GLAGOLITIC CAPITAL LETTER ZHIVETE- '\x2c06'# -> unI64 11318- -- GLAGOLITIC CAPITAL LETTER DZELO- '\x2c07'# -> unI64 11319- -- GLAGOLITIC CAPITAL LETTER ZEMLJA- '\x2c08'# -> unI64 11320- -- GLAGOLITIC CAPITAL LETTER IZHE- '\x2c09'# -> unI64 11321- -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE- '\x2c0a'# -> unI64 11322- -- GLAGOLITIC CAPITAL LETTER I- '\x2c0b'# -> unI64 11323- -- GLAGOLITIC CAPITAL LETTER DJERVI- '\x2c0c'# -> unI64 11324- -- GLAGOLITIC CAPITAL LETTER KAKO- '\x2c0d'# -> unI64 11325- -- GLAGOLITIC CAPITAL LETTER LJUDIJE- '\x2c0e'# -> unI64 11326- -- GLAGOLITIC CAPITAL LETTER MYSLITE- '\x2c0f'# -> unI64 11327- -- GLAGOLITIC CAPITAL LETTER NASHI- '\x2c10'# -> unI64 11328- -- GLAGOLITIC CAPITAL LETTER ONU- '\x2c11'# -> unI64 11329- -- GLAGOLITIC CAPITAL LETTER POKOJI- '\x2c12'# -> unI64 11330- -- GLAGOLITIC CAPITAL LETTER RITSI- '\x2c13'# -> unI64 11331- -- GLAGOLITIC CAPITAL LETTER SLOVO- '\x2c14'# -> unI64 11332- -- GLAGOLITIC CAPITAL LETTER TVRIDO- '\x2c15'# -> unI64 11333- -- GLAGOLITIC CAPITAL LETTER UKU- '\x2c16'# -> unI64 11334- -- GLAGOLITIC CAPITAL LETTER FRITU- '\x2c17'# -> unI64 11335- -- GLAGOLITIC CAPITAL LETTER HERU- '\x2c18'# -> unI64 11336- -- GLAGOLITIC CAPITAL LETTER OTU- '\x2c19'# -> unI64 11337- -- GLAGOLITIC CAPITAL LETTER PE- '\x2c1a'# -> unI64 11338- -- GLAGOLITIC CAPITAL LETTER SHTA- '\x2c1b'# -> unI64 11339- -- GLAGOLITIC CAPITAL LETTER TSI- '\x2c1c'# -> unI64 11340- -- GLAGOLITIC CAPITAL LETTER CHRIVI- '\x2c1d'# -> unI64 11341- -- GLAGOLITIC CAPITAL LETTER SHA- '\x2c1e'# -> unI64 11342- -- GLAGOLITIC CAPITAL LETTER YERU- '\x2c1f'# -> unI64 11343- -- GLAGOLITIC CAPITAL LETTER YERI- '\x2c20'# -> unI64 11344- -- GLAGOLITIC CAPITAL LETTER YATI- '\x2c21'# -> unI64 11345- -- GLAGOLITIC CAPITAL LETTER SPIDERY HA- '\x2c22'# -> unI64 11346- -- GLAGOLITIC CAPITAL LETTER YU- '\x2c23'# -> unI64 11347- -- GLAGOLITIC CAPITAL LETTER SMALL YUS- '\x2c24'# -> unI64 11348- -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL- '\x2c25'# -> unI64 11349- -- GLAGOLITIC CAPITAL LETTER YO- '\x2c26'# -> unI64 11350- -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS- '\x2c27'# -> unI64 11351- -- GLAGOLITIC CAPITAL LETTER BIG YUS- '\x2c28'# -> unI64 11352- -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS- '\x2c29'# -> unI64 11353- -- GLAGOLITIC CAPITAL LETTER FITA- '\x2c2a'# -> unI64 11354- -- GLAGOLITIC CAPITAL LETTER IZHITSA- '\x2c2b'# -> unI64 11355- -- GLAGOLITIC CAPITAL LETTER SHTAPIC- '\x2c2c'# -> unI64 11356- -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A- '\x2c2d'# -> unI64 11357- -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE- '\x2c2e'# -> unI64 11358- -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI- '\x2c2f'# -> unI64 11359- -- LATIN CAPITAL LETTER L WITH DOUBLE BAR- '\x2c60'# -> unI64 11361- -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE- '\x2c62'# -> unI64 619- -- LATIN CAPITAL LETTER P WITH STROKE- '\x2c63'# -> unI64 7549- -- LATIN CAPITAL LETTER R WITH TAIL- '\x2c64'# -> unI64 637- -- LATIN CAPITAL LETTER H WITH DESCENDER- '\x2c67'# -> unI64 11368- -- LATIN CAPITAL LETTER K WITH DESCENDER- '\x2c69'# -> unI64 11370- -- LATIN CAPITAL LETTER Z WITH DESCENDER- '\x2c6b'# -> unI64 11372- -- LATIN CAPITAL LETTER ALPHA- '\x2c6d'# -> unI64 593- -- LATIN CAPITAL LETTER M WITH HOOK- '\x2c6e'# -> unI64 625- -- LATIN CAPITAL LETTER TURNED A- '\x2c6f'# -> unI64 592- -- LATIN CAPITAL LETTER TURNED ALPHA- '\x2c70'# -> unI64 594- -- LATIN CAPITAL LETTER W WITH HOOK- '\x2c72'# -> unI64 11379- -- LATIN CAPITAL LETTER HALF H- '\x2c75'# -> unI64 11382- -- LATIN CAPITAL LETTER S WITH SWASH TAIL- '\x2c7e'# -> unI64 575- -- LATIN CAPITAL LETTER Z WITH SWASH TAIL- '\x2c7f'# -> unI64 576- -- COPTIC CAPITAL LETTER ALFA- '\x2c80'# -> unI64 11393- -- COPTIC CAPITAL LETTER VIDA- '\x2c82'# -> unI64 11395- -- COPTIC CAPITAL LETTER GAMMA- '\x2c84'# -> unI64 11397- -- COPTIC CAPITAL LETTER DALDA- '\x2c86'# -> unI64 11399- -- COPTIC CAPITAL LETTER EIE- '\x2c88'# -> unI64 11401- -- COPTIC CAPITAL LETTER SOU- '\x2c8a'# -> unI64 11403- -- COPTIC CAPITAL LETTER ZATA- '\x2c8c'# -> unI64 11405- -- COPTIC CAPITAL LETTER HATE- '\x2c8e'# -> unI64 11407- -- COPTIC CAPITAL LETTER THETHE- '\x2c90'# -> unI64 11409- -- COPTIC CAPITAL LETTER IAUDA- '\x2c92'# -> unI64 11411- -- COPTIC CAPITAL LETTER KAPA- '\x2c94'# -> unI64 11413- -- COPTIC CAPITAL LETTER LAULA- '\x2c96'# -> unI64 11415- -- COPTIC CAPITAL LETTER MI- '\x2c98'# -> unI64 11417- -- COPTIC CAPITAL LETTER NI- '\x2c9a'# -> unI64 11419- -- COPTIC CAPITAL LETTER KSI- '\x2c9c'# -> unI64 11421- -- COPTIC CAPITAL LETTER O- '\x2c9e'# -> unI64 11423- -- COPTIC CAPITAL LETTER PI- '\x2ca0'# -> unI64 11425- -- COPTIC CAPITAL LETTER RO- '\x2ca2'# -> unI64 11427- -- COPTIC CAPITAL LETTER SIMA- '\x2ca4'# -> unI64 11429- -- COPTIC CAPITAL LETTER TAU- '\x2ca6'# -> unI64 11431- -- COPTIC CAPITAL LETTER UA- '\x2ca8'# -> unI64 11433- -- COPTIC CAPITAL LETTER FI- '\x2caa'# -> unI64 11435- -- COPTIC CAPITAL LETTER KHI- '\x2cac'# -> unI64 11437- -- COPTIC CAPITAL LETTER PSI- '\x2cae'# -> unI64 11439- -- COPTIC CAPITAL LETTER OOU- '\x2cb0'# -> unI64 11441- -- COPTIC CAPITAL LETTER DIALECT-P ALEF- '\x2cb2'# -> unI64 11443- -- COPTIC CAPITAL LETTER OLD COPTIC AIN- '\x2cb4'# -> unI64 11445- -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE- '\x2cb6'# -> unI64 11447- -- COPTIC CAPITAL LETTER DIALECT-P KAPA- '\x2cb8'# -> unI64 11449- -- COPTIC CAPITAL LETTER DIALECT-P NI- '\x2cba'# -> unI64 11451- -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI- '\x2cbc'# -> unI64 11453- -- COPTIC CAPITAL LETTER OLD COPTIC OOU- '\x2cbe'# -> unI64 11455- -- COPTIC CAPITAL LETTER SAMPI- '\x2cc0'# -> unI64 11457- -- COPTIC CAPITAL LETTER CROSSED SHEI- '\x2cc2'# -> unI64 11459- -- COPTIC CAPITAL LETTER OLD COPTIC SHEI- '\x2cc4'# -> unI64 11461- -- COPTIC CAPITAL LETTER OLD COPTIC ESH- '\x2cc6'# -> unI64 11463- -- COPTIC CAPITAL LETTER AKHMIMIC KHEI- '\x2cc8'# -> unI64 11465- -- COPTIC CAPITAL LETTER DIALECT-P HORI- '\x2cca'# -> unI64 11467- -- COPTIC CAPITAL LETTER OLD COPTIC HORI- '\x2ccc'# -> unI64 11469- -- COPTIC CAPITAL LETTER OLD COPTIC HA- '\x2cce'# -> unI64 11471- -- COPTIC CAPITAL LETTER L-SHAPED HA- '\x2cd0'# -> unI64 11473- -- COPTIC CAPITAL LETTER OLD COPTIC HEI- '\x2cd2'# -> unI64 11475- -- COPTIC CAPITAL LETTER OLD COPTIC HAT- '\x2cd4'# -> unI64 11477- -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA- '\x2cd6'# -> unI64 11479- -- COPTIC CAPITAL LETTER OLD COPTIC DJA- '\x2cd8'# -> unI64 11481- -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA- '\x2cda'# -> unI64 11483- -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA- '\x2cdc'# -> unI64 11485- -- COPTIC CAPITAL LETTER OLD NUBIAN NGI- '\x2cde'# -> unI64 11487- -- COPTIC CAPITAL LETTER OLD NUBIAN NYI- '\x2ce0'# -> unI64 11489- -- COPTIC CAPITAL LETTER OLD NUBIAN WAU- '\x2ce2'# -> unI64 11491- -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI- '\x2ceb'# -> unI64 11500- -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA- '\x2ced'# -> unI64 11502- -- COPTIC CAPITAL LETTER BOHAIRIC KHEI- '\x2cf2'# -> unI64 11507- -- CYRILLIC CAPITAL LETTER ZEMLYA- '\xa640'# -> unI64 42561- -- CYRILLIC CAPITAL LETTER DZELO- '\xa642'# -> unI64 42563- -- CYRILLIC CAPITAL LETTER REVERSED DZE- '\xa644'# -> unI64 42565- -- CYRILLIC CAPITAL LETTER IOTA- '\xa646'# -> unI64 42567- -- CYRILLIC CAPITAL LETTER DJERV- '\xa648'# -> unI64 42569- -- CYRILLIC CAPITAL LETTER MONOGRAPH UK- '\xa64a'# -> unI64 42571- -- CYRILLIC CAPITAL LETTER BROAD OMEGA- '\xa64c'# -> unI64 42573- -- CYRILLIC CAPITAL LETTER NEUTRAL YER- '\xa64e'# -> unI64 42575- -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER- '\xa650'# -> unI64 42577- -- CYRILLIC CAPITAL LETTER IOTIFIED YAT- '\xa652'# -> unI64 42579- -- CYRILLIC CAPITAL LETTER REVERSED YU- '\xa654'# -> unI64 42581- -- CYRILLIC CAPITAL LETTER IOTIFIED A- '\xa656'# -> unI64 42583- -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS- '\xa658'# -> unI64 42585- -- CYRILLIC CAPITAL LETTER BLENDED YUS- '\xa65a'# -> unI64 42587- -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS- '\xa65c'# -> unI64 42589- -- CYRILLIC CAPITAL LETTER YN- '\xa65e'# -> unI64 42591- -- CYRILLIC CAPITAL LETTER REVERSED TSE- '\xa660'# -> unI64 42593- -- CYRILLIC CAPITAL LETTER SOFT DE- '\xa662'# -> unI64 42595- -- CYRILLIC CAPITAL LETTER SOFT EL- '\xa664'# -> unI64 42597- -- CYRILLIC CAPITAL LETTER SOFT EM- '\xa666'# -> unI64 42599- -- CYRILLIC CAPITAL LETTER MONOCULAR O- '\xa668'# -> unI64 42601- -- CYRILLIC CAPITAL LETTER BINOCULAR O- '\xa66a'# -> unI64 42603- -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O- '\xa66c'# -> unI64 42605- -- CYRILLIC CAPITAL LETTER DWE- '\xa680'# -> unI64 42625- -- CYRILLIC CAPITAL LETTER DZWE- '\xa682'# -> unI64 42627- -- CYRILLIC CAPITAL LETTER ZHWE- '\xa684'# -> unI64 42629- -- CYRILLIC CAPITAL LETTER CCHE- '\xa686'# -> unI64 42631- -- CYRILLIC CAPITAL LETTER DZZE- '\xa688'# -> unI64 42633- -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK- '\xa68a'# -> unI64 42635- -- CYRILLIC CAPITAL LETTER TWE- '\xa68c'# -> unI64 42637- -- CYRILLIC CAPITAL LETTER TSWE- '\xa68e'# -> unI64 42639- -- CYRILLIC CAPITAL LETTER TSSE- '\xa690'# -> unI64 42641- -- CYRILLIC CAPITAL LETTER TCHE- '\xa692'# -> unI64 42643- -- CYRILLIC CAPITAL LETTER HWE- '\xa694'# -> unI64 42645- -- CYRILLIC CAPITAL LETTER SHWE- '\xa696'# -> unI64 42647- -- CYRILLIC CAPITAL LETTER DOUBLE O- '\xa698'# -> unI64 42649- -- CYRILLIC CAPITAL LETTER CROSSED O- '\xa69a'# -> unI64 42651- -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF- '\xa722'# -> unI64 42787- -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN- '\xa724'# -> unI64 42789- -- LATIN CAPITAL LETTER HENG- '\xa726'# -> unI64 42791- -- LATIN CAPITAL LETTER TZ- '\xa728'# -> unI64 42793- -- LATIN CAPITAL LETTER TRESILLO- '\xa72a'# -> unI64 42795- -- LATIN CAPITAL LETTER CUATRILLO- '\xa72c'# -> unI64 42797- -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA- '\xa72e'# -> unI64 42799- -- LATIN CAPITAL LETTER AA- '\xa732'# -> unI64 42803- -- LATIN CAPITAL LETTER AO- '\xa734'# -> unI64 42805- -- LATIN CAPITAL LETTER AU- '\xa736'# -> unI64 42807- -- LATIN CAPITAL LETTER AV- '\xa738'# -> unI64 42809- -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR- '\xa73a'# -> unI64 42811- -- LATIN CAPITAL LETTER AY- '\xa73c'# -> unI64 42813- -- LATIN CAPITAL LETTER REVERSED C WITH DOT- '\xa73e'# -> unI64 42815- -- LATIN CAPITAL LETTER K WITH STROKE- '\xa740'# -> unI64 42817- -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE- '\xa742'# -> unI64 42819- -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE- '\xa744'# -> unI64 42821- -- LATIN CAPITAL LETTER BROKEN L- '\xa746'# -> unI64 42823- -- LATIN CAPITAL LETTER L WITH HIGH STROKE- '\xa748'# -> unI64 42825- -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY- '\xa74a'# -> unI64 42827- -- LATIN CAPITAL LETTER O WITH LOOP- '\xa74c'# -> unI64 42829- -- LATIN CAPITAL LETTER OO- '\xa74e'# -> unI64 42831- -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER- '\xa750'# -> unI64 42833- -- LATIN CAPITAL LETTER P WITH FLOURISH- '\xa752'# -> unI64 42835- -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL- '\xa754'# -> unI64 42837- -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER- '\xa756'# -> unI64 42839- -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE- '\xa758'# -> unI64 42841- -- LATIN CAPITAL LETTER R ROTUNDA- '\xa75a'# -> unI64 42843- -- LATIN CAPITAL LETTER RUM ROTUNDA- '\xa75c'# -> unI64 42845- -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE- '\xa75e'# -> unI64 42847- -- LATIN CAPITAL LETTER VY- '\xa760'# -> unI64 42849- -- LATIN CAPITAL LETTER VISIGOTHIC Z- '\xa762'# -> unI64 42851- -- LATIN CAPITAL LETTER THORN WITH STROKE- '\xa764'# -> unI64 42853- -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER- '\xa766'# -> unI64 42855- -- LATIN CAPITAL LETTER VEND- '\xa768'# -> unI64 42857- -- LATIN CAPITAL LETTER ET- '\xa76a'# -> unI64 42859- -- LATIN CAPITAL LETTER IS- '\xa76c'# -> unI64 42861- -- LATIN CAPITAL LETTER CON- '\xa76e'# -> unI64 42863- -- LATIN CAPITAL LETTER INSULAR D- '\xa779'# -> unI64 42874- -- LATIN CAPITAL LETTER INSULAR F- '\xa77b'# -> unI64 42876- -- LATIN CAPITAL LETTER INSULAR G- '\xa77d'# -> unI64 7545- -- LATIN CAPITAL LETTER TURNED INSULAR G- '\xa77e'# -> unI64 42879- -- LATIN CAPITAL LETTER TURNED L- '\xa780'# -> unI64 42881- -- LATIN CAPITAL LETTER INSULAR R- '\xa782'# -> unI64 42883- -- LATIN CAPITAL LETTER INSULAR S- '\xa784'# -> unI64 42885- -- LATIN CAPITAL LETTER INSULAR T- '\xa786'# -> unI64 42887- -- LATIN CAPITAL LETTER SALTILLO- '\xa78b'# -> unI64 42892- -- LATIN CAPITAL LETTER TURNED H- '\xa78d'# -> unI64 613- -- LATIN CAPITAL LETTER N WITH DESCENDER- '\xa790'# -> unI64 42897- -- LATIN CAPITAL LETTER C WITH BAR- '\xa792'# -> unI64 42899- -- LATIN CAPITAL LETTER B WITH FLOURISH- '\xa796'# -> unI64 42903- -- LATIN CAPITAL LETTER F WITH STROKE- '\xa798'# -> unI64 42905- -- LATIN CAPITAL LETTER VOLAPUK AE- '\xa79a'# -> unI64 42907- -- LATIN CAPITAL LETTER VOLAPUK OE- '\xa79c'# -> unI64 42909- -- LATIN CAPITAL LETTER VOLAPUK UE- '\xa79e'# -> unI64 42911- -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE- '\xa7a0'# -> unI64 42913- -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE- '\xa7a2'# -> unI64 42915- -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE- '\xa7a4'# -> unI64 42917- -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE- '\xa7a6'# -> unI64 42919- -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE- '\xa7a8'# -> unI64 42921- -- LATIN CAPITAL LETTER H WITH HOOK- '\xa7aa'# -> unI64 614- -- LATIN CAPITAL LETTER REVERSED OPEN E- '\xa7ab'# -> unI64 604- -- LATIN CAPITAL LETTER SCRIPT G- '\xa7ac'# -> unI64 609- -- LATIN CAPITAL LETTER L WITH BELT- '\xa7ad'# -> unI64 620- -- LATIN CAPITAL LETTER SMALL CAPITAL I- '\xa7ae'# -> unI64 618- -- LATIN CAPITAL LETTER TURNED K- '\xa7b0'# -> unI64 670- -- LATIN CAPITAL LETTER TURNED T- '\xa7b1'# -> unI64 647- -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL- '\xa7b2'# -> unI64 669- -- LATIN CAPITAL LETTER CHI- '\xa7b3'# -> unI64 43859- -- LATIN CAPITAL LETTER BETA- '\xa7b4'# -> unI64 42933- -- LATIN CAPITAL LETTER OMEGA- '\xa7b6'# -> unI64 42935- -- LATIN CAPITAL LETTER U WITH STROKE- '\xa7b8'# -> unI64 42937- -- LATIN CAPITAL LETTER GLOTTAL A- '\xa7ba'# -> unI64 42939- -- LATIN CAPITAL LETTER GLOTTAL I- '\xa7bc'# -> unI64 42941- -- LATIN CAPITAL LETTER GLOTTAL U- '\xa7be'# -> unI64 42943- -- LATIN CAPITAL LETTER OLD POLISH O- '\xa7c0'# -> unI64 42945- -- LATIN CAPITAL LETTER ANGLICANA W- '\xa7c2'# -> unI64 42947- -- LATIN CAPITAL LETTER C WITH PALATAL HOOK- '\xa7c4'# -> unI64 42900- -- LATIN CAPITAL LETTER S WITH HOOK- '\xa7c5'# -> unI64 642- -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK- '\xa7c6'# -> unI64 7566- -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY- '\xa7c7'# -> unI64 42952- -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY- '\xa7c9'# -> unI64 42954- -- LATIN CAPITAL LETTER RAMS HORN- '\xa7cb'# -> unI64 612- -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE- '\xa7cc'# -> unI64 42957- -- LATIN CAPITAL LETTER CLOSED INSULAR G- '\xa7d0'# -> unI64 42961- -- LATIN CAPITAL LETTER MIDDLE SCOTS S- '\xa7d6'# -> unI64 42967- -- LATIN CAPITAL LETTER SIGMOID S- '\xa7d8'# -> unI64 42969- -- LATIN CAPITAL LETTER LAMBDA- '\xa7da'# -> unI64 42971- -- LATIN CAPITAL LETTER LAMBDA WITH STROKE- '\xa7dc'# -> unI64 411- -- LATIN CAPITAL LETTER REVERSED HALF H- '\xa7f5'# -> unI64 42998- -- CHEROKEE SMALL LETTER A- '\xab70'# -> unI64 5024- -- CHEROKEE SMALL LETTER E- '\xab71'# -> unI64 5025- -- CHEROKEE SMALL LETTER I- '\xab72'# -> unI64 5026- -- CHEROKEE SMALL LETTER O- '\xab73'# -> unI64 5027- -- CHEROKEE SMALL LETTER U- '\xab74'# -> unI64 5028- -- CHEROKEE SMALL LETTER V- '\xab75'# -> unI64 5029- -- CHEROKEE SMALL LETTER GA- '\xab76'# -> unI64 5030- -- CHEROKEE SMALL LETTER KA- '\xab77'# -> unI64 5031- -- CHEROKEE SMALL LETTER GE- '\xab78'# -> unI64 5032- -- CHEROKEE SMALL LETTER GI- '\xab79'# -> unI64 5033- -- CHEROKEE SMALL LETTER GO- '\xab7a'# -> unI64 5034- -- CHEROKEE SMALL LETTER GU- '\xab7b'# -> unI64 5035- -- CHEROKEE SMALL LETTER GV- '\xab7c'# -> unI64 5036- -- CHEROKEE SMALL LETTER HA- '\xab7d'# -> unI64 5037- -- CHEROKEE SMALL LETTER HE- '\xab7e'# -> unI64 5038- -- CHEROKEE SMALL LETTER HI- '\xab7f'# -> unI64 5039- -- CHEROKEE SMALL LETTER HO- '\xab80'# -> unI64 5040- -- CHEROKEE SMALL LETTER HU- '\xab81'# -> unI64 5041- -- CHEROKEE SMALL LETTER HV- '\xab82'# -> unI64 5042- -- CHEROKEE SMALL LETTER LA- '\xab83'# -> unI64 5043- -- CHEROKEE SMALL LETTER LE- '\xab84'# -> unI64 5044- -- CHEROKEE SMALL LETTER LI- '\xab85'# -> unI64 5045- -- CHEROKEE SMALL LETTER LO- '\xab86'# -> unI64 5046- -- CHEROKEE SMALL LETTER LU- '\xab87'# -> unI64 5047- -- CHEROKEE SMALL LETTER LV- '\xab88'# -> unI64 5048- -- CHEROKEE SMALL LETTER MA- '\xab89'# -> unI64 5049- -- CHEROKEE SMALL LETTER ME- '\xab8a'# -> unI64 5050- -- CHEROKEE SMALL LETTER MI- '\xab8b'# -> unI64 5051- -- CHEROKEE SMALL LETTER MO- '\xab8c'# -> unI64 5052- -- CHEROKEE SMALL LETTER MU- '\xab8d'# -> unI64 5053- -- CHEROKEE SMALL LETTER NA- '\xab8e'# -> unI64 5054- -- CHEROKEE SMALL LETTER HNA- '\xab8f'# -> unI64 5055- -- CHEROKEE SMALL LETTER NAH- '\xab90'# -> unI64 5056- -- CHEROKEE SMALL LETTER NE- '\xab91'# -> unI64 5057- -- CHEROKEE SMALL LETTER NI- '\xab92'# -> unI64 5058- -- CHEROKEE SMALL LETTER NO- '\xab93'# -> unI64 5059- -- CHEROKEE SMALL LETTER NU- '\xab94'# -> unI64 5060- -- CHEROKEE SMALL LETTER NV- '\xab95'# -> unI64 5061- -- CHEROKEE SMALL LETTER QUA- '\xab96'# -> unI64 5062- -- CHEROKEE SMALL LETTER QUE- '\xab97'# -> unI64 5063- -- CHEROKEE SMALL LETTER QUI- '\xab98'# -> unI64 5064- -- CHEROKEE SMALL LETTER QUO- '\xab99'# -> unI64 5065- -- CHEROKEE SMALL LETTER QUU- '\xab9a'# -> unI64 5066- -- CHEROKEE SMALL LETTER QUV- '\xab9b'# -> unI64 5067- -- CHEROKEE SMALL LETTER SA- '\xab9c'# -> unI64 5068- -- CHEROKEE SMALL LETTER S- '\xab9d'# -> unI64 5069- -- CHEROKEE SMALL LETTER SE- '\xab9e'# -> unI64 5070- -- CHEROKEE SMALL LETTER SI- '\xab9f'# -> unI64 5071- -- CHEROKEE SMALL LETTER SO- '\xaba0'# -> unI64 5072- -- CHEROKEE SMALL LETTER SU- '\xaba1'# -> unI64 5073- -- CHEROKEE SMALL LETTER SV- '\xaba2'# -> unI64 5074- -- CHEROKEE SMALL LETTER DA- '\xaba3'# -> unI64 5075- -- CHEROKEE SMALL LETTER TA- '\xaba4'# -> unI64 5076- -- CHEROKEE SMALL LETTER DE- '\xaba5'# -> unI64 5077- -- CHEROKEE SMALL LETTER TE- '\xaba6'# -> unI64 5078- -- CHEROKEE SMALL LETTER DI- '\xaba7'# -> unI64 5079- -- CHEROKEE SMALL LETTER TI- '\xaba8'# -> unI64 5080- -- CHEROKEE SMALL LETTER DO- '\xaba9'# -> unI64 5081- -- CHEROKEE SMALL LETTER DU- '\xabaa'# -> unI64 5082- -- CHEROKEE SMALL LETTER DV- '\xabab'# -> unI64 5083- -- CHEROKEE SMALL LETTER DLA- '\xabac'# -> unI64 5084- -- CHEROKEE SMALL LETTER TLA- '\xabad'# -> unI64 5085- -- CHEROKEE SMALL LETTER TLE- '\xabae'# -> unI64 5086- -- CHEROKEE SMALL LETTER TLI- '\xabaf'# -> unI64 5087- -- CHEROKEE SMALL LETTER TLO- '\xabb0'# -> unI64 5088- -- CHEROKEE SMALL LETTER TLU- '\xabb1'# -> unI64 5089- -- CHEROKEE SMALL LETTER TLV- '\xabb2'# -> unI64 5090- -- CHEROKEE SMALL LETTER TSA- '\xabb3'# -> unI64 5091- -- CHEROKEE SMALL LETTER TSE- '\xabb4'# -> unI64 5092- -- CHEROKEE SMALL LETTER TSI- '\xabb5'# -> unI64 5093- -- CHEROKEE SMALL LETTER TSO- '\xabb6'# -> unI64 5094- -- CHEROKEE SMALL LETTER TSU- '\xabb7'# -> unI64 5095- -- CHEROKEE SMALL LETTER TSV- '\xabb8'# -> unI64 5096- -- CHEROKEE SMALL LETTER WA- '\xabb9'# -> unI64 5097- -- CHEROKEE SMALL LETTER WE- '\xabba'# -> unI64 5098- -- CHEROKEE SMALL LETTER WI- '\xabbb'# -> unI64 5099- -- CHEROKEE SMALL LETTER WO- '\xabbc'# -> unI64 5100- -- CHEROKEE SMALL LETTER WU- '\xabbd'# -> unI64 5101- -- CHEROKEE SMALL LETTER WV- '\xabbe'# -> unI64 5102- -- CHEROKEE SMALL LETTER YA- '\xabbf'# -> unI64 5103- -- LATIN SMALL LIGATURE FF- '\xfb00'# -> unI64 213909606- -- LATIN SMALL LIGATURE FI- '\xfb01'# -> unI64 220201062- -- LATIN SMALL LIGATURE FL- '\xfb02'# -> unI64 226492518- -- LATIN SMALL LIGATURE FFI- '\xfb03'# -> unI64 461795097575526- -- LATIN SMALL LIGATURE FFL- '\xfb04'# -> unI64 474989237108838- -- LATIN SMALL LIGATURE LONG S T- '\xfb05'# -> unI64 243269747- -- LATIN SMALL LIGATURE ST- '\xfb06'# -> unI64 243269747- -- ARMENIAN SMALL LIGATURE MEN NOW- '\xfb13'# -> unI64 2931819892- -- ARMENIAN SMALL LIGATURE MEN ECH- '\xfb14'# -> unI64 2896168308- -- ARMENIAN SMALL LIGATURE MEN INI- '\xfb15'# -> unI64 2908751220- -- ARMENIAN SMALL LIGATURE VEW NOW- '\xfb16'# -> unI64 2931819902- -- ARMENIAN SMALL LIGATURE MEN XEH- '\xfb17'# -> unI64 2912945524- -- FULLWIDTH LATIN CAPITAL LETTER A- '\xff21'# -> unI64 65345- -- FULLWIDTH LATIN CAPITAL LETTER B- '\xff22'# -> unI64 65346- -- FULLWIDTH LATIN CAPITAL LETTER C- '\xff23'# -> unI64 65347- -- FULLWIDTH LATIN CAPITAL LETTER D- '\xff24'# -> unI64 65348- -- FULLWIDTH LATIN CAPITAL LETTER E- '\xff25'# -> unI64 65349- -- FULLWIDTH LATIN CAPITAL LETTER F- '\xff26'# -> unI64 65350- -- FULLWIDTH LATIN CAPITAL LETTER G- '\xff27'# -> unI64 65351- -- FULLWIDTH LATIN CAPITAL LETTER H- '\xff28'# -> unI64 65352- -- FULLWIDTH LATIN CAPITAL LETTER I- '\xff29'# -> unI64 65353- -- FULLWIDTH LATIN CAPITAL LETTER J- '\xff2a'# -> unI64 65354- -- FULLWIDTH LATIN CAPITAL LETTER K- '\xff2b'# -> unI64 65355- -- FULLWIDTH LATIN CAPITAL LETTER L- '\xff2c'# -> unI64 65356- -- FULLWIDTH LATIN CAPITAL LETTER M- '\xff2d'# -> unI64 65357- -- FULLWIDTH LATIN CAPITAL LETTER N- '\xff2e'# -> unI64 65358- -- FULLWIDTH LATIN CAPITAL LETTER O- '\xff2f'# -> unI64 65359- -- FULLWIDTH LATIN CAPITAL LETTER P- '\xff30'# -> unI64 65360- -- FULLWIDTH LATIN CAPITAL LETTER Q- '\xff31'# -> unI64 65361- -- FULLWIDTH LATIN CAPITAL LETTER R- '\xff32'# -> unI64 65362- -- FULLWIDTH LATIN CAPITAL LETTER S- '\xff33'# -> unI64 65363- -- FULLWIDTH LATIN CAPITAL LETTER T- '\xff34'# -> unI64 65364- -- FULLWIDTH LATIN CAPITAL LETTER U- '\xff35'# -> unI64 65365- -- FULLWIDTH LATIN CAPITAL LETTER V- '\xff36'# -> unI64 65366- -- FULLWIDTH LATIN CAPITAL LETTER W- '\xff37'# -> unI64 65367- -- FULLWIDTH LATIN CAPITAL LETTER X- '\xff38'# -> unI64 65368- -- FULLWIDTH LATIN CAPITAL LETTER Y- '\xff39'# -> unI64 65369- -- FULLWIDTH LATIN CAPITAL LETTER Z- '\xff3a'# -> unI64 65370- -- DESERET CAPITAL LETTER LONG I- '\x10400'# -> unI64 66600- -- DESERET CAPITAL LETTER LONG E- '\x10401'# -> unI64 66601- -- DESERET CAPITAL LETTER LONG A- '\x10402'# -> unI64 66602- -- DESERET CAPITAL LETTER LONG AH- '\x10403'# -> unI64 66603- -- DESERET CAPITAL LETTER LONG O- '\x10404'# -> unI64 66604- -- DESERET CAPITAL LETTER LONG OO- '\x10405'# -> unI64 66605- -- DESERET CAPITAL LETTER SHORT I- '\x10406'# -> unI64 66606- -- DESERET CAPITAL LETTER SHORT E- '\x10407'# -> unI64 66607- -- DESERET CAPITAL LETTER SHORT A- '\x10408'# -> unI64 66608- -- DESERET CAPITAL LETTER SHORT AH- '\x10409'# -> unI64 66609- -- DESERET CAPITAL LETTER SHORT O- '\x1040a'# -> unI64 66610- -- DESERET CAPITAL LETTER SHORT OO- '\x1040b'# -> unI64 66611- -- DESERET CAPITAL LETTER AY- '\x1040c'# -> unI64 66612- -- DESERET CAPITAL LETTER OW- '\x1040d'# -> unI64 66613- -- DESERET CAPITAL LETTER WU- '\x1040e'# -> unI64 66614- -- DESERET CAPITAL LETTER YEE- '\x1040f'# -> unI64 66615- -- DESERET CAPITAL LETTER H- '\x10410'# -> unI64 66616- -- DESERET CAPITAL LETTER PEE- '\x10411'# -> unI64 66617- -- DESERET CAPITAL LETTER BEE- '\x10412'# -> unI64 66618- -- DESERET CAPITAL LETTER TEE- '\x10413'# -> unI64 66619- -- DESERET CAPITAL LETTER DEE- '\x10414'# -> unI64 66620- -- DESERET CAPITAL LETTER CHEE- '\x10415'# -> unI64 66621- -- DESERET CAPITAL LETTER JEE- '\x10416'# -> unI64 66622- -- DESERET CAPITAL LETTER KAY- '\x10417'# -> unI64 66623- -- DESERET CAPITAL LETTER GAY- '\x10418'# -> unI64 66624- -- DESERET CAPITAL LETTER EF- '\x10419'# -> unI64 66625- -- DESERET CAPITAL LETTER VEE- '\x1041a'# -> unI64 66626- -- DESERET CAPITAL LETTER ETH- '\x1041b'# -> unI64 66627- -- DESERET CAPITAL LETTER THEE- '\x1041c'# -> unI64 66628- -- DESERET CAPITAL LETTER ES- '\x1041d'# -> unI64 66629- -- DESERET CAPITAL LETTER ZEE- '\x1041e'# -> unI64 66630- -- DESERET CAPITAL LETTER ESH- '\x1041f'# -> unI64 66631- -- DESERET CAPITAL LETTER ZHEE- '\x10420'# -> unI64 66632- -- DESERET CAPITAL LETTER ER- '\x10421'# -> unI64 66633- -- DESERET CAPITAL LETTER EL- '\x10422'# -> unI64 66634- -- DESERET CAPITAL LETTER EM- '\x10423'# -> unI64 66635- -- DESERET CAPITAL LETTER EN- '\x10424'# -> unI64 66636- -- DESERET CAPITAL LETTER ENG- '\x10425'# -> unI64 66637- -- DESERET CAPITAL LETTER OI- '\x10426'# -> unI64 66638- -- DESERET CAPITAL LETTER EW- '\x10427'# -> unI64 66639- -- OSAGE CAPITAL LETTER A- '\x104b0'# -> unI64 66776- -- OSAGE CAPITAL LETTER AI- '\x104b1'# -> unI64 66777- -- OSAGE CAPITAL LETTER AIN- '\x104b2'# -> unI64 66778- -- OSAGE CAPITAL LETTER AH- '\x104b3'# -> unI64 66779- -- OSAGE CAPITAL LETTER BRA- '\x104b4'# -> unI64 66780- -- OSAGE CAPITAL LETTER CHA- '\x104b5'# -> unI64 66781- -- OSAGE CAPITAL LETTER EHCHA- '\x104b6'# -> unI64 66782- -- OSAGE CAPITAL LETTER E- '\x104b7'# -> unI64 66783- -- OSAGE CAPITAL LETTER EIN- '\x104b8'# -> unI64 66784- -- OSAGE CAPITAL LETTER HA- '\x104b9'# -> unI64 66785- -- OSAGE CAPITAL LETTER HYA- '\x104ba'# -> unI64 66786- -- OSAGE CAPITAL LETTER I- '\x104bb'# -> unI64 66787- -- OSAGE CAPITAL LETTER KA- '\x104bc'# -> unI64 66788- -- OSAGE CAPITAL LETTER EHKA- '\x104bd'# -> unI64 66789- -- OSAGE CAPITAL LETTER KYA- '\x104be'# -> unI64 66790- -- OSAGE CAPITAL LETTER LA- '\x104bf'# -> unI64 66791- -- OSAGE CAPITAL LETTER MA- '\x104c0'# -> unI64 66792- -- OSAGE CAPITAL LETTER NA- '\x104c1'# -> unI64 66793- -- OSAGE CAPITAL LETTER O- '\x104c2'# -> unI64 66794- -- OSAGE CAPITAL LETTER OIN- '\x104c3'# -> unI64 66795- -- OSAGE CAPITAL LETTER PA- '\x104c4'# -> unI64 66796- -- OSAGE CAPITAL LETTER EHPA- '\x104c5'# -> unI64 66797- -- OSAGE CAPITAL LETTER SA- '\x104c6'# -> unI64 66798- -- OSAGE CAPITAL LETTER SHA- '\x104c7'# -> unI64 66799- -- OSAGE CAPITAL LETTER TA- '\x104c8'# -> unI64 66800- -- OSAGE CAPITAL LETTER EHTA- '\x104c9'# -> unI64 66801- -- OSAGE CAPITAL LETTER TSA- '\x104ca'# -> unI64 66802- -- OSAGE CAPITAL LETTER EHTSA- '\x104cb'# -> unI64 66803- -- OSAGE CAPITAL LETTER TSHA- '\x104cc'# -> unI64 66804- -- OSAGE CAPITAL LETTER DHA- '\x104cd'# -> unI64 66805- -- OSAGE CAPITAL LETTER U- '\x104ce'# -> unI64 66806- -- OSAGE CAPITAL LETTER WA- '\x104cf'# -> unI64 66807- -- OSAGE CAPITAL LETTER KHA- '\x104d0'# -> unI64 66808- -- OSAGE CAPITAL LETTER GHA- '\x104d1'# -> unI64 66809- -- OSAGE CAPITAL LETTER ZA- '\x104d2'# -> unI64 66810- -- OSAGE CAPITAL LETTER ZHA- '\x104d3'# -> unI64 66811- -- VITHKUQI CAPITAL LETTER A- '\x10570'# -> unI64 66967- -- VITHKUQI CAPITAL LETTER BBE- '\x10571'# -> unI64 66968- -- VITHKUQI CAPITAL LETTER BE- '\x10572'# -> unI64 66969- -- VITHKUQI CAPITAL LETTER CE- '\x10573'# -> unI64 66970- -- VITHKUQI CAPITAL LETTER CHE- '\x10574'# -> unI64 66971- -- VITHKUQI CAPITAL LETTER DE- '\x10575'# -> unI64 66972- -- VITHKUQI CAPITAL LETTER DHE- '\x10576'# -> unI64 66973- -- VITHKUQI CAPITAL LETTER EI- '\x10577'# -> unI64 66974- -- VITHKUQI CAPITAL LETTER E- '\x10578'# -> unI64 66975- -- VITHKUQI CAPITAL LETTER FE- '\x10579'# -> unI64 66976- -- VITHKUQI CAPITAL LETTER GA- '\x1057a'# -> unI64 66977- -- VITHKUQI CAPITAL LETTER HA- '\x1057c'# -> unI64 66979- -- VITHKUQI CAPITAL LETTER HHA- '\x1057d'# -> unI64 66980- -- VITHKUQI CAPITAL LETTER I- '\x1057e'# -> unI64 66981- -- VITHKUQI CAPITAL LETTER IJE- '\x1057f'# -> unI64 66982- -- VITHKUQI CAPITAL LETTER JE- '\x10580'# -> unI64 66983- -- VITHKUQI CAPITAL LETTER KA- '\x10581'# -> unI64 66984- -- VITHKUQI CAPITAL LETTER LA- '\x10582'# -> unI64 66985- -- VITHKUQI CAPITAL LETTER LLA- '\x10583'# -> unI64 66986- -- VITHKUQI CAPITAL LETTER ME- '\x10584'# -> unI64 66987- -- VITHKUQI CAPITAL LETTER NE- '\x10585'# -> unI64 66988- -- VITHKUQI CAPITAL LETTER NJE- '\x10586'# -> unI64 66989- -- VITHKUQI CAPITAL LETTER O- '\x10587'# -> unI64 66990- -- VITHKUQI CAPITAL LETTER PE- '\x10588'# -> unI64 66991- -- VITHKUQI CAPITAL LETTER QA- '\x10589'# -> unI64 66992- -- VITHKUQI CAPITAL LETTER RE- '\x1058a'# -> unI64 66993- -- VITHKUQI CAPITAL LETTER SE- '\x1058c'# -> unI64 66995- -- VITHKUQI CAPITAL LETTER SHE- '\x1058d'# -> unI64 66996- -- VITHKUQI CAPITAL LETTER TE- '\x1058e'# -> unI64 66997- -- VITHKUQI CAPITAL LETTER THE- '\x1058f'# -> unI64 66998- -- VITHKUQI CAPITAL LETTER U- '\x10590'# -> unI64 66999- -- VITHKUQI CAPITAL LETTER VE- '\x10591'# -> unI64 67000- -- VITHKUQI CAPITAL LETTER XE- '\x10592'# -> unI64 67001- -- VITHKUQI CAPITAL LETTER Y- '\x10594'# -> unI64 67003- -- VITHKUQI CAPITAL LETTER ZE- '\x10595'# -> unI64 67004- -- OLD HUNGARIAN CAPITAL LETTER A- '\x10c80'# -> unI64 68800- -- OLD HUNGARIAN CAPITAL LETTER AA- '\x10c81'# -> unI64 68801- -- OLD HUNGARIAN CAPITAL LETTER EB- '\x10c82'# -> unI64 68802- -- OLD HUNGARIAN CAPITAL LETTER AMB- '\x10c83'# -> unI64 68803- -- OLD HUNGARIAN CAPITAL LETTER EC- '\x10c84'# -> unI64 68804- -- OLD HUNGARIAN CAPITAL LETTER ENC- '\x10c85'# -> unI64 68805- -- OLD HUNGARIAN CAPITAL LETTER ECS- '\x10c86'# -> unI64 68806- -- OLD HUNGARIAN CAPITAL LETTER ED- '\x10c87'# -> unI64 68807- -- OLD HUNGARIAN CAPITAL LETTER AND- '\x10c88'# -> unI64 68808- -- OLD HUNGARIAN CAPITAL LETTER E- '\x10c89'# -> unI64 68809- -- OLD HUNGARIAN CAPITAL LETTER CLOSE E- '\x10c8a'# -> unI64 68810- -- OLD HUNGARIAN CAPITAL LETTER EE- '\x10c8b'# -> unI64 68811- -- OLD HUNGARIAN CAPITAL LETTER EF- '\x10c8c'# -> unI64 68812- -- OLD HUNGARIAN CAPITAL LETTER EG- '\x10c8d'# -> unI64 68813- -- OLD HUNGARIAN CAPITAL LETTER EGY- '\x10c8e'# -> unI64 68814- -- OLD HUNGARIAN CAPITAL LETTER EH- '\x10c8f'# -> unI64 68815- -- OLD HUNGARIAN CAPITAL LETTER I- '\x10c90'# -> unI64 68816- -- OLD HUNGARIAN CAPITAL LETTER II- '\x10c91'# -> unI64 68817- -- OLD HUNGARIAN CAPITAL LETTER EJ- '\x10c92'# -> unI64 68818- -- OLD HUNGARIAN CAPITAL LETTER EK- '\x10c93'# -> unI64 68819- -- OLD HUNGARIAN CAPITAL LETTER AK- '\x10c94'# -> unI64 68820- -- OLD HUNGARIAN CAPITAL LETTER UNK- '\x10c95'# -> unI64 68821- -- OLD HUNGARIAN CAPITAL LETTER EL- '\x10c96'# -> unI64 68822- -- OLD HUNGARIAN CAPITAL LETTER ELY- '\x10c97'# -> unI64 68823- -- OLD HUNGARIAN CAPITAL LETTER EM- '\x10c98'# -> unI64 68824- -- OLD HUNGARIAN CAPITAL LETTER EN- '\x10c99'# -> unI64 68825- -- OLD HUNGARIAN CAPITAL LETTER ENY- '\x10c9a'# -> unI64 68826- -- OLD HUNGARIAN CAPITAL LETTER O- '\x10c9b'# -> unI64 68827- -- OLD HUNGARIAN CAPITAL LETTER OO- '\x10c9c'# -> unI64 68828- -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE- '\x10c9d'# -> unI64 68829- -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE- '\x10c9e'# -> unI64 68830- -- OLD HUNGARIAN CAPITAL LETTER OEE- '\x10c9f'# -> unI64 68831- -- OLD HUNGARIAN CAPITAL LETTER EP- '\x10ca0'# -> unI64 68832- -- OLD HUNGARIAN CAPITAL LETTER EMP- '\x10ca1'# -> unI64 68833- -- OLD HUNGARIAN CAPITAL LETTER ER- '\x10ca2'# -> unI64 68834- -- OLD HUNGARIAN CAPITAL LETTER SHORT ER- '\x10ca3'# -> unI64 68835- -- OLD HUNGARIAN CAPITAL LETTER ES- '\x10ca4'# -> unI64 68836- -- OLD HUNGARIAN CAPITAL LETTER ESZ- '\x10ca5'# -> unI64 68837- -- OLD HUNGARIAN CAPITAL LETTER ET- '\x10ca6'# -> unI64 68838- -- OLD HUNGARIAN CAPITAL LETTER ENT- '\x10ca7'# -> unI64 68839- -- OLD HUNGARIAN CAPITAL LETTER ETY- '\x10ca8'# -> unI64 68840- -- OLD HUNGARIAN CAPITAL LETTER ECH- '\x10ca9'# -> unI64 68841- -- OLD HUNGARIAN CAPITAL LETTER U- '\x10caa'# -> unI64 68842- -- OLD HUNGARIAN CAPITAL LETTER UU- '\x10cab'# -> unI64 68843- -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE- '\x10cac'# -> unI64 68844- -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE- '\x10cad'# -> unI64 68845- -- OLD HUNGARIAN CAPITAL LETTER EV- '\x10cae'# -> unI64 68846- -- OLD HUNGARIAN CAPITAL LETTER EZ- '\x10caf'# -> unI64 68847- -- OLD HUNGARIAN CAPITAL LETTER EZS- '\x10cb0'# -> unI64 68848- -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN- '\x10cb1'# -> unI64 68849- -- OLD HUNGARIAN CAPITAL LETTER US- '\x10cb2'# -> unI64 68850- -- GARAY CAPITAL LETTER A- '\x10d50'# -> unI64 68976- -- GARAY CAPITAL LETTER CA- '\x10d51'# -> unI64 68977- -- GARAY CAPITAL LETTER MA- '\x10d52'# -> unI64 68978- -- GARAY CAPITAL LETTER KA- '\x10d53'# -> unI64 68979- -- GARAY CAPITAL LETTER BA- '\x10d54'# -> unI64 68980- -- GARAY CAPITAL LETTER JA- '\x10d55'# -> unI64 68981- -- GARAY CAPITAL LETTER SA- '\x10d56'# -> unI64 68982- -- GARAY CAPITAL LETTER WA- '\x10d57'# -> unI64 68983- -- GARAY CAPITAL LETTER LA- '\x10d58'# -> unI64 68984- -- GARAY CAPITAL LETTER GA- '\x10d59'# -> unI64 68985- -- GARAY CAPITAL LETTER DA- '\x10d5a'# -> unI64 68986- -- GARAY CAPITAL LETTER XA- '\x10d5b'# -> unI64 68987- -- GARAY CAPITAL LETTER YA- '\x10d5c'# -> unI64 68988- -- GARAY CAPITAL LETTER TA- '\x10d5d'# -> unI64 68989- -- GARAY CAPITAL LETTER RA- '\x10d5e'# -> unI64 68990- -- GARAY CAPITAL LETTER NYA- '\x10d5f'# -> unI64 68991- -- GARAY CAPITAL LETTER FA- '\x10d60'# -> unI64 68992- -- GARAY CAPITAL LETTER NA- '\x10d61'# -> unI64 68993- -- GARAY CAPITAL LETTER PA- '\x10d62'# -> unI64 68994- -- GARAY CAPITAL LETTER HA- '\x10d63'# -> unI64 68995- -- GARAY CAPITAL LETTER OLD KA- '\x10d64'# -> unI64 68996- -- GARAY CAPITAL LETTER OLD NA- '\x10d65'# -> unI64 68997- -- WARANG CITI CAPITAL LETTER NGAA- '\x118a0'# -> unI64 71872- -- WARANG CITI CAPITAL LETTER A- '\x118a1'# -> unI64 71873- -- WARANG CITI CAPITAL LETTER WI- '\x118a2'# -> unI64 71874- -- WARANG CITI CAPITAL LETTER YU- '\x118a3'# -> unI64 71875- -- WARANG CITI CAPITAL LETTER YA- '\x118a4'# -> unI64 71876- -- WARANG CITI CAPITAL LETTER YO- '\x118a5'# -> unI64 71877- -- WARANG CITI CAPITAL LETTER II- '\x118a6'# -> unI64 71878- -- WARANG CITI CAPITAL LETTER UU- '\x118a7'# -> unI64 71879- -- WARANG CITI CAPITAL LETTER E- '\x118a8'# -> unI64 71880- -- WARANG CITI CAPITAL LETTER O- '\x118a9'# -> unI64 71881- -- WARANG CITI CAPITAL LETTER ANG- '\x118aa'# -> unI64 71882- -- WARANG CITI CAPITAL LETTER GA- '\x118ab'# -> unI64 71883- -- WARANG CITI CAPITAL LETTER KO- '\x118ac'# -> unI64 71884- -- WARANG CITI CAPITAL LETTER ENY- '\x118ad'# -> unI64 71885- -- WARANG CITI CAPITAL LETTER YUJ- '\x118ae'# -> unI64 71886- -- WARANG CITI CAPITAL LETTER UC- '\x118af'# -> unI64 71887- -- WARANG CITI CAPITAL LETTER ENN- '\x118b0'# -> unI64 71888- -- WARANG CITI CAPITAL LETTER ODD- '\x118b1'# -> unI64 71889- -- WARANG CITI CAPITAL LETTER TTE- '\x118b2'# -> unI64 71890- -- WARANG CITI CAPITAL LETTER NUNG- '\x118b3'# -> unI64 71891- -- WARANG CITI CAPITAL LETTER DA- '\x118b4'# -> unI64 71892- -- WARANG CITI CAPITAL LETTER AT- '\x118b5'# -> unI64 71893- -- WARANG CITI CAPITAL LETTER AM- '\x118b6'# -> unI64 71894- -- WARANG CITI CAPITAL LETTER BU- '\x118b7'# -> unI64 71895- -- WARANG CITI CAPITAL LETTER PU- '\x118b8'# -> unI64 71896- -- WARANG CITI CAPITAL LETTER HIYO- '\x118b9'# -> unI64 71897- -- WARANG CITI CAPITAL LETTER HOLO- '\x118ba'# -> unI64 71898- -- WARANG CITI CAPITAL LETTER HORR- '\x118bb'# -> unI64 71899- -- WARANG CITI CAPITAL LETTER HAR- '\x118bc'# -> unI64 71900- -- WARANG CITI CAPITAL LETTER SSUU- '\x118bd'# -> unI64 71901- -- WARANG CITI CAPITAL LETTER SII- '\x118be'# -> unI64 71902- -- WARANG CITI CAPITAL LETTER VIYO- '\x118bf'# -> unI64 71903- -- MEDEFAIDRIN CAPITAL LETTER M- '\x16e40'# -> unI64 93792- -- MEDEFAIDRIN CAPITAL LETTER S- '\x16e41'# -> unI64 93793- -- MEDEFAIDRIN CAPITAL LETTER V- '\x16e42'# -> unI64 93794- -- MEDEFAIDRIN CAPITAL LETTER W- '\x16e43'# -> unI64 93795- -- MEDEFAIDRIN CAPITAL LETTER ATIU- '\x16e44'# -> unI64 93796- -- MEDEFAIDRIN CAPITAL LETTER Z- '\x16e45'# -> unI64 93797- -- MEDEFAIDRIN CAPITAL LETTER KP- '\x16e46'# -> unI64 93798- -- MEDEFAIDRIN CAPITAL LETTER P- '\x16e47'# -> unI64 93799- -- MEDEFAIDRIN CAPITAL LETTER T- '\x16e48'# -> unI64 93800- -- MEDEFAIDRIN CAPITAL LETTER G- '\x16e49'# -> unI64 93801- -- MEDEFAIDRIN CAPITAL LETTER F- '\x16e4a'# -> unI64 93802- -- MEDEFAIDRIN CAPITAL LETTER I- '\x16e4b'# -> unI64 93803- -- MEDEFAIDRIN CAPITAL LETTER K- '\x16e4c'# -> unI64 93804- -- MEDEFAIDRIN CAPITAL LETTER A- '\x16e4d'# -> unI64 93805- -- MEDEFAIDRIN CAPITAL LETTER J- '\x16e4e'# -> unI64 93806- -- MEDEFAIDRIN CAPITAL LETTER E- '\x16e4f'# -> unI64 93807- -- MEDEFAIDRIN CAPITAL LETTER B- '\x16e50'# -> unI64 93808- -- MEDEFAIDRIN CAPITAL LETTER C- '\x16e51'# -> unI64 93809- -- MEDEFAIDRIN CAPITAL LETTER U- '\x16e52'# -> unI64 93810- -- MEDEFAIDRIN CAPITAL LETTER YU- '\x16e53'# -> unI64 93811- -- MEDEFAIDRIN CAPITAL LETTER L- '\x16e54'# -> unI64 93812- -- MEDEFAIDRIN CAPITAL LETTER Q- '\x16e55'# -> unI64 93813- -- MEDEFAIDRIN CAPITAL LETTER HP- '\x16e56'# -> unI64 93814- -- MEDEFAIDRIN CAPITAL LETTER NY- '\x16e57'# -> unI64 93815- -- MEDEFAIDRIN CAPITAL LETTER X- '\x16e58'# -> unI64 93816- -- MEDEFAIDRIN CAPITAL LETTER D- '\x16e59'# -> unI64 93817- -- MEDEFAIDRIN CAPITAL LETTER OE- '\x16e5a'# -> unI64 93818- -- MEDEFAIDRIN CAPITAL LETTER N- '\x16e5b'# -> unI64 93819- -- MEDEFAIDRIN CAPITAL LETTER R- '\x16e5c'# -> unI64 93820- -- MEDEFAIDRIN CAPITAL LETTER O- '\x16e5d'# -> unI64 93821- -- MEDEFAIDRIN CAPITAL LETTER AI- '\x16e5e'# -> unI64 93822- -- MEDEFAIDRIN CAPITAL LETTER Y- '\x16e5f'# -> unI64 93823- -- ADLAM CAPITAL LETTER ALIF- '\x1e900'# -> unI64 125218- -- ADLAM CAPITAL LETTER DAALI- '\x1e901'# -> unI64 125219- -- ADLAM CAPITAL LETTER LAAM- '\x1e902'# -> unI64 125220- -- ADLAM CAPITAL LETTER MIIM- '\x1e903'# -> unI64 125221- -- ADLAM CAPITAL LETTER BA- '\x1e904'# -> unI64 125222- -- ADLAM CAPITAL LETTER SINNYIIYHE- '\x1e905'# -> unI64 125223- -- ADLAM CAPITAL LETTER PE- '\x1e906'# -> unI64 125224- -- ADLAM CAPITAL LETTER BHE- '\x1e907'# -> unI64 125225- -- ADLAM CAPITAL LETTER RA- '\x1e908'# -> unI64 125226- -- ADLAM CAPITAL LETTER E- '\x1e909'# -> unI64 125227- -- ADLAM CAPITAL LETTER FA- '\x1e90a'# -> unI64 125228- -- ADLAM CAPITAL LETTER I- '\x1e90b'# -> unI64 125229- -- ADLAM CAPITAL LETTER O- '\x1e90c'# -> unI64 125230- -- ADLAM CAPITAL LETTER DHA- '\x1e90d'# -> unI64 125231- -- ADLAM CAPITAL LETTER YHE- '\x1e90e'# -> unI64 125232- -- ADLAM CAPITAL LETTER WAW- '\x1e90f'# -> unI64 125233- -- ADLAM CAPITAL LETTER NUN- '\x1e910'# -> unI64 125234- -- ADLAM CAPITAL LETTER KAF- '\x1e911'# -> unI64 125235- -- ADLAM CAPITAL LETTER YA- '\x1e912'# -> unI64 125236- -- ADLAM CAPITAL LETTER U- '\x1e913'# -> unI64 125237- -- ADLAM CAPITAL LETTER JIIM- '\x1e914'# -> unI64 125238- -- ADLAM CAPITAL LETTER CHI- '\x1e915'# -> unI64 125239- -- ADLAM CAPITAL LETTER HA- '\x1e916'# -> unI64 125240- -- ADLAM CAPITAL LETTER QAAF- '\x1e917'# -> unI64 125241- -- ADLAM CAPITAL LETTER GA- '\x1e918'# -> unI64 125242- -- ADLAM CAPITAL LETTER NYA- '\x1e919'# -> unI64 125243- -- ADLAM CAPITAL LETTER TU- '\x1e91a'# -> unI64 125244- -- ADLAM CAPITAL LETTER NHA- '\x1e91b'# -> unI64 125245- -- ADLAM CAPITAL LETTER VA- '\x1e91c'# -> unI64 125246- -- ADLAM CAPITAL LETTER KHA- '\x1e91d'# -> unI64 125247- -- ADLAM CAPITAL LETTER GBE- '\x1e91e'# -> unI64 125248- -- ADLAM CAPITAL LETTER ZAL- '\x1e91f'# -> unI64 125249- -- ADLAM CAPITAL LETTER KPO- '\x1e920'# -> unI64 125250- -- ADLAM CAPITAL LETTER SHA- '\x1e921'# -> unI64 125251- _ -> unI64 0+-- AUTOMATICALLY GENERATED - DO NOT EDIT +-- Generated by scripts/CaseMapping.hs +-- CaseFolding-17.0.0.txt +-- Date: 2025-07-30, 23:54:36 GMT +-- SpecialCasing-17.0.0.txt +-- Date: 2025-07-31, 22:11:55 GMT + +{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +module Data.Text.Internal.Fusion.CaseMapping where +import GHC.Int +import GHC.Exts +import Data.Version (Version, makeVersion) +unicodeVersion :: Version +unicodeVersion = makeVersion [17, 0, 0] +unI64 :: Int64 -> _ {- unboxed Int64 -} +unI64 (I64# n) = n + +upperMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE upperMapping #-} +upperMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 174063699 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 146800710 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 153092166 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 159383622 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 321057542111302 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 334251681644614 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 176160851 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 176160851 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2856322357 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2831156548 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2795504964 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2808087876 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2831156558 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2812282180 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1931484936 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1931484937 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1931484938 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1931484939 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1931484940 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1931484941 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1931484942 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1931484943 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1931484936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1931484937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1931484938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1931484939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1931484940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1931484941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1931484942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1931484943 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1931484968 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1931484969 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1931484970 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1931484971 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1931484972 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1931484973 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1931484974 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1931484975 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1931484968 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1931484969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1931484970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1931484971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1931484972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1931484973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1931484974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1931484975 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1931485032 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1931485033 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1931485034 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1931485035 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1931485036 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1931485037 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1931485038 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1931485039 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1931485032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1931485033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1931485034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1931485035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1931485036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1931485037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1931485038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1931485039 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1931477905 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1931477905 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1931477911 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1931477911 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1931477929 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1931477929 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1931485114 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1931477894 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1931485130 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1931477897 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1931485178 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1931477903 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4050602585752465 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4050602585752471 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4050602585752489 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c5'# -> unI64 452 + '\x01c6'# -> unI64 452 + '\x01c8'# -> unI64 455 + '\x01c9'# -> unI64 455 + '\x01cb'# -> unI64 458 + '\x01cc'# -> unI64 458 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f2'# -> unI64 497 + '\x01f3'# -> unI64 497 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x10d0'# -> unI64 7312 + '\x10d1'# -> unI64 7313 + '\x10d2'# -> unI64 7314 + '\x10d3'# -> unI64 7315 + '\x10d4'# -> unI64 7316 + '\x10d5'# -> unI64 7317 + '\x10d6'# -> unI64 7318 + '\x10d7'# -> unI64 7319 + '\x10d8'# -> unI64 7320 + '\x10d9'# -> unI64 7321 + '\x10da'# -> unI64 7322 + '\x10db'# -> unI64 7323 + '\x10dc'# -> unI64 7324 + '\x10dd'# -> unI64 7325 + '\x10de'# -> unI64 7326 + '\x10df'# -> unI64 7327 + '\x10e0'# -> unI64 7328 + '\x10e1'# -> unI64 7329 + '\x10e2'# -> unI64 7330 + '\x10e3'# -> unI64 7331 + '\x10e4'# -> unI64 7332 + '\x10e5'# -> unI64 7333 + '\x10e6'# -> unI64 7334 + '\x10e7'# -> unI64 7335 + '\x10e8'# -> unI64 7336 + '\x10e9'# -> unI64 7337 + '\x10ea'# -> unI64 7338 + '\x10eb'# -> unI64 7339 + '\x10ec'# -> unI64 7340 + '\x10ed'# -> unI64 7341 + '\x10ee'# -> unI64 7342 + '\x10ef'# -> unI64 7343 + '\x10f0'# -> unI64 7344 + '\x10f1'# -> unI64 7345 + '\x10f2'# -> unI64 7346 + '\x10f3'# -> unI64 7347 + '\x10f4'# -> unI64 7348 + '\x10f5'# -> unI64 7349 + '\x10f6'# -> unI64 7350 + '\x10f7'# -> unI64 7351 + '\x10f8'# -> unI64 7352 + '\x10f9'# -> unI64 7353 + '\x10fa'# -> unI64 7354 + '\x10fd'# -> unI64 7357 + '\x10fe'# -> unI64 7358 + '\x10ff'# -> unI64 7359 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +lowerMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE lowerMapping #-} +lowerMapping = \case + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 8064 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 8065 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 8066 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 8067 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 8068 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 8069 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 8070 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 8071 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 8080 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 8081 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 8082 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 8083 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 8084 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 8085 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 8086 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 8087 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 8096 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 8097 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 8098 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 8099 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 8100 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 8101 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 8102 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 8103 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 8115 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 8131 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 8179 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c89'# -> unI64 7306 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1e9e'# -> unI64 223 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7cb'# -> unI64 612 + '\xa7cc'# -> unI64 42957 + '\xa7ce'# -> unI64 42959 + '\xa7d0'# -> unI64 42961 + '\xa7d2'# -> unI64 42963 + '\xa7d4'# -> unI64 42965 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7da'# -> unI64 42971 + '\xa7dc'# -> unI64 411 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x10d50'# -> unI64 68976 + '\x10d51'# -> unI64 68977 + '\x10d52'# -> unI64 68978 + '\x10d53'# -> unI64 68979 + '\x10d54'# -> unI64 68980 + '\x10d55'# -> unI64 68981 + '\x10d56'# -> unI64 68982 + '\x10d57'# -> unI64 68983 + '\x10d58'# -> unI64 68984 + '\x10d59'# -> unI64 68985 + '\x10d5a'# -> unI64 68986 + '\x10d5b'# -> unI64 68987 + '\x10d5c'# -> unI64 68988 + '\x10d5d'# -> unI64 68989 + '\x10d5e'# -> unI64 68990 + '\x10d5f'# -> unI64 68991 + '\x10d60'# -> unI64 68992 + '\x10d61'# -> unI64 68993 + '\x10d62'# -> unI64 68994 + '\x10d63'# -> unI64 68995 + '\x10d64'# -> unI64 68996 + '\x10d65'# -> unI64 68997 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x16ea0'# -> unI64 93883 + '\x16ea1'# -> unI64 93884 + '\x16ea2'# -> unI64 93885 + '\x16ea3'# -> unI64 93886 + '\x16ea4'# -> unI64 93887 + '\x16ea5'# -> unI64 93888 + '\x16ea6'# -> unI64 93889 + '\x16ea7'# -> unI64 93890 + '\x16ea8'# -> unI64 93891 + '\x16ea9'# -> unI64 93892 + '\x16eaa'# -> unI64 93893 + '\x16eab'# -> unI64 93894 + '\x16eac'# -> unI64 93895 + '\x16ead'# -> unI64 93896 + '\x16eae'# -> unI64 93897 + '\x16eaf'# -> unI64 93898 + '\x16eb0'# -> unI64 93899 + '\x16eb1'# -> unI64 93900 + '\x16eb2'# -> unI64 93901 + '\x16eb3'# -> unI64 93902 + '\x16eb4'# -> unI64 93903 + '\x16eb5'# -> unI64 93904 + '\x16eb6'# -> unI64 93905 + '\x16eb7'# -> unI64 93906 + '\x16eb8'# -> unI64 93907 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 +titleMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE titleMapping #-} +titleMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172563 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909574 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201030 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492486 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575494 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108806 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269715 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269715 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985653 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819844 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168260 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751172 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819854 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945476 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 8072 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 8073 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 8074 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 8075 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 8076 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 8077 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 8078 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 8079 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 8088 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 8089 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 8090 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 8091 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 8092 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 8093 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 8094 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 8095 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 8104 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 8105 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 8106 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 8107 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 8108 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 8109 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 8110 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 8111 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 8124 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 8140 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 8188 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1755324346 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1755317126 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1755324362 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1755317129 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1755324410 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1755317135 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 3681166678819729 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 3681166678819735 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 3681166678819753 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019b'# -> unI64 42972 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c4'# -> unI64 453 + '\x01c6'# -> unI64 453 + '\x01c7'# -> unI64 456 + '\x01c9'# -> unI64 456 + '\x01ca'# -> unI64 459 + '\x01cc'# -> unI64 459 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f1'# -> unI64 498 + '\x01f3'# -> unI64 498 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0264'# -> unI64 42955 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1c8a'# -> unI64 7305 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7cd'# -> unI64 42956 + '\xa7cf'# -> unI64 42958 + '\xa7d1'# -> unI64 42960 + '\xa7d3'# -> unI64 42962 + '\xa7d5'# -> unI64 42964 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7db'# -> unI64 42970 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x10d70'# -> unI64 68944 + '\x10d71'# -> unI64 68945 + '\x10d72'# -> unI64 68946 + '\x10d73'# -> unI64 68947 + '\x10d74'# -> unI64 68948 + '\x10d75'# -> unI64 68949 + '\x10d76'# -> unI64 68950 + '\x10d77'# -> unI64 68951 + '\x10d78'# -> unI64 68952 + '\x10d79'# -> unI64 68953 + '\x10d7a'# -> unI64 68954 + '\x10d7b'# -> unI64 68955 + '\x10d7c'# -> unI64 68956 + '\x10d7d'# -> unI64 68957 + '\x10d7e'# -> unI64 68958 + '\x10d7f'# -> unI64 68959 + '\x10d80'# -> unI64 68960 + '\x10d81'# -> unI64 68961 + '\x10d82'# -> unI64 68962 + '\x10d83'# -> unI64 68963 + '\x10d84'# -> unI64 68964 + '\x10d85'# -> unI64 68965 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x16ebb'# -> unI64 93856 + '\x16ebc'# -> unI64 93857 + '\x16ebd'# -> unI64 93858 + '\x16ebe'# -> unI64 93859 + '\x16ebf'# -> unI64 93860 + '\x16ec0'# -> unI64 93861 + '\x16ec1'# -> unI64 93862 + '\x16ec2'# -> unI64 93863 + '\x16ec3'# -> unI64 93864 + '\x16ec4'# -> unI64 93865 + '\x16ec5'# -> unI64 93866 + '\x16ec6'# -> unI64 93867 + '\x16ec7'# -> unI64 93868 + '\x16ec8'# -> unI64 93869 + '\x16ec9'# -> unI64 93870 + '\x16eca'# -> unI64 93871 + '\x16ecb'# -> unI64 93872 + '\x16ecc'# -> unI64 93873 + '\x16ecd'# -> unI64 93874 + '\x16ece'# -> unI64 93875 + '\x16ecf'# -> unI64 93876 + '\x16ed0'# -> unI64 93877 + '\x16ed1'# -> unI64 93878 + '\x16ed2'# -> unI64 93879 + '\x16ed3'# -> unI64 93880 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +foldMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE foldMapping #-} +foldMapping = \case + -- LATIN CAPITAL LETTER A + '\x0041'# -> unI64 97 + -- LATIN CAPITAL LETTER B + '\x0042'# -> unI64 98 + -- LATIN CAPITAL LETTER C + '\x0043'# -> unI64 99 + -- LATIN CAPITAL LETTER D + '\x0044'# -> unI64 100 + -- LATIN CAPITAL LETTER E + '\x0045'# -> unI64 101 + -- LATIN CAPITAL LETTER F + '\x0046'# -> unI64 102 + -- LATIN CAPITAL LETTER G + '\x0047'# -> unI64 103 + -- LATIN CAPITAL LETTER H + '\x0048'# -> unI64 104 + -- LATIN CAPITAL LETTER I + '\x0049'# -> unI64 105 + -- LATIN CAPITAL LETTER J + '\x004a'# -> unI64 106 + -- LATIN CAPITAL LETTER K + '\x004b'# -> unI64 107 + -- LATIN CAPITAL LETTER L + '\x004c'# -> unI64 108 + -- LATIN CAPITAL LETTER M + '\x004d'# -> unI64 109 + -- LATIN CAPITAL LETTER N + '\x004e'# -> unI64 110 + -- LATIN CAPITAL LETTER O + '\x004f'# -> unI64 111 + -- LATIN CAPITAL LETTER P + '\x0050'# -> unI64 112 + -- LATIN CAPITAL LETTER Q + '\x0051'# -> unI64 113 + -- LATIN CAPITAL LETTER R + '\x0052'# -> unI64 114 + -- LATIN CAPITAL LETTER S + '\x0053'# -> unI64 115 + -- LATIN CAPITAL LETTER T + '\x0054'# -> unI64 116 + -- LATIN CAPITAL LETTER U + '\x0055'# -> unI64 117 + -- LATIN CAPITAL LETTER V + '\x0056'# -> unI64 118 + -- LATIN CAPITAL LETTER W + '\x0057'# -> unI64 119 + -- LATIN CAPITAL LETTER X + '\x0058'# -> unI64 120 + -- LATIN CAPITAL LETTER Y + '\x0059'# -> unI64 121 + -- LATIN CAPITAL LETTER Z + '\x005a'# -> unI64 122 + -- MICRO SIGN + '\x00b5'# -> unI64 956 + -- LATIN CAPITAL LETTER A WITH GRAVE + '\x00c0'# -> unI64 224 + -- LATIN CAPITAL LETTER A WITH ACUTE + '\x00c1'# -> unI64 225 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX + '\x00c2'# -> unI64 226 + -- LATIN CAPITAL LETTER A WITH TILDE + '\x00c3'# -> unI64 227 + -- LATIN CAPITAL LETTER A WITH DIAERESIS + '\x00c4'# -> unI64 228 + -- LATIN CAPITAL LETTER A WITH RING ABOVE + '\x00c5'# -> unI64 229 + -- LATIN CAPITAL LETTER AE + '\x00c6'# -> unI64 230 + -- LATIN CAPITAL LETTER C WITH CEDILLA + '\x00c7'# -> unI64 231 + -- LATIN CAPITAL LETTER E WITH GRAVE + '\x00c8'# -> unI64 232 + -- LATIN CAPITAL LETTER E WITH ACUTE + '\x00c9'# -> unI64 233 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX + '\x00ca'# -> unI64 234 + -- LATIN CAPITAL LETTER E WITH DIAERESIS + '\x00cb'# -> unI64 235 + -- LATIN CAPITAL LETTER I WITH GRAVE + '\x00cc'# -> unI64 236 + -- LATIN CAPITAL LETTER I WITH ACUTE + '\x00cd'# -> unI64 237 + -- LATIN CAPITAL LETTER I WITH CIRCUMFLEX + '\x00ce'# -> unI64 238 + -- LATIN CAPITAL LETTER I WITH DIAERESIS + '\x00cf'# -> unI64 239 + -- LATIN CAPITAL LETTER ETH + '\x00d0'# -> unI64 240 + -- LATIN CAPITAL LETTER N WITH TILDE + '\x00d1'# -> unI64 241 + -- LATIN CAPITAL LETTER O WITH GRAVE + '\x00d2'# -> unI64 242 + -- LATIN CAPITAL LETTER O WITH ACUTE + '\x00d3'# -> unI64 243 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX + '\x00d4'# -> unI64 244 + -- LATIN CAPITAL LETTER O WITH TILDE + '\x00d5'# -> unI64 245 + -- LATIN CAPITAL LETTER O WITH DIAERESIS + '\x00d6'# -> unI64 246 + -- LATIN CAPITAL LETTER O WITH STROKE + '\x00d8'# -> unI64 248 + -- LATIN CAPITAL LETTER U WITH GRAVE + '\x00d9'# -> unI64 249 + -- LATIN CAPITAL LETTER U WITH ACUTE + '\x00da'# -> unI64 250 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX + '\x00db'# -> unI64 251 + -- LATIN CAPITAL LETTER U WITH DIAERESIS + '\x00dc'# -> unI64 252 + -- LATIN CAPITAL LETTER Y WITH ACUTE + '\x00dd'# -> unI64 253 + -- LATIN CAPITAL LETTER THORN + '\x00de'# -> unI64 254 + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH MACRON + '\x0100'# -> unI64 257 + -- LATIN CAPITAL LETTER A WITH BREVE + '\x0102'# -> unI64 259 + -- LATIN CAPITAL LETTER A WITH OGONEK + '\x0104'# -> unI64 261 + -- LATIN CAPITAL LETTER C WITH ACUTE + '\x0106'# -> unI64 263 + -- LATIN CAPITAL LETTER C WITH CIRCUMFLEX + '\x0108'# -> unI64 265 + -- LATIN CAPITAL LETTER C WITH DOT ABOVE + '\x010a'# -> unI64 267 + -- LATIN CAPITAL LETTER C WITH CARON + '\x010c'# -> unI64 269 + -- LATIN CAPITAL LETTER D WITH CARON + '\x010e'# -> unI64 271 + -- LATIN CAPITAL LETTER D WITH STROKE + '\x0110'# -> unI64 273 + -- LATIN CAPITAL LETTER E WITH MACRON + '\x0112'# -> unI64 275 + -- LATIN CAPITAL LETTER E WITH BREVE + '\x0114'# -> unI64 277 + -- LATIN CAPITAL LETTER E WITH DOT ABOVE + '\x0116'# -> unI64 279 + -- LATIN CAPITAL LETTER E WITH OGONEK + '\x0118'# -> unI64 281 + -- LATIN CAPITAL LETTER E WITH CARON + '\x011a'# -> unI64 283 + -- LATIN CAPITAL LETTER G WITH CIRCUMFLEX + '\x011c'# -> unI64 285 + -- LATIN CAPITAL LETTER G WITH BREVE + '\x011e'# -> unI64 287 + -- LATIN CAPITAL LETTER G WITH DOT ABOVE + '\x0120'# -> unI64 289 + -- LATIN CAPITAL LETTER G WITH CEDILLA + '\x0122'# -> unI64 291 + -- LATIN CAPITAL LETTER H WITH CIRCUMFLEX + '\x0124'# -> unI64 293 + -- LATIN CAPITAL LETTER H WITH STROKE + '\x0126'# -> unI64 295 + -- LATIN CAPITAL LETTER I WITH TILDE + '\x0128'# -> unI64 297 + -- LATIN CAPITAL LETTER I WITH MACRON + '\x012a'# -> unI64 299 + -- LATIN CAPITAL LETTER I WITH BREVE + '\x012c'# -> unI64 301 + -- LATIN CAPITAL LETTER I WITH OGONEK + '\x012e'# -> unI64 303 + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- LATIN CAPITAL LIGATURE IJ + '\x0132'# -> unI64 307 + -- LATIN CAPITAL LETTER J WITH CIRCUMFLEX + '\x0134'# -> unI64 309 + -- LATIN CAPITAL LETTER K WITH CEDILLA + '\x0136'# -> unI64 311 + -- LATIN CAPITAL LETTER L WITH ACUTE + '\x0139'# -> unI64 314 + -- LATIN CAPITAL LETTER L WITH CEDILLA + '\x013b'# -> unI64 316 + -- LATIN CAPITAL LETTER L WITH CARON + '\x013d'# -> unI64 318 + -- LATIN CAPITAL LETTER L WITH MIDDLE DOT + '\x013f'# -> unI64 320 + -- LATIN CAPITAL LETTER L WITH STROKE + '\x0141'# -> unI64 322 + -- LATIN CAPITAL LETTER N WITH ACUTE + '\x0143'# -> unI64 324 + -- LATIN CAPITAL LETTER N WITH CEDILLA + '\x0145'# -> unI64 326 + -- LATIN CAPITAL LETTER N WITH CARON + '\x0147'# -> unI64 328 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 230687420 + -- LATIN CAPITAL LETTER ENG + '\x014a'# -> unI64 331 + -- LATIN CAPITAL LETTER O WITH MACRON + '\x014c'# -> unI64 333 + -- LATIN CAPITAL LETTER O WITH BREVE + '\x014e'# -> unI64 335 + -- LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + '\x0150'# -> unI64 337 + -- LATIN CAPITAL LIGATURE OE + '\x0152'# -> unI64 339 + -- LATIN CAPITAL LETTER R WITH ACUTE + '\x0154'# -> unI64 341 + -- LATIN CAPITAL LETTER R WITH CEDILLA + '\x0156'# -> unI64 343 + -- LATIN CAPITAL LETTER R WITH CARON + '\x0158'# -> unI64 345 + -- LATIN CAPITAL LETTER S WITH ACUTE + '\x015a'# -> unI64 347 + -- LATIN CAPITAL LETTER S WITH CIRCUMFLEX + '\x015c'# -> unI64 349 + -- LATIN CAPITAL LETTER S WITH CEDILLA + '\x015e'# -> unI64 351 + -- LATIN CAPITAL LETTER S WITH CARON + '\x0160'# -> unI64 353 + -- LATIN CAPITAL LETTER T WITH CEDILLA + '\x0162'# -> unI64 355 + -- LATIN CAPITAL LETTER T WITH CARON + '\x0164'# -> unI64 357 + -- LATIN CAPITAL LETTER T WITH STROKE + '\x0166'# -> unI64 359 + -- LATIN CAPITAL LETTER U WITH TILDE + '\x0168'# -> unI64 361 + -- LATIN CAPITAL LETTER U WITH MACRON + '\x016a'# -> unI64 363 + -- LATIN CAPITAL LETTER U WITH BREVE + '\x016c'# -> unI64 365 + -- LATIN CAPITAL LETTER U WITH RING ABOVE + '\x016e'# -> unI64 367 + -- LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + '\x0170'# -> unI64 369 + -- LATIN CAPITAL LETTER U WITH OGONEK + '\x0172'# -> unI64 371 + -- LATIN CAPITAL LETTER W WITH CIRCUMFLEX + '\x0174'# -> unI64 373 + -- LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + '\x0176'# -> unI64 375 + -- LATIN CAPITAL LETTER Y WITH DIAERESIS + '\x0178'# -> unI64 255 + -- LATIN CAPITAL LETTER Z WITH ACUTE + '\x0179'# -> unI64 378 + -- LATIN CAPITAL LETTER Z WITH DOT ABOVE + '\x017b'# -> unI64 380 + -- LATIN CAPITAL LETTER Z WITH CARON + '\x017d'# -> unI64 382 + -- LATIN SMALL LETTER LONG S + '\x017f'# -> unI64 115 + -- LATIN CAPITAL LETTER B WITH HOOK + '\x0181'# -> unI64 595 + -- LATIN CAPITAL LETTER B WITH TOPBAR + '\x0182'# -> unI64 387 + -- LATIN CAPITAL LETTER TONE SIX + '\x0184'# -> unI64 389 + -- LATIN CAPITAL LETTER OPEN O + '\x0186'# -> unI64 596 + -- LATIN CAPITAL LETTER C WITH HOOK + '\x0187'# -> unI64 392 + -- LATIN CAPITAL LETTER AFRICAN D + '\x0189'# -> unI64 598 + -- LATIN CAPITAL LETTER D WITH HOOK + '\x018a'# -> unI64 599 + -- LATIN CAPITAL LETTER D WITH TOPBAR + '\x018b'# -> unI64 396 + -- LATIN CAPITAL LETTER REVERSED E + '\x018e'# -> unI64 477 + -- LATIN CAPITAL LETTER SCHWA + '\x018f'# -> unI64 601 + -- LATIN CAPITAL LETTER OPEN E + '\x0190'# -> unI64 603 + -- LATIN CAPITAL LETTER F WITH HOOK + '\x0191'# -> unI64 402 + -- LATIN CAPITAL LETTER G WITH HOOK + '\x0193'# -> unI64 608 + -- LATIN CAPITAL LETTER GAMMA + '\x0194'# -> unI64 611 + -- LATIN CAPITAL LETTER IOTA + '\x0196'# -> unI64 617 + -- LATIN CAPITAL LETTER I WITH STROKE + '\x0197'# -> unI64 616 + -- LATIN CAPITAL LETTER K WITH HOOK + '\x0198'# -> unI64 409 + -- LATIN CAPITAL LETTER TURNED M + '\x019c'# -> unI64 623 + -- LATIN CAPITAL LETTER N WITH LEFT HOOK + '\x019d'# -> unI64 626 + -- LATIN CAPITAL LETTER O WITH MIDDLE TILDE + '\x019f'# -> unI64 629 + -- LATIN CAPITAL LETTER O WITH HORN + '\x01a0'# -> unI64 417 + -- LATIN CAPITAL LETTER OI + '\x01a2'# -> unI64 419 + -- LATIN CAPITAL LETTER P WITH HOOK + '\x01a4'# -> unI64 421 + -- LATIN LETTER YR + '\x01a6'# -> unI64 640 + -- LATIN CAPITAL LETTER TONE TWO + '\x01a7'# -> unI64 424 + -- LATIN CAPITAL LETTER ESH + '\x01a9'# -> unI64 643 + -- LATIN CAPITAL LETTER T WITH HOOK + '\x01ac'# -> unI64 429 + -- LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + '\x01ae'# -> unI64 648 + -- LATIN CAPITAL LETTER U WITH HORN + '\x01af'# -> unI64 432 + -- LATIN CAPITAL LETTER UPSILON + '\x01b1'# -> unI64 650 + -- LATIN CAPITAL LETTER V WITH HOOK + '\x01b2'# -> unI64 651 + -- LATIN CAPITAL LETTER Y WITH HOOK + '\x01b3'# -> unI64 436 + -- LATIN CAPITAL LETTER Z WITH STROKE + '\x01b5'# -> unI64 438 + -- LATIN CAPITAL LETTER EZH + '\x01b7'# -> unI64 658 + -- LATIN CAPITAL LETTER EZH REVERSED + '\x01b8'# -> unI64 441 + -- LATIN CAPITAL LETTER TONE FIVE + '\x01bc'# -> unI64 445 + -- LATIN CAPITAL LETTER DZ WITH CARON + '\x01c4'# -> unI64 454 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + '\x01c5'# -> unI64 454 + -- LATIN CAPITAL LETTER LJ + '\x01c7'# -> unI64 457 + -- LATIN CAPITAL LETTER L WITH SMALL LETTER J + '\x01c8'# -> unI64 457 + -- LATIN CAPITAL LETTER NJ + '\x01ca'# -> unI64 460 + -- LATIN CAPITAL LETTER N WITH SMALL LETTER J + '\x01cb'# -> unI64 460 + -- LATIN CAPITAL LETTER A WITH CARON + '\x01cd'# -> unI64 462 + -- LATIN CAPITAL LETTER I WITH CARON + '\x01cf'# -> unI64 464 + -- LATIN CAPITAL LETTER O WITH CARON + '\x01d1'# -> unI64 466 + -- LATIN CAPITAL LETTER U WITH CARON + '\x01d3'# -> unI64 468 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + '\x01d5'# -> unI64 470 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + '\x01d7'# -> unI64 472 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + '\x01d9'# -> unI64 474 + -- LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + '\x01db'# -> unI64 476 + -- LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + '\x01de'# -> unI64 479 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + '\x01e0'# -> unI64 481 + -- LATIN CAPITAL LETTER AE WITH MACRON + '\x01e2'# -> unI64 483 + -- LATIN CAPITAL LETTER G WITH STROKE + '\x01e4'# -> unI64 485 + -- LATIN CAPITAL LETTER G WITH CARON + '\x01e6'# -> unI64 487 + -- LATIN CAPITAL LETTER K WITH CARON + '\x01e8'# -> unI64 489 + -- LATIN CAPITAL LETTER O WITH OGONEK + '\x01ea'# -> unI64 491 + -- LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + '\x01ec'# -> unI64 493 + -- LATIN CAPITAL LETTER EZH WITH CARON + '\x01ee'# -> unI64 495 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778666 + -- LATIN CAPITAL LETTER DZ + '\x01f1'# -> unI64 499 + -- LATIN CAPITAL LETTER D WITH SMALL LETTER Z + '\x01f2'# -> unI64 499 + -- LATIN CAPITAL LETTER G WITH ACUTE + '\x01f4'# -> unI64 501 + -- LATIN CAPITAL LETTER HWAIR + '\x01f6'# -> unI64 405 + -- LATIN CAPITAL LETTER WYNN + '\x01f7'# -> unI64 447 + -- LATIN CAPITAL LETTER N WITH GRAVE + '\x01f8'# -> unI64 505 + -- LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + '\x01fa'# -> unI64 507 + -- LATIN CAPITAL LETTER AE WITH ACUTE + '\x01fc'# -> unI64 509 + -- LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + '\x01fe'# -> unI64 511 + -- LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + '\x0200'# -> unI64 513 + -- LATIN CAPITAL LETTER A WITH INVERTED BREVE + '\x0202'# -> unI64 515 + -- LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + '\x0204'# -> unI64 517 + -- LATIN CAPITAL LETTER E WITH INVERTED BREVE + '\x0206'# -> unI64 519 + -- LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + '\x0208'# -> unI64 521 + -- LATIN CAPITAL LETTER I WITH INVERTED BREVE + '\x020a'# -> unI64 523 + -- LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + '\x020c'# -> unI64 525 + -- LATIN CAPITAL LETTER O WITH INVERTED BREVE + '\x020e'# -> unI64 527 + -- LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + '\x0210'# -> unI64 529 + -- LATIN CAPITAL LETTER R WITH INVERTED BREVE + '\x0212'# -> unI64 531 + -- LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + '\x0214'# -> unI64 533 + -- LATIN CAPITAL LETTER U WITH INVERTED BREVE + '\x0216'# -> unI64 535 + -- LATIN CAPITAL LETTER S WITH COMMA BELOW + '\x0218'# -> unI64 537 + -- LATIN CAPITAL LETTER T WITH COMMA BELOW + '\x021a'# -> unI64 539 + -- LATIN CAPITAL LETTER YOGH + '\x021c'# -> unI64 541 + -- LATIN CAPITAL LETTER H WITH CARON + '\x021e'# -> unI64 543 + -- LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + '\x0220'# -> unI64 414 + -- LATIN CAPITAL LETTER OU + '\x0222'# -> unI64 547 + -- LATIN CAPITAL LETTER Z WITH HOOK + '\x0224'# -> unI64 549 + -- LATIN CAPITAL LETTER A WITH DOT ABOVE + '\x0226'# -> unI64 551 + -- LATIN CAPITAL LETTER E WITH CEDILLA + '\x0228'# -> unI64 553 + -- LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + '\x022a'# -> unI64 555 + -- LATIN CAPITAL LETTER O WITH TILDE AND MACRON + '\x022c'# -> unI64 557 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE + '\x022e'# -> unI64 559 + -- LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + '\x0230'# -> unI64 561 + -- LATIN CAPITAL LETTER Y WITH MACRON + '\x0232'# -> unI64 563 + -- LATIN CAPITAL LETTER A WITH STROKE + '\x023a'# -> unI64 11365 + -- LATIN CAPITAL LETTER C WITH STROKE + '\x023b'# -> unI64 572 + -- LATIN CAPITAL LETTER L WITH BAR + '\x023d'# -> unI64 410 + -- LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + '\x023e'# -> unI64 11366 + -- LATIN CAPITAL LETTER GLOTTAL STOP + '\x0241'# -> unI64 578 + -- LATIN CAPITAL LETTER B WITH STROKE + '\x0243'# -> unI64 384 + -- LATIN CAPITAL LETTER U BAR + '\x0244'# -> unI64 649 + -- LATIN CAPITAL LETTER TURNED V + '\x0245'# -> unI64 652 + -- LATIN CAPITAL LETTER E WITH STROKE + '\x0246'# -> unI64 583 + -- LATIN CAPITAL LETTER J WITH STROKE + '\x0248'# -> unI64 585 + -- LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL + '\x024a'# -> unI64 587 + -- LATIN CAPITAL LETTER R WITH STROKE + '\x024c'# -> unI64 589 + -- LATIN CAPITAL LETTER Y WITH STROKE + '\x024e'# -> unI64 591 + -- COMBINING GREEK YPOGEGRAMMENI + '\x0345'# -> unI64 953 + -- GREEK CAPITAL LETTER HETA + '\x0370'# -> unI64 881 + -- GREEK CAPITAL LETTER ARCHAIC SAMPI + '\x0372'# -> unI64 883 + -- GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA + '\x0376'# -> unI64 887 + -- GREEK CAPITAL LETTER YOT + '\x037f'# -> unI64 1011 + -- GREEK CAPITAL LETTER ALPHA WITH TONOS + '\x0386'# -> unI64 940 + -- GREEK CAPITAL LETTER EPSILON WITH TONOS + '\x0388'# -> unI64 941 + -- GREEK CAPITAL LETTER ETA WITH TONOS + '\x0389'# -> unI64 942 + -- GREEK CAPITAL LETTER IOTA WITH TONOS + '\x038a'# -> unI64 943 + -- GREEK CAPITAL LETTER OMICRON WITH TONOS + '\x038c'# -> unI64 972 + -- GREEK CAPITAL LETTER UPSILON WITH TONOS + '\x038e'# -> unI64 973 + -- GREEK CAPITAL LETTER OMEGA WITH TONOS + '\x038f'# -> unI64 974 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429881 + -- GREEK CAPITAL LETTER ALPHA + '\x0391'# -> unI64 945 + -- GREEK CAPITAL LETTER BETA + '\x0392'# -> unI64 946 + -- GREEK CAPITAL LETTER GAMMA + '\x0393'# -> unI64 947 + -- GREEK CAPITAL LETTER DELTA + '\x0394'# -> unI64 948 + -- GREEK CAPITAL LETTER EPSILON + '\x0395'# -> unI64 949 + -- GREEK CAPITAL LETTER ZETA + '\x0396'# -> unI64 950 + -- GREEK CAPITAL LETTER ETA + '\x0397'# -> unI64 951 + -- GREEK CAPITAL LETTER THETA + '\x0398'# -> unI64 952 + -- GREEK CAPITAL LETTER IOTA + '\x0399'# -> unI64 953 + -- GREEK CAPITAL LETTER KAPPA + '\x039a'# -> unI64 954 + -- GREEK CAPITAL LETTER LAMDA + '\x039b'# -> unI64 955 + -- GREEK CAPITAL LETTER MU + '\x039c'# -> unI64 956 + -- GREEK CAPITAL LETTER NU + '\x039d'# -> unI64 957 + -- GREEK CAPITAL LETTER XI + '\x039e'# -> unI64 958 + -- GREEK CAPITAL LETTER OMICRON + '\x039f'# -> unI64 959 + -- GREEK CAPITAL LETTER PI + '\x03a0'# -> unI64 960 + -- GREEK CAPITAL LETTER RHO + '\x03a1'# -> unI64 961 + -- GREEK CAPITAL LETTER SIGMA + '\x03a3'# -> unI64 963 + -- GREEK CAPITAL LETTER TAU + '\x03a4'# -> unI64 964 + -- GREEK CAPITAL LETTER UPSILON + '\x03a5'# -> unI64 965 + -- GREEK CAPITAL LETTER PHI + '\x03a6'# -> unI64 966 + -- GREEK CAPITAL LETTER CHI + '\x03a7'# -> unI64 967 + -- GREEK CAPITAL LETTER PSI + '\x03a8'# -> unI64 968 + -- GREEK CAPITAL LETTER OMEGA + '\x03a9'# -> unI64 969 + -- GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + '\x03aa'# -> unI64 970 + -- GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + '\x03ab'# -> unI64 971 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER FINAL SIGMA + '\x03c2'# -> unI64 963 + -- GREEK CAPITAL KAI SYMBOL + '\x03cf'# -> unI64 983 + -- GREEK BETA SYMBOL + '\x03d0'# -> unI64 946 + -- GREEK THETA SYMBOL + '\x03d1'# -> unI64 952 + -- GREEK PHI SYMBOL + '\x03d5'# -> unI64 966 + -- GREEK PI SYMBOL + '\x03d6'# -> unI64 960 + -- GREEK LETTER ARCHAIC KOPPA + '\x03d8'# -> unI64 985 + -- GREEK LETTER STIGMA + '\x03da'# -> unI64 987 + -- GREEK LETTER DIGAMMA + '\x03dc'# -> unI64 989 + -- GREEK LETTER KOPPA + '\x03de'# -> unI64 991 + -- GREEK LETTER SAMPI + '\x03e0'# -> unI64 993 + -- COPTIC CAPITAL LETTER SHEI + '\x03e2'# -> unI64 995 + -- COPTIC CAPITAL LETTER FEI + '\x03e4'# -> unI64 997 + -- COPTIC CAPITAL LETTER KHEI + '\x03e6'# -> unI64 999 + -- COPTIC CAPITAL LETTER HORI + '\x03e8'# -> unI64 1001 + -- COPTIC CAPITAL LETTER GANGIA + '\x03ea'# -> unI64 1003 + -- COPTIC CAPITAL LETTER SHIMA + '\x03ec'# -> unI64 1005 + -- COPTIC CAPITAL LETTER DEI + '\x03ee'# -> unI64 1007 + -- GREEK KAPPA SYMBOL + '\x03f0'# -> unI64 954 + -- GREEK RHO SYMBOL + '\x03f1'# -> unI64 961 + -- GREEK CAPITAL THETA SYMBOL + '\x03f4'# -> unI64 952 + -- GREEK LUNATE EPSILON SYMBOL + '\x03f5'# -> unI64 949 + -- GREEK CAPITAL LETTER SHO + '\x03f7'# -> unI64 1016 + -- GREEK CAPITAL LUNATE SIGMA SYMBOL + '\x03f9'# -> unI64 1010 + -- GREEK CAPITAL LETTER SAN + '\x03fa'# -> unI64 1019 + -- GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL + '\x03fd'# -> unI64 891 + -- GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL + '\x03fe'# -> unI64 892 + -- GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + '\x03ff'# -> unI64 893 + -- CYRILLIC CAPITAL LETTER IE WITH GRAVE + '\x0400'# -> unI64 1104 + -- CYRILLIC CAPITAL LETTER IO + '\x0401'# -> unI64 1105 + -- CYRILLIC CAPITAL LETTER DJE + '\x0402'# -> unI64 1106 + -- CYRILLIC CAPITAL LETTER GJE + '\x0403'# -> unI64 1107 + -- CYRILLIC CAPITAL LETTER UKRAINIAN IE + '\x0404'# -> unI64 1108 + -- CYRILLIC CAPITAL LETTER DZE + '\x0405'# -> unI64 1109 + -- CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + '\x0406'# -> unI64 1110 + -- CYRILLIC CAPITAL LETTER YI + '\x0407'# -> unI64 1111 + -- CYRILLIC CAPITAL LETTER JE + '\x0408'# -> unI64 1112 + -- CYRILLIC CAPITAL LETTER LJE + '\x0409'# -> unI64 1113 + -- CYRILLIC CAPITAL LETTER NJE + '\x040a'# -> unI64 1114 + -- CYRILLIC CAPITAL LETTER TSHE + '\x040b'# -> unI64 1115 + -- CYRILLIC CAPITAL LETTER KJE + '\x040c'# -> unI64 1116 + -- CYRILLIC CAPITAL LETTER I WITH GRAVE + '\x040d'# -> unI64 1117 + -- CYRILLIC CAPITAL LETTER SHORT U + '\x040e'# -> unI64 1118 + -- CYRILLIC CAPITAL LETTER DZHE + '\x040f'# -> unI64 1119 + -- CYRILLIC CAPITAL LETTER A + '\x0410'# -> unI64 1072 + -- CYRILLIC CAPITAL LETTER BE + '\x0411'# -> unI64 1073 + -- CYRILLIC CAPITAL LETTER VE + '\x0412'# -> unI64 1074 + -- CYRILLIC CAPITAL LETTER GHE + '\x0413'# -> unI64 1075 + -- CYRILLIC CAPITAL LETTER DE + '\x0414'# -> unI64 1076 + -- CYRILLIC CAPITAL LETTER IE + '\x0415'# -> unI64 1077 + -- CYRILLIC CAPITAL LETTER ZHE + '\x0416'# -> unI64 1078 + -- CYRILLIC CAPITAL LETTER ZE + '\x0417'# -> unI64 1079 + -- CYRILLIC CAPITAL LETTER I + '\x0418'# -> unI64 1080 + -- CYRILLIC CAPITAL LETTER SHORT I + '\x0419'# -> unI64 1081 + -- CYRILLIC CAPITAL LETTER KA + '\x041a'# -> unI64 1082 + -- CYRILLIC CAPITAL LETTER EL + '\x041b'# -> unI64 1083 + -- CYRILLIC CAPITAL LETTER EM + '\x041c'# -> unI64 1084 + -- CYRILLIC CAPITAL LETTER EN + '\x041d'# -> unI64 1085 + -- CYRILLIC CAPITAL LETTER O + '\x041e'# -> unI64 1086 + -- CYRILLIC CAPITAL LETTER PE + '\x041f'# -> unI64 1087 + -- CYRILLIC CAPITAL LETTER ER + '\x0420'# -> unI64 1088 + -- CYRILLIC CAPITAL LETTER ES + '\x0421'# -> unI64 1089 + -- CYRILLIC CAPITAL LETTER TE + '\x0422'# -> unI64 1090 + -- CYRILLIC CAPITAL LETTER U + '\x0423'# -> unI64 1091 + -- CYRILLIC CAPITAL LETTER EF + '\x0424'# -> unI64 1092 + -- CYRILLIC CAPITAL LETTER HA + '\x0425'# -> unI64 1093 + -- CYRILLIC CAPITAL LETTER TSE + '\x0426'# -> unI64 1094 + -- CYRILLIC CAPITAL LETTER CHE + '\x0427'# -> unI64 1095 + -- CYRILLIC CAPITAL LETTER SHA + '\x0428'# -> unI64 1096 + -- CYRILLIC CAPITAL LETTER SHCHA + '\x0429'# -> unI64 1097 + -- CYRILLIC CAPITAL LETTER HARD SIGN + '\x042a'# -> unI64 1098 + -- CYRILLIC CAPITAL LETTER YERU + '\x042b'# -> unI64 1099 + -- CYRILLIC CAPITAL LETTER SOFT SIGN + '\x042c'# -> unI64 1100 + -- CYRILLIC CAPITAL LETTER E + '\x042d'# -> unI64 1101 + -- CYRILLIC CAPITAL LETTER YU + '\x042e'# -> unI64 1102 + -- CYRILLIC CAPITAL LETTER YA + '\x042f'# -> unI64 1103 + -- CYRILLIC CAPITAL LETTER OMEGA + '\x0460'# -> unI64 1121 + -- CYRILLIC CAPITAL LETTER YAT + '\x0462'# -> unI64 1123 + -- CYRILLIC CAPITAL LETTER IOTIFIED E + '\x0464'# -> unI64 1125 + -- CYRILLIC CAPITAL LETTER LITTLE YUS + '\x0466'# -> unI64 1127 + -- CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS + '\x0468'# -> unI64 1129 + -- CYRILLIC CAPITAL LETTER BIG YUS + '\x046a'# -> unI64 1131 + -- CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS + '\x046c'# -> unI64 1133 + -- CYRILLIC CAPITAL LETTER KSI + '\x046e'# -> unI64 1135 + -- CYRILLIC CAPITAL LETTER PSI + '\x0470'# -> unI64 1137 + -- CYRILLIC CAPITAL LETTER FITA + '\x0472'# -> unI64 1139 + -- CYRILLIC CAPITAL LETTER IZHITSA + '\x0474'# -> unI64 1141 + -- CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + '\x0476'# -> unI64 1143 + -- CYRILLIC CAPITAL LETTER UK + '\x0478'# -> unI64 1145 + -- CYRILLIC CAPITAL LETTER ROUND OMEGA + '\x047a'# -> unI64 1147 + -- CYRILLIC CAPITAL LETTER OMEGA WITH TITLO + '\x047c'# -> unI64 1149 + -- CYRILLIC CAPITAL LETTER OT + '\x047e'# -> unI64 1151 + -- CYRILLIC CAPITAL LETTER KOPPA + '\x0480'# -> unI64 1153 + -- CYRILLIC CAPITAL LETTER SHORT I WITH TAIL + '\x048a'# -> unI64 1163 + -- CYRILLIC CAPITAL LETTER SEMISOFT SIGN + '\x048c'# -> unI64 1165 + -- CYRILLIC CAPITAL LETTER ER WITH TICK + '\x048e'# -> unI64 1167 + -- CYRILLIC CAPITAL LETTER GHE WITH UPTURN + '\x0490'# -> unI64 1169 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE + '\x0492'# -> unI64 1171 + -- CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK + '\x0494'# -> unI64 1173 + -- CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + '\x0496'# -> unI64 1175 + -- CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + '\x0498'# -> unI64 1177 + -- CYRILLIC CAPITAL LETTER KA WITH DESCENDER + '\x049a'# -> unI64 1179 + -- CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + '\x049c'# -> unI64 1181 + -- CYRILLIC CAPITAL LETTER KA WITH STROKE + '\x049e'# -> unI64 1183 + -- CYRILLIC CAPITAL LETTER BASHKIR KA + '\x04a0'# -> unI64 1185 + -- CYRILLIC CAPITAL LETTER EN WITH DESCENDER + '\x04a2'# -> unI64 1187 + -- CYRILLIC CAPITAL LIGATURE EN GHE + '\x04a4'# -> unI64 1189 + -- CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK + '\x04a6'# -> unI64 1191 + -- CYRILLIC CAPITAL LETTER ABKHASIAN HA + '\x04a8'# -> unI64 1193 + -- CYRILLIC CAPITAL LETTER ES WITH DESCENDER + '\x04aa'# -> unI64 1195 + -- CYRILLIC CAPITAL LETTER TE WITH DESCENDER + '\x04ac'# -> unI64 1197 + -- CYRILLIC CAPITAL LETTER STRAIGHT U + '\x04ae'# -> unI64 1199 + -- CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + '\x04b0'# -> unI64 1201 + -- CYRILLIC CAPITAL LETTER HA WITH DESCENDER + '\x04b2'# -> unI64 1203 + -- CYRILLIC CAPITAL LIGATURE TE TSE + '\x04b4'# -> unI64 1205 + -- CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + '\x04b6'# -> unI64 1207 + -- CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + '\x04b8'# -> unI64 1209 + -- CYRILLIC CAPITAL LETTER SHHA + '\x04ba'# -> unI64 1211 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE + '\x04bc'# -> unI64 1213 + -- CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER + '\x04be'# -> unI64 1215 + -- CYRILLIC LETTER PALOCHKA + '\x04c0'# -> unI64 1231 + -- CYRILLIC CAPITAL LETTER ZHE WITH BREVE + '\x04c1'# -> unI64 1218 + -- CYRILLIC CAPITAL LETTER KA WITH HOOK + '\x04c3'# -> unI64 1220 + -- CYRILLIC CAPITAL LETTER EL WITH TAIL + '\x04c5'# -> unI64 1222 + -- CYRILLIC CAPITAL LETTER EN WITH HOOK + '\x04c7'# -> unI64 1224 + -- CYRILLIC CAPITAL LETTER EN WITH TAIL + '\x04c9'# -> unI64 1226 + -- CYRILLIC CAPITAL LETTER KHAKASSIAN CHE + '\x04cb'# -> unI64 1228 + -- CYRILLIC CAPITAL LETTER EM WITH TAIL + '\x04cd'# -> unI64 1230 + -- CYRILLIC CAPITAL LETTER A WITH BREVE + '\x04d0'# -> unI64 1233 + -- CYRILLIC CAPITAL LETTER A WITH DIAERESIS + '\x04d2'# -> unI64 1235 + -- CYRILLIC CAPITAL LIGATURE A IE + '\x04d4'# -> unI64 1237 + -- CYRILLIC CAPITAL LETTER IE WITH BREVE + '\x04d6'# -> unI64 1239 + -- CYRILLIC CAPITAL LETTER SCHWA + '\x04d8'# -> unI64 1241 + -- CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + '\x04da'# -> unI64 1243 + -- CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + '\x04dc'# -> unI64 1245 + -- CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + '\x04de'# -> unI64 1247 + -- CYRILLIC CAPITAL LETTER ABKHASIAN DZE + '\x04e0'# -> unI64 1249 + -- CYRILLIC CAPITAL LETTER I WITH MACRON + '\x04e2'# -> unI64 1251 + -- CYRILLIC CAPITAL LETTER I WITH DIAERESIS + '\x04e4'# -> unI64 1253 + -- CYRILLIC CAPITAL LETTER O WITH DIAERESIS + '\x04e6'# -> unI64 1255 + -- CYRILLIC CAPITAL LETTER BARRED O + '\x04e8'# -> unI64 1257 + -- CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + '\x04ea'# -> unI64 1259 + -- CYRILLIC CAPITAL LETTER E WITH DIAERESIS + '\x04ec'# -> unI64 1261 + -- CYRILLIC CAPITAL LETTER U WITH MACRON + '\x04ee'# -> unI64 1263 + -- CYRILLIC CAPITAL LETTER U WITH DIAERESIS + '\x04f0'# -> unI64 1265 + -- CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + '\x04f2'# -> unI64 1267 + -- CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + '\x04f4'# -> unI64 1269 + -- CYRILLIC CAPITAL LETTER GHE WITH DESCENDER + '\x04f6'# -> unI64 1271 + -- CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + '\x04f8'# -> unI64 1273 + -- CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK + '\x04fa'# -> unI64 1275 + -- CYRILLIC CAPITAL LETTER HA WITH HOOK + '\x04fc'# -> unI64 1277 + -- CYRILLIC CAPITAL LETTER HA WITH STROKE + '\x04fe'# -> unI64 1279 + -- CYRILLIC CAPITAL LETTER KOMI DE + '\x0500'# -> unI64 1281 + -- CYRILLIC CAPITAL LETTER KOMI DJE + '\x0502'# -> unI64 1283 + -- CYRILLIC CAPITAL LETTER KOMI ZJE + '\x0504'# -> unI64 1285 + -- CYRILLIC CAPITAL LETTER KOMI DZJE + '\x0506'# -> unI64 1287 + -- CYRILLIC CAPITAL LETTER KOMI LJE + '\x0508'# -> unI64 1289 + -- CYRILLIC CAPITAL LETTER KOMI NJE + '\x050a'# -> unI64 1291 + -- CYRILLIC CAPITAL LETTER KOMI SJE + '\x050c'# -> unI64 1293 + -- CYRILLIC CAPITAL LETTER KOMI TJE + '\x050e'# -> unI64 1295 + -- CYRILLIC CAPITAL LETTER REVERSED ZE + '\x0510'# -> unI64 1297 + -- CYRILLIC CAPITAL LETTER EL WITH HOOK + '\x0512'# -> unI64 1299 + -- CYRILLIC CAPITAL LETTER LHA + '\x0514'# -> unI64 1301 + -- CYRILLIC CAPITAL LETTER RHA + '\x0516'# -> unI64 1303 + -- CYRILLIC CAPITAL LETTER YAE + '\x0518'# -> unI64 1305 + -- CYRILLIC CAPITAL LETTER QA + '\x051a'# -> unI64 1307 + -- CYRILLIC CAPITAL LETTER WE + '\x051c'# -> unI64 1309 + -- CYRILLIC CAPITAL LETTER ALEUT KA + '\x051e'# -> unI64 1311 + -- CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK + '\x0520'# -> unI64 1313 + -- CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK + '\x0522'# -> unI64 1315 + -- CYRILLIC CAPITAL LETTER PE WITH DESCENDER + '\x0524'# -> unI64 1317 + -- CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER + '\x0526'# -> unI64 1319 + -- CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK + '\x0528'# -> unI64 1321 + -- CYRILLIC CAPITAL LETTER DZZHE + '\x052a'# -> unI64 1323 + -- CYRILLIC CAPITAL LETTER DCHE + '\x052c'# -> unI64 1325 + -- CYRILLIC CAPITAL LETTER EL WITH DESCENDER + '\x052e'# -> unI64 1327 + -- ARMENIAN CAPITAL LETTER AYB + '\x0531'# -> unI64 1377 + -- ARMENIAN CAPITAL LETTER BEN + '\x0532'# -> unI64 1378 + -- ARMENIAN CAPITAL LETTER GIM + '\x0533'# -> unI64 1379 + -- ARMENIAN CAPITAL LETTER DA + '\x0534'# -> unI64 1380 + -- ARMENIAN CAPITAL LETTER ECH + '\x0535'# -> unI64 1381 + -- ARMENIAN CAPITAL LETTER ZA + '\x0536'# -> unI64 1382 + -- ARMENIAN CAPITAL LETTER EH + '\x0537'# -> unI64 1383 + -- ARMENIAN CAPITAL LETTER ET + '\x0538'# -> unI64 1384 + -- ARMENIAN CAPITAL LETTER TO + '\x0539'# -> unI64 1385 + -- ARMENIAN CAPITAL LETTER ZHE + '\x053a'# -> unI64 1386 + -- ARMENIAN CAPITAL LETTER INI + '\x053b'# -> unI64 1387 + -- ARMENIAN CAPITAL LETTER LIWN + '\x053c'# -> unI64 1388 + -- ARMENIAN CAPITAL LETTER XEH + '\x053d'# -> unI64 1389 + -- ARMENIAN CAPITAL LETTER CA + '\x053e'# -> unI64 1390 + -- ARMENIAN CAPITAL LETTER KEN + '\x053f'# -> unI64 1391 + -- ARMENIAN CAPITAL LETTER HO + '\x0540'# -> unI64 1392 + -- ARMENIAN CAPITAL LETTER JA + '\x0541'# -> unI64 1393 + -- ARMENIAN CAPITAL LETTER GHAD + '\x0542'# -> unI64 1394 + -- ARMENIAN CAPITAL LETTER CHEH + '\x0543'# -> unI64 1395 + -- ARMENIAN CAPITAL LETTER MEN + '\x0544'# -> unI64 1396 + -- ARMENIAN CAPITAL LETTER YI + '\x0545'# -> unI64 1397 + -- ARMENIAN CAPITAL LETTER NOW + '\x0546'# -> unI64 1398 + -- ARMENIAN CAPITAL LETTER SHA + '\x0547'# -> unI64 1399 + -- ARMENIAN CAPITAL LETTER VO + '\x0548'# -> unI64 1400 + -- ARMENIAN CAPITAL LETTER CHA + '\x0549'# -> unI64 1401 + -- ARMENIAN CAPITAL LETTER PEH + '\x054a'# -> unI64 1402 + -- ARMENIAN CAPITAL LETTER JHEH + '\x054b'# -> unI64 1403 + -- ARMENIAN CAPITAL LETTER RA + '\x054c'# -> unI64 1404 + -- ARMENIAN CAPITAL LETTER SEH + '\x054d'# -> unI64 1405 + -- ARMENIAN CAPITAL LETTER VEW + '\x054e'# -> unI64 1406 + -- ARMENIAN CAPITAL LETTER TIWN + '\x054f'# -> unI64 1407 + -- ARMENIAN CAPITAL LETTER REH + '\x0550'# -> unI64 1408 + -- ARMENIAN CAPITAL LETTER CO + '\x0551'# -> unI64 1409 + -- ARMENIAN CAPITAL LETTER YIWN + '\x0552'# -> unI64 1410 + -- ARMENIAN CAPITAL LETTER PIWR + '\x0553'# -> unI64 1411 + -- ARMENIAN CAPITAL LETTER KEH + '\x0554'# -> unI64 1412 + -- ARMENIAN CAPITAL LETTER OH + '\x0555'# -> unI64 1413 + -- ARMENIAN CAPITAL LETTER FEH + '\x0556'# -> unI64 1414 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985701 + -- GEORGIAN CAPITAL LETTER AN + '\x10a0'# -> unI64 11520 + -- GEORGIAN CAPITAL LETTER BAN + '\x10a1'# -> unI64 11521 + -- GEORGIAN CAPITAL LETTER GAN + '\x10a2'# -> unI64 11522 + -- GEORGIAN CAPITAL LETTER DON + '\x10a3'# -> unI64 11523 + -- GEORGIAN CAPITAL LETTER EN + '\x10a4'# -> unI64 11524 + -- GEORGIAN CAPITAL LETTER VIN + '\x10a5'# -> unI64 11525 + -- GEORGIAN CAPITAL LETTER ZEN + '\x10a6'# -> unI64 11526 + -- GEORGIAN CAPITAL LETTER TAN + '\x10a7'# -> unI64 11527 + -- GEORGIAN CAPITAL LETTER IN + '\x10a8'# -> unI64 11528 + -- GEORGIAN CAPITAL LETTER KAN + '\x10a9'# -> unI64 11529 + -- GEORGIAN CAPITAL LETTER LAS + '\x10aa'# -> unI64 11530 + -- GEORGIAN CAPITAL LETTER MAN + '\x10ab'# -> unI64 11531 + -- GEORGIAN CAPITAL LETTER NAR + '\x10ac'# -> unI64 11532 + -- GEORGIAN CAPITAL LETTER ON + '\x10ad'# -> unI64 11533 + -- GEORGIAN CAPITAL LETTER PAR + '\x10ae'# -> unI64 11534 + -- GEORGIAN CAPITAL LETTER ZHAR + '\x10af'# -> unI64 11535 + -- GEORGIAN CAPITAL LETTER RAE + '\x10b0'# -> unI64 11536 + -- GEORGIAN CAPITAL LETTER SAN + '\x10b1'# -> unI64 11537 + -- GEORGIAN CAPITAL LETTER TAR + '\x10b2'# -> unI64 11538 + -- GEORGIAN CAPITAL LETTER UN + '\x10b3'# -> unI64 11539 + -- GEORGIAN CAPITAL LETTER PHAR + '\x10b4'# -> unI64 11540 + -- GEORGIAN CAPITAL LETTER KHAR + '\x10b5'# -> unI64 11541 + -- GEORGIAN CAPITAL LETTER GHAN + '\x10b6'# -> unI64 11542 + -- GEORGIAN CAPITAL LETTER QAR + '\x10b7'# -> unI64 11543 + -- GEORGIAN CAPITAL LETTER SHIN + '\x10b8'# -> unI64 11544 + -- GEORGIAN CAPITAL LETTER CHIN + '\x10b9'# -> unI64 11545 + -- GEORGIAN CAPITAL LETTER CAN + '\x10ba'# -> unI64 11546 + -- GEORGIAN CAPITAL LETTER JIL + '\x10bb'# -> unI64 11547 + -- GEORGIAN CAPITAL LETTER CIL + '\x10bc'# -> unI64 11548 + -- GEORGIAN CAPITAL LETTER CHAR + '\x10bd'# -> unI64 11549 + -- GEORGIAN CAPITAL LETTER XAN + '\x10be'# -> unI64 11550 + -- GEORGIAN CAPITAL LETTER JHAN + '\x10bf'# -> unI64 11551 + -- GEORGIAN CAPITAL LETTER HAE + '\x10c0'# -> unI64 11552 + -- GEORGIAN CAPITAL LETTER HE + '\x10c1'# -> unI64 11553 + -- GEORGIAN CAPITAL LETTER HIE + '\x10c2'# -> unI64 11554 + -- GEORGIAN CAPITAL LETTER WE + '\x10c3'# -> unI64 11555 + -- GEORGIAN CAPITAL LETTER HAR + '\x10c4'# -> unI64 11556 + -- GEORGIAN CAPITAL LETTER HOE + '\x10c5'# -> unI64 11557 + -- GEORGIAN CAPITAL LETTER YN + '\x10c7'# -> unI64 11559 + -- GEORGIAN CAPITAL LETTER AEN + '\x10cd'# -> unI64 11565 + -- CHEROKEE SMALL LETTER YE + '\x13f8'# -> unI64 5104 + -- CHEROKEE SMALL LETTER YI + '\x13f9'# -> unI64 5105 + -- CHEROKEE SMALL LETTER YO + '\x13fa'# -> unI64 5106 + -- CHEROKEE SMALL LETTER YU + '\x13fb'# -> unI64 5107 + -- CHEROKEE SMALL LETTER YV + '\x13fc'# -> unI64 5108 + -- CHEROKEE SMALL LETTER MV + '\x13fd'# -> unI64 5109 + -- CYRILLIC SMALL LETTER ROUNDED VE + '\x1c80'# -> unI64 1074 + -- CYRILLIC SMALL LETTER LONG-LEGGED DE + '\x1c81'# -> unI64 1076 + -- CYRILLIC SMALL LETTER NARROW O + '\x1c82'# -> unI64 1086 + -- CYRILLIC SMALL LETTER WIDE ES + '\x1c83'# -> unI64 1089 + -- CYRILLIC SMALL LETTER TALL TE + '\x1c84'# -> unI64 1090 + -- CYRILLIC SMALL LETTER THREE-LEGGED TE + '\x1c85'# -> unI64 1090 + -- CYRILLIC SMALL LETTER TALL HARD SIGN + '\x1c86'# -> unI64 1098 + -- CYRILLIC SMALL LETTER TALL YAT + '\x1c87'# -> unI64 1123 + -- CYRILLIC SMALL LETTER UNBLENDED UK + '\x1c88'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER TJE + '\x1c89'# -> unI64 7306 + -- GEORGIAN MTAVRULI CAPITAL LETTER AN + '\x1c90'# -> unI64 4304 + -- GEORGIAN MTAVRULI CAPITAL LETTER BAN + '\x1c91'# -> unI64 4305 + -- GEORGIAN MTAVRULI CAPITAL LETTER GAN + '\x1c92'# -> unI64 4306 + -- GEORGIAN MTAVRULI CAPITAL LETTER DON + '\x1c93'# -> unI64 4307 + -- GEORGIAN MTAVRULI CAPITAL LETTER EN + '\x1c94'# -> unI64 4308 + -- GEORGIAN MTAVRULI CAPITAL LETTER VIN + '\x1c95'# -> unI64 4309 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZEN + '\x1c96'# -> unI64 4310 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAN + '\x1c97'# -> unI64 4311 + -- GEORGIAN MTAVRULI CAPITAL LETTER IN + '\x1c98'# -> unI64 4312 + -- GEORGIAN MTAVRULI CAPITAL LETTER KAN + '\x1c99'# -> unI64 4313 + -- GEORGIAN MTAVRULI CAPITAL LETTER LAS + '\x1c9a'# -> unI64 4314 + -- GEORGIAN MTAVRULI CAPITAL LETTER MAN + '\x1c9b'# -> unI64 4315 + -- GEORGIAN MTAVRULI CAPITAL LETTER NAR + '\x1c9c'# -> unI64 4316 + -- GEORGIAN MTAVRULI CAPITAL LETTER ON + '\x1c9d'# -> unI64 4317 + -- GEORGIAN MTAVRULI CAPITAL LETTER PAR + '\x1c9e'# -> unI64 4318 + -- GEORGIAN MTAVRULI CAPITAL LETTER ZHAR + '\x1c9f'# -> unI64 4319 + -- GEORGIAN MTAVRULI CAPITAL LETTER RAE + '\x1ca0'# -> unI64 4320 + -- GEORGIAN MTAVRULI CAPITAL LETTER SAN + '\x1ca1'# -> unI64 4321 + -- GEORGIAN MTAVRULI CAPITAL LETTER TAR + '\x1ca2'# -> unI64 4322 + -- GEORGIAN MTAVRULI CAPITAL LETTER UN + '\x1ca3'# -> unI64 4323 + -- GEORGIAN MTAVRULI CAPITAL LETTER PHAR + '\x1ca4'# -> unI64 4324 + -- GEORGIAN MTAVRULI CAPITAL LETTER KHAR + '\x1ca5'# -> unI64 4325 + -- GEORGIAN MTAVRULI CAPITAL LETTER GHAN + '\x1ca6'# -> unI64 4326 + -- GEORGIAN MTAVRULI CAPITAL LETTER QAR + '\x1ca7'# -> unI64 4327 + -- GEORGIAN MTAVRULI CAPITAL LETTER SHIN + '\x1ca8'# -> unI64 4328 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHIN + '\x1ca9'# -> unI64 4329 + -- GEORGIAN MTAVRULI CAPITAL LETTER CAN + '\x1caa'# -> unI64 4330 + -- GEORGIAN MTAVRULI CAPITAL LETTER JIL + '\x1cab'# -> unI64 4331 + -- GEORGIAN MTAVRULI CAPITAL LETTER CIL + '\x1cac'# -> unI64 4332 + -- GEORGIAN MTAVRULI CAPITAL LETTER CHAR + '\x1cad'# -> unI64 4333 + -- GEORGIAN MTAVRULI CAPITAL LETTER XAN + '\x1cae'# -> unI64 4334 + -- GEORGIAN MTAVRULI CAPITAL LETTER JHAN + '\x1caf'# -> unI64 4335 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAE + '\x1cb0'# -> unI64 4336 + -- GEORGIAN MTAVRULI CAPITAL LETTER HE + '\x1cb1'# -> unI64 4337 + -- GEORGIAN MTAVRULI CAPITAL LETTER HIE + '\x1cb2'# -> unI64 4338 + -- GEORGIAN MTAVRULI CAPITAL LETTER WE + '\x1cb3'# -> unI64 4339 + -- GEORGIAN MTAVRULI CAPITAL LETTER HAR + '\x1cb4'# -> unI64 4340 + -- GEORGIAN MTAVRULI CAPITAL LETTER HOE + '\x1cb5'# -> unI64 4341 + -- GEORGIAN MTAVRULI CAPITAL LETTER FI + '\x1cb6'# -> unI64 4342 + -- GEORGIAN MTAVRULI CAPITAL LETTER YN + '\x1cb7'# -> unI64 4343 + -- GEORGIAN MTAVRULI CAPITAL LETTER ELIFI + '\x1cb8'# -> unI64 4344 + -- GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN + '\x1cb9'# -> unI64 4345 + -- GEORGIAN MTAVRULI CAPITAL LETTER AIN + '\x1cba'# -> unI64 4346 + -- GEORGIAN MTAVRULI CAPITAL LETTER AEN + '\x1cbd'# -> unI64 4349 + -- GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN + '\x1cbe'# -> unI64 4350 + -- GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + '\x1cbf'# -> unI64 4351 + -- LATIN CAPITAL LETTER A WITH RING BELOW + '\x1e00'# -> unI64 7681 + -- LATIN CAPITAL LETTER B WITH DOT ABOVE + '\x1e02'# -> unI64 7683 + -- LATIN CAPITAL LETTER B WITH DOT BELOW + '\x1e04'# -> unI64 7685 + -- LATIN CAPITAL LETTER B WITH LINE BELOW + '\x1e06'# -> unI64 7687 + -- LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + '\x1e08'# -> unI64 7689 + -- LATIN CAPITAL LETTER D WITH DOT ABOVE + '\x1e0a'# -> unI64 7691 + -- LATIN CAPITAL LETTER D WITH DOT BELOW + '\x1e0c'# -> unI64 7693 + -- LATIN CAPITAL LETTER D WITH LINE BELOW + '\x1e0e'# -> unI64 7695 + -- LATIN CAPITAL LETTER D WITH CEDILLA + '\x1e10'# -> unI64 7697 + -- LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + '\x1e12'# -> unI64 7699 + -- LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + '\x1e14'# -> unI64 7701 + -- LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + '\x1e16'# -> unI64 7703 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + '\x1e18'# -> unI64 7705 + -- LATIN CAPITAL LETTER E WITH TILDE BELOW + '\x1e1a'# -> unI64 7707 + -- LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + '\x1e1c'# -> unI64 7709 + -- LATIN CAPITAL LETTER F WITH DOT ABOVE + '\x1e1e'# -> unI64 7711 + -- LATIN CAPITAL LETTER G WITH MACRON + '\x1e20'# -> unI64 7713 + -- LATIN CAPITAL LETTER H WITH DOT ABOVE + '\x1e22'# -> unI64 7715 + -- LATIN CAPITAL LETTER H WITH DOT BELOW + '\x1e24'# -> unI64 7717 + -- LATIN CAPITAL LETTER H WITH DIAERESIS + '\x1e26'# -> unI64 7719 + -- LATIN CAPITAL LETTER H WITH CEDILLA + '\x1e28'# -> unI64 7721 + -- LATIN CAPITAL LETTER H WITH BREVE BELOW + '\x1e2a'# -> unI64 7723 + -- LATIN CAPITAL LETTER I WITH TILDE BELOW + '\x1e2c'# -> unI64 7725 + -- LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + '\x1e2e'# -> unI64 7727 + -- LATIN CAPITAL LETTER K WITH ACUTE + '\x1e30'# -> unI64 7729 + -- LATIN CAPITAL LETTER K WITH DOT BELOW + '\x1e32'# -> unI64 7731 + -- LATIN CAPITAL LETTER K WITH LINE BELOW + '\x1e34'# -> unI64 7733 + -- LATIN CAPITAL LETTER L WITH DOT BELOW + '\x1e36'# -> unI64 7735 + -- LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + '\x1e38'# -> unI64 7737 + -- LATIN CAPITAL LETTER L WITH LINE BELOW + '\x1e3a'# -> unI64 7739 + -- LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + '\x1e3c'# -> unI64 7741 + -- LATIN CAPITAL LETTER M WITH ACUTE + '\x1e3e'# -> unI64 7743 + -- LATIN CAPITAL LETTER M WITH DOT ABOVE + '\x1e40'# -> unI64 7745 + -- LATIN CAPITAL LETTER M WITH DOT BELOW + '\x1e42'# -> unI64 7747 + -- LATIN CAPITAL LETTER N WITH DOT ABOVE + '\x1e44'# -> unI64 7749 + -- LATIN CAPITAL LETTER N WITH DOT BELOW + '\x1e46'# -> unI64 7751 + -- LATIN CAPITAL LETTER N WITH LINE BELOW + '\x1e48'# -> unI64 7753 + -- LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + '\x1e4a'# -> unI64 7755 + -- LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + '\x1e4c'# -> unI64 7757 + -- LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + '\x1e4e'# -> unI64 7759 + -- LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + '\x1e50'# -> unI64 7761 + -- LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + '\x1e52'# -> unI64 7763 + -- LATIN CAPITAL LETTER P WITH ACUTE + '\x1e54'# -> unI64 7765 + -- LATIN CAPITAL LETTER P WITH DOT ABOVE + '\x1e56'# -> unI64 7767 + -- LATIN CAPITAL LETTER R WITH DOT ABOVE + '\x1e58'# -> unI64 7769 + -- LATIN CAPITAL LETTER R WITH DOT BELOW + '\x1e5a'# -> unI64 7771 + -- LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + '\x1e5c'# -> unI64 7773 + -- LATIN CAPITAL LETTER R WITH LINE BELOW + '\x1e5e'# -> unI64 7775 + -- LATIN CAPITAL LETTER S WITH DOT ABOVE + '\x1e60'# -> unI64 7777 + -- LATIN CAPITAL LETTER S WITH DOT BELOW + '\x1e62'# -> unI64 7779 + -- LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + '\x1e64'# -> unI64 7781 + -- LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + '\x1e66'# -> unI64 7783 + -- LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + '\x1e68'# -> unI64 7785 + -- LATIN CAPITAL LETTER T WITH DOT ABOVE + '\x1e6a'# -> unI64 7787 + -- LATIN CAPITAL LETTER T WITH DOT BELOW + '\x1e6c'# -> unI64 7789 + -- LATIN CAPITAL LETTER T WITH LINE BELOW + '\x1e6e'# -> unI64 7791 + -- LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + '\x1e70'# -> unI64 7793 + -- LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + '\x1e72'# -> unI64 7795 + -- LATIN CAPITAL LETTER U WITH TILDE BELOW + '\x1e74'# -> unI64 7797 + -- LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + '\x1e76'# -> unI64 7799 + -- LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + '\x1e78'# -> unI64 7801 + -- LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + '\x1e7a'# -> unI64 7803 + -- LATIN CAPITAL LETTER V WITH TILDE + '\x1e7c'# -> unI64 7805 + -- LATIN CAPITAL LETTER V WITH DOT BELOW + '\x1e7e'# -> unI64 7807 + -- LATIN CAPITAL LETTER W WITH GRAVE + '\x1e80'# -> unI64 7809 + -- LATIN CAPITAL LETTER W WITH ACUTE + '\x1e82'# -> unI64 7811 + -- LATIN CAPITAL LETTER W WITH DIAERESIS + '\x1e84'# -> unI64 7813 + -- LATIN CAPITAL LETTER W WITH DOT ABOVE + '\x1e86'# -> unI64 7815 + -- LATIN CAPITAL LETTER W WITH DOT BELOW + '\x1e88'# -> unI64 7817 + -- LATIN CAPITAL LETTER X WITH DOT ABOVE + '\x1e8a'# -> unI64 7819 + -- LATIN CAPITAL LETTER X WITH DIAERESIS + '\x1e8c'# -> unI64 7821 + -- LATIN CAPITAL LETTER Y WITH DOT ABOVE + '\x1e8e'# -> unI64 7823 + -- LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + '\x1e90'# -> unI64 7825 + -- LATIN CAPITAL LETTER Z WITH DOT BELOW + '\x1e92'# -> unI64 7827 + -- LATIN CAPITAL LETTER Z WITH LINE BELOW + '\x1e94'# -> unI64 7829 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373288 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390068 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584375 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584377 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200801 + -- LATIN SMALL LETTER LONG S WITH DOT ABOVE + '\x1e9b'# -> unI64 7777 + -- LATIN CAPITAL LETTER SHARP S + '\x1e9e'# -> unI64 241172595 + -- LATIN CAPITAL LETTER A WITH DOT BELOW + '\x1ea0'# -> unI64 7841 + -- LATIN CAPITAL LETTER A WITH HOOK ABOVE + '\x1ea2'# -> unI64 7843 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + '\x1ea4'# -> unI64 7845 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + '\x1ea6'# -> unI64 7847 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ea8'# -> unI64 7849 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + '\x1eaa'# -> unI64 7851 + -- LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + '\x1eac'# -> unI64 7853 + -- LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + '\x1eae'# -> unI64 7855 + -- LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + '\x1eb0'# -> unI64 7857 + -- LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + '\x1eb2'# -> unI64 7859 + -- LATIN CAPITAL LETTER A WITH BREVE AND TILDE + '\x1eb4'# -> unI64 7861 + -- LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + '\x1eb6'# -> unI64 7863 + -- LATIN CAPITAL LETTER E WITH DOT BELOW + '\x1eb8'# -> unI64 7865 + -- LATIN CAPITAL LETTER E WITH HOOK ABOVE + '\x1eba'# -> unI64 7867 + -- LATIN CAPITAL LETTER E WITH TILDE + '\x1ebc'# -> unI64 7869 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + '\x1ebe'# -> unI64 7871 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + '\x1ec0'# -> unI64 7873 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ec2'# -> unI64 7875 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + '\x1ec4'# -> unI64 7877 + -- LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + '\x1ec6'# -> unI64 7879 + -- LATIN CAPITAL LETTER I WITH HOOK ABOVE + '\x1ec8'# -> unI64 7881 + -- LATIN CAPITAL LETTER I WITH DOT BELOW + '\x1eca'# -> unI64 7883 + -- LATIN CAPITAL LETTER O WITH DOT BELOW + '\x1ecc'# -> unI64 7885 + -- LATIN CAPITAL LETTER O WITH HOOK ABOVE + '\x1ece'# -> unI64 7887 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + '\x1ed0'# -> unI64 7889 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + '\x1ed2'# -> unI64 7891 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + '\x1ed4'# -> unI64 7893 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + '\x1ed6'# -> unI64 7895 + -- LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + '\x1ed8'# -> unI64 7897 + -- LATIN CAPITAL LETTER O WITH HORN AND ACUTE + '\x1eda'# -> unI64 7899 + -- LATIN CAPITAL LETTER O WITH HORN AND GRAVE + '\x1edc'# -> unI64 7901 + -- LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + '\x1ede'# -> unI64 7903 + -- LATIN CAPITAL LETTER O WITH HORN AND TILDE + '\x1ee0'# -> unI64 7905 + -- LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + '\x1ee2'# -> unI64 7907 + -- LATIN CAPITAL LETTER U WITH DOT BELOW + '\x1ee4'# -> unI64 7909 + -- LATIN CAPITAL LETTER U WITH HOOK ABOVE + '\x1ee6'# -> unI64 7911 + -- LATIN CAPITAL LETTER U WITH HORN AND ACUTE + '\x1ee8'# -> unI64 7913 + -- LATIN CAPITAL LETTER U WITH HORN AND GRAVE + '\x1eea'# -> unI64 7915 + -- LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + '\x1eec'# -> unI64 7917 + -- LATIN CAPITAL LETTER U WITH HORN AND TILDE + '\x1eee'# -> unI64 7919 + -- LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + '\x1ef0'# -> unI64 7921 + -- LATIN CAPITAL LETTER Y WITH GRAVE + '\x1ef2'# -> unI64 7923 + -- LATIN CAPITAL LETTER Y WITH DOT BELOW + '\x1ef4'# -> unI64 7925 + -- LATIN CAPITAL LETTER Y WITH HOOK ABOVE + '\x1ef6'# -> unI64 7927 + -- LATIN CAPITAL LETTER Y WITH TILDE + '\x1ef8'# -> unI64 7929 + -- LATIN CAPITAL LETTER MIDDLE-WELSH LL + '\x1efa'# -> unI64 7931 + -- LATIN CAPITAL LETTER MIDDLE-WELSH V + '\x1efc'# -> unI64 7933 + -- LATIN CAPITAL LETTER Y WITH LOOP + '\x1efe'# -> unI64 7935 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI + '\x1f08'# -> unI64 7936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA + '\x1f09'# -> unI64 7937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA + '\x1f0a'# -> unI64 7938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA + '\x1f0b'# -> unI64 7939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA + '\x1f0c'# -> unI64 7940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA + '\x1f0d'# -> unI64 7941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI + '\x1f0e'# -> unI64 7942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + '\x1f0f'# -> unI64 7943 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI + '\x1f18'# -> unI64 7952 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA + '\x1f19'# -> unI64 7953 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA + '\x1f1a'# -> unI64 7954 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA + '\x1f1b'# -> unI64 7955 + -- GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA + '\x1f1c'# -> unI64 7956 + -- GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + '\x1f1d'# -> unI64 7957 + -- GREEK CAPITAL LETTER ETA WITH PSILI + '\x1f28'# -> unI64 7968 + -- GREEK CAPITAL LETTER ETA WITH DASIA + '\x1f29'# -> unI64 7969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA + '\x1f2a'# -> unI64 7970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA + '\x1f2b'# -> unI64 7971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA + '\x1f2c'# -> unI64 7972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA + '\x1f2d'# -> unI64 7973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI + '\x1f2e'# -> unI64 7974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + '\x1f2f'# -> unI64 7975 + -- GREEK CAPITAL LETTER IOTA WITH PSILI + '\x1f38'# -> unI64 7984 + -- GREEK CAPITAL LETTER IOTA WITH DASIA + '\x1f39'# -> unI64 7985 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA + '\x1f3a'# -> unI64 7986 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA + '\x1f3b'# -> unI64 7987 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA + '\x1f3c'# -> unI64 7988 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA + '\x1f3d'# -> unI64 7989 + -- GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI + '\x1f3e'# -> unI64 7990 + -- GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + '\x1f3f'# -> unI64 7991 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI + '\x1f48'# -> unI64 8000 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA + '\x1f49'# -> unI64 8001 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA + '\x1f4a'# -> unI64 8002 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA + '\x1f4b'# -> unI64 8003 + -- GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA + '\x1f4c'# -> unI64 8004 + -- GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + '\x1f4d'# -> unI64 8005 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459589 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987461 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498565 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720325 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA + '\x1f59'# -> unI64 8017 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + '\x1f5b'# -> unI64 8019 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + '\x1f5d'# -> unI64 8021 + -- GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + '\x1f5f'# -> unI64 8023 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI + '\x1f68'# -> unI64 8032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA + '\x1f69'# -> unI64 8033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA + '\x1f6a'# -> unI64 8034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA + '\x1f6b'# -> unI64 8035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA + '\x1f6c'# -> unI64 8036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA + '\x1f6d'# -> unI64 8037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI + '\x1f6e'# -> unI64 8038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + '\x1f6f'# -> unI64 8039 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1998593792 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1998593793 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1998593794 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1998593795 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1998593796 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1998593797 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1998593798 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1998593799 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1998593792 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1998593793 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1998593794 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1998593795 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1998593796 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1998593797 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1998593798 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1998593799 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1998593824 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1998593825 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1998593826 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1998593827 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1998593828 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1998593829 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1998593830 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1998593831 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1998593824 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1998593825 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1998593826 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1998593827 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1998593828 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1998593829 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1998593830 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1998593831 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1998593888 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1998593889 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1998593890 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1998593891 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1998593892 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1998593893 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1998593894 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1998593895 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1998593888 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1998593889 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1998593890 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1998593891 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1998593892 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1998593893 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1998593894 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1998593895 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1998593904 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1998586801 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1998586796 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025713 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4191340074107825 + -- GREEK CAPITAL LETTER ALPHA WITH VRACHY + '\x1fb8'# -> unI64 8112 + -- GREEK CAPITAL LETTER ALPHA WITH MACRON + '\x1fb9'# -> unI64 8113 + -- GREEK CAPITAL LETTER ALPHA WITH VARIA + '\x1fba'# -> unI64 8048 + -- GREEK CAPITAL LETTER ALPHA WITH OXIA + '\x1fbb'# -> unI64 8049 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1998586801 + -- GREEK PROSGEGRAMMENI + '\x1fbe'# -> unI64 953 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1998593908 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1998586807 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1998586798 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025719 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4191340074107831 + -- GREEK CAPITAL LETTER EPSILON WITH VARIA + '\x1fc8'# -> unI64 8050 + -- GREEK CAPITAL LETTER EPSILON WITH OXIA + '\x1fc9'# -> unI64 8051 + -- GREEK CAPITAL LETTER ETA WITH VARIA + '\x1fca'# -> unI64 8052 + -- GREEK CAPITAL LETTER ETA WITH OXIA + '\x1fcb'# -> unI64 8053 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1998586807 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918777 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025721 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651641 + -- GREEK CAPITAL LETTER IOTA WITH VRACHY + '\x1fd8'# -> unI64 8144 + -- GREEK CAPITAL LETTER IOTA WITH MACRON + '\x1fd9'# -> unI64 8145 + -- GREEK CAPITAL LETTER IOTA WITH VARIA + '\x1fda'# -> unI64 8054 + -- GREEK CAPITAL LETTER IOTA WITH OXIA + '\x1fdb'# -> unI64 8055 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918789 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459585 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025733 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651653 + -- GREEK CAPITAL LETTER UPSILON WITH VRACHY + '\x1fe8'# -> unI64 8160 + -- GREEK CAPITAL LETTER UPSILON WITH MACRON + '\x1fe9'# -> unI64 8161 + -- GREEK CAPITAL LETTER UPSILON WITH VARIA + '\x1fea'# -> unI64 8058 + -- GREEK CAPITAL LETTER UPSILON WITH OXIA + '\x1feb'# -> unI64 8059 + -- GREEK CAPITAL LETTER RHO WITH DASIA + '\x1fec'# -> unI64 8165 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1998593916 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1998586825 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1998586830 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025737 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4191340074107849 + -- GREEK CAPITAL LETTER OMICRON WITH VARIA + '\x1ff8'# -> unI64 8056 + -- GREEK CAPITAL LETTER OMICRON WITH OXIA + '\x1ff9'# -> unI64 8057 + -- GREEK CAPITAL LETTER OMEGA WITH VARIA + '\x1ffa'# -> unI64 8060 + -- GREEK CAPITAL LETTER OMEGA WITH OXIA + '\x1ffb'# -> unI64 8061 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1998586825 + -- OHM SIGN + '\x2126'# -> unI64 969 + -- KELVIN SIGN + '\x212a'# -> unI64 107 + -- ANGSTROM SIGN + '\x212b'# -> unI64 229 + -- TURNED CAPITAL F + '\x2132'# -> unI64 8526 + -- ROMAN NUMERAL ONE + '\x2160'# -> unI64 8560 + -- ROMAN NUMERAL TWO + '\x2161'# -> unI64 8561 + -- ROMAN NUMERAL THREE + '\x2162'# -> unI64 8562 + -- ROMAN NUMERAL FOUR + '\x2163'# -> unI64 8563 + -- ROMAN NUMERAL FIVE + '\x2164'# -> unI64 8564 + -- ROMAN NUMERAL SIX + '\x2165'# -> unI64 8565 + -- ROMAN NUMERAL SEVEN + '\x2166'# -> unI64 8566 + -- ROMAN NUMERAL EIGHT + '\x2167'# -> unI64 8567 + -- ROMAN NUMERAL NINE + '\x2168'# -> unI64 8568 + -- ROMAN NUMERAL TEN + '\x2169'# -> unI64 8569 + -- ROMAN NUMERAL ELEVEN + '\x216a'# -> unI64 8570 + -- ROMAN NUMERAL TWELVE + '\x216b'# -> unI64 8571 + -- ROMAN NUMERAL FIFTY + '\x216c'# -> unI64 8572 + -- ROMAN NUMERAL ONE HUNDRED + '\x216d'# -> unI64 8573 + -- ROMAN NUMERAL FIVE HUNDRED + '\x216e'# -> unI64 8574 + -- ROMAN NUMERAL ONE THOUSAND + '\x216f'# -> unI64 8575 + -- ROMAN NUMERAL REVERSED ONE HUNDRED + '\x2183'# -> unI64 8580 + -- CIRCLED LATIN CAPITAL LETTER A + '\x24b6'# -> unI64 9424 + -- CIRCLED LATIN CAPITAL LETTER B + '\x24b7'# -> unI64 9425 + -- CIRCLED LATIN CAPITAL LETTER C + '\x24b8'# -> unI64 9426 + -- CIRCLED LATIN CAPITAL LETTER D + '\x24b9'# -> unI64 9427 + -- CIRCLED LATIN CAPITAL LETTER E + '\x24ba'# -> unI64 9428 + -- CIRCLED LATIN CAPITAL LETTER F + '\x24bb'# -> unI64 9429 + -- CIRCLED LATIN CAPITAL LETTER G + '\x24bc'# -> unI64 9430 + -- CIRCLED LATIN CAPITAL LETTER H + '\x24bd'# -> unI64 9431 + -- CIRCLED LATIN CAPITAL LETTER I + '\x24be'# -> unI64 9432 + -- CIRCLED LATIN CAPITAL LETTER J + '\x24bf'# -> unI64 9433 + -- CIRCLED LATIN CAPITAL LETTER K + '\x24c0'# -> unI64 9434 + -- CIRCLED LATIN CAPITAL LETTER L + '\x24c1'# -> unI64 9435 + -- CIRCLED LATIN CAPITAL LETTER M + '\x24c2'# -> unI64 9436 + -- CIRCLED LATIN CAPITAL LETTER N + '\x24c3'# -> unI64 9437 + -- CIRCLED LATIN CAPITAL LETTER O + '\x24c4'# -> unI64 9438 + -- CIRCLED LATIN CAPITAL LETTER P + '\x24c5'# -> unI64 9439 + -- CIRCLED LATIN CAPITAL LETTER Q + '\x24c6'# -> unI64 9440 + -- CIRCLED LATIN CAPITAL LETTER R + '\x24c7'# -> unI64 9441 + -- CIRCLED LATIN CAPITAL LETTER S + '\x24c8'# -> unI64 9442 + -- CIRCLED LATIN CAPITAL LETTER T + '\x24c9'# -> unI64 9443 + -- CIRCLED LATIN CAPITAL LETTER U + '\x24ca'# -> unI64 9444 + -- CIRCLED LATIN CAPITAL LETTER V + '\x24cb'# -> unI64 9445 + -- CIRCLED LATIN CAPITAL LETTER W + '\x24cc'# -> unI64 9446 + -- CIRCLED LATIN CAPITAL LETTER X + '\x24cd'# -> unI64 9447 + -- CIRCLED LATIN CAPITAL LETTER Y + '\x24ce'# -> unI64 9448 + -- CIRCLED LATIN CAPITAL LETTER Z + '\x24cf'# -> unI64 9449 + -- GLAGOLITIC CAPITAL LETTER AZU + '\x2c00'# -> unI64 11312 + -- GLAGOLITIC CAPITAL LETTER BUKY + '\x2c01'# -> unI64 11313 + -- GLAGOLITIC CAPITAL LETTER VEDE + '\x2c02'# -> unI64 11314 + -- GLAGOLITIC CAPITAL LETTER GLAGOLI + '\x2c03'# -> unI64 11315 + -- GLAGOLITIC CAPITAL LETTER DOBRO + '\x2c04'# -> unI64 11316 + -- GLAGOLITIC CAPITAL LETTER YESTU + '\x2c05'# -> unI64 11317 + -- GLAGOLITIC CAPITAL LETTER ZHIVETE + '\x2c06'# -> unI64 11318 + -- GLAGOLITIC CAPITAL LETTER DZELO + '\x2c07'# -> unI64 11319 + -- GLAGOLITIC CAPITAL LETTER ZEMLJA + '\x2c08'# -> unI64 11320 + -- GLAGOLITIC CAPITAL LETTER IZHE + '\x2c09'# -> unI64 11321 + -- GLAGOLITIC CAPITAL LETTER INITIAL IZHE + '\x2c0a'# -> unI64 11322 + -- GLAGOLITIC CAPITAL LETTER I + '\x2c0b'# -> unI64 11323 + -- GLAGOLITIC CAPITAL LETTER DJERVI + '\x2c0c'# -> unI64 11324 + -- GLAGOLITIC CAPITAL LETTER KAKO + '\x2c0d'# -> unI64 11325 + -- GLAGOLITIC CAPITAL LETTER LJUDIJE + '\x2c0e'# -> unI64 11326 + -- GLAGOLITIC CAPITAL LETTER MYSLITE + '\x2c0f'# -> unI64 11327 + -- GLAGOLITIC CAPITAL LETTER NASHI + '\x2c10'# -> unI64 11328 + -- GLAGOLITIC CAPITAL LETTER ONU + '\x2c11'# -> unI64 11329 + -- GLAGOLITIC CAPITAL LETTER POKOJI + '\x2c12'# -> unI64 11330 + -- GLAGOLITIC CAPITAL LETTER RITSI + '\x2c13'# -> unI64 11331 + -- GLAGOLITIC CAPITAL LETTER SLOVO + '\x2c14'# -> unI64 11332 + -- GLAGOLITIC CAPITAL LETTER TVRIDO + '\x2c15'# -> unI64 11333 + -- GLAGOLITIC CAPITAL LETTER UKU + '\x2c16'# -> unI64 11334 + -- GLAGOLITIC CAPITAL LETTER FRITU + '\x2c17'# -> unI64 11335 + -- GLAGOLITIC CAPITAL LETTER HERU + '\x2c18'# -> unI64 11336 + -- GLAGOLITIC CAPITAL LETTER OTU + '\x2c19'# -> unI64 11337 + -- GLAGOLITIC CAPITAL LETTER PE + '\x2c1a'# -> unI64 11338 + -- GLAGOLITIC CAPITAL LETTER SHTA + '\x2c1b'# -> unI64 11339 + -- GLAGOLITIC CAPITAL LETTER TSI + '\x2c1c'# -> unI64 11340 + -- GLAGOLITIC CAPITAL LETTER CHRIVI + '\x2c1d'# -> unI64 11341 + -- GLAGOLITIC CAPITAL LETTER SHA + '\x2c1e'# -> unI64 11342 + -- GLAGOLITIC CAPITAL LETTER YERU + '\x2c1f'# -> unI64 11343 + -- GLAGOLITIC CAPITAL LETTER YERI + '\x2c20'# -> unI64 11344 + -- GLAGOLITIC CAPITAL LETTER YATI + '\x2c21'# -> unI64 11345 + -- GLAGOLITIC CAPITAL LETTER SPIDERY HA + '\x2c22'# -> unI64 11346 + -- GLAGOLITIC CAPITAL LETTER YU + '\x2c23'# -> unI64 11347 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS + '\x2c24'# -> unI64 11348 + -- GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL + '\x2c25'# -> unI64 11349 + -- GLAGOLITIC CAPITAL LETTER YO + '\x2c26'# -> unI64 11350 + -- GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS + '\x2c27'# -> unI64 11351 + -- GLAGOLITIC CAPITAL LETTER BIG YUS + '\x2c28'# -> unI64 11352 + -- GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS + '\x2c29'# -> unI64 11353 + -- GLAGOLITIC CAPITAL LETTER FITA + '\x2c2a'# -> unI64 11354 + -- GLAGOLITIC CAPITAL LETTER IZHITSA + '\x2c2b'# -> unI64 11355 + -- GLAGOLITIC CAPITAL LETTER SHTAPIC + '\x2c2c'# -> unI64 11356 + -- GLAGOLITIC CAPITAL LETTER TROKUTASTI A + '\x2c2d'# -> unI64 11357 + -- GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE + '\x2c2e'# -> unI64 11358 + -- GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI + '\x2c2f'# -> unI64 11359 + -- LATIN CAPITAL LETTER L WITH DOUBLE BAR + '\x2c60'# -> unI64 11361 + -- LATIN CAPITAL LETTER L WITH MIDDLE TILDE + '\x2c62'# -> unI64 619 + -- LATIN CAPITAL LETTER P WITH STROKE + '\x2c63'# -> unI64 7549 + -- LATIN CAPITAL LETTER R WITH TAIL + '\x2c64'# -> unI64 637 + -- LATIN CAPITAL LETTER H WITH DESCENDER + '\x2c67'# -> unI64 11368 + -- LATIN CAPITAL LETTER K WITH DESCENDER + '\x2c69'# -> unI64 11370 + -- LATIN CAPITAL LETTER Z WITH DESCENDER + '\x2c6b'# -> unI64 11372 + -- LATIN CAPITAL LETTER ALPHA + '\x2c6d'# -> unI64 593 + -- LATIN CAPITAL LETTER M WITH HOOK + '\x2c6e'# -> unI64 625 + -- LATIN CAPITAL LETTER TURNED A + '\x2c6f'# -> unI64 592 + -- LATIN CAPITAL LETTER TURNED ALPHA + '\x2c70'# -> unI64 594 + -- LATIN CAPITAL LETTER W WITH HOOK + '\x2c72'# -> unI64 11379 + -- LATIN CAPITAL LETTER HALF H + '\x2c75'# -> unI64 11382 + -- LATIN CAPITAL LETTER S WITH SWASH TAIL + '\x2c7e'# -> unI64 575 + -- LATIN CAPITAL LETTER Z WITH SWASH TAIL + '\x2c7f'# -> unI64 576 + -- COPTIC CAPITAL LETTER ALFA + '\x2c80'# -> unI64 11393 + -- COPTIC CAPITAL LETTER VIDA + '\x2c82'# -> unI64 11395 + -- COPTIC CAPITAL LETTER GAMMA + '\x2c84'# -> unI64 11397 + -- COPTIC CAPITAL LETTER DALDA + '\x2c86'# -> unI64 11399 + -- COPTIC CAPITAL LETTER EIE + '\x2c88'# -> unI64 11401 + -- COPTIC CAPITAL LETTER SOU + '\x2c8a'# -> unI64 11403 + -- COPTIC CAPITAL LETTER ZATA + '\x2c8c'# -> unI64 11405 + -- COPTIC CAPITAL LETTER HATE + '\x2c8e'# -> unI64 11407 + -- COPTIC CAPITAL LETTER THETHE + '\x2c90'# -> unI64 11409 + -- COPTIC CAPITAL LETTER IAUDA + '\x2c92'# -> unI64 11411 + -- COPTIC CAPITAL LETTER KAPA + '\x2c94'# -> unI64 11413 + -- COPTIC CAPITAL LETTER LAULA + '\x2c96'# -> unI64 11415 + -- COPTIC CAPITAL LETTER MI + '\x2c98'# -> unI64 11417 + -- COPTIC CAPITAL LETTER NI + '\x2c9a'# -> unI64 11419 + -- COPTIC CAPITAL LETTER KSI + '\x2c9c'# -> unI64 11421 + -- COPTIC CAPITAL LETTER O + '\x2c9e'# -> unI64 11423 + -- COPTIC CAPITAL LETTER PI + '\x2ca0'# -> unI64 11425 + -- COPTIC CAPITAL LETTER RO + '\x2ca2'# -> unI64 11427 + -- COPTIC CAPITAL LETTER SIMA + '\x2ca4'# -> unI64 11429 + -- COPTIC CAPITAL LETTER TAU + '\x2ca6'# -> unI64 11431 + -- COPTIC CAPITAL LETTER UA + '\x2ca8'# -> unI64 11433 + -- COPTIC CAPITAL LETTER FI + '\x2caa'# -> unI64 11435 + -- COPTIC CAPITAL LETTER KHI + '\x2cac'# -> unI64 11437 + -- COPTIC CAPITAL LETTER PSI + '\x2cae'# -> unI64 11439 + -- COPTIC CAPITAL LETTER OOU + '\x2cb0'# -> unI64 11441 + -- COPTIC CAPITAL LETTER DIALECT-P ALEF + '\x2cb2'# -> unI64 11443 + -- COPTIC CAPITAL LETTER OLD COPTIC AIN + '\x2cb4'# -> unI64 11445 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE + '\x2cb6'# -> unI64 11447 + -- COPTIC CAPITAL LETTER DIALECT-P KAPA + '\x2cb8'# -> unI64 11449 + -- COPTIC CAPITAL LETTER DIALECT-P NI + '\x2cba'# -> unI64 11451 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI + '\x2cbc'# -> unI64 11453 + -- COPTIC CAPITAL LETTER OLD COPTIC OOU + '\x2cbe'# -> unI64 11455 + -- COPTIC CAPITAL LETTER SAMPI + '\x2cc0'# -> unI64 11457 + -- COPTIC CAPITAL LETTER CROSSED SHEI + '\x2cc2'# -> unI64 11459 + -- COPTIC CAPITAL LETTER OLD COPTIC SHEI + '\x2cc4'# -> unI64 11461 + -- COPTIC CAPITAL LETTER OLD COPTIC ESH + '\x2cc6'# -> unI64 11463 + -- COPTIC CAPITAL LETTER AKHMIMIC KHEI + '\x2cc8'# -> unI64 11465 + -- COPTIC CAPITAL LETTER DIALECT-P HORI + '\x2cca'# -> unI64 11467 + -- COPTIC CAPITAL LETTER OLD COPTIC HORI + '\x2ccc'# -> unI64 11469 + -- COPTIC CAPITAL LETTER OLD COPTIC HA + '\x2cce'# -> unI64 11471 + -- COPTIC CAPITAL LETTER L-SHAPED HA + '\x2cd0'# -> unI64 11473 + -- COPTIC CAPITAL LETTER OLD COPTIC HEI + '\x2cd2'# -> unI64 11475 + -- COPTIC CAPITAL LETTER OLD COPTIC HAT + '\x2cd4'# -> unI64 11477 + -- COPTIC CAPITAL LETTER OLD COPTIC GANGIA + '\x2cd6'# -> unI64 11479 + -- COPTIC CAPITAL LETTER OLD COPTIC DJA + '\x2cd8'# -> unI64 11481 + -- COPTIC CAPITAL LETTER OLD COPTIC SHIMA + '\x2cda'# -> unI64 11483 + -- COPTIC CAPITAL LETTER OLD NUBIAN SHIMA + '\x2cdc'# -> unI64 11485 + -- COPTIC CAPITAL LETTER OLD NUBIAN NGI + '\x2cde'# -> unI64 11487 + -- COPTIC CAPITAL LETTER OLD NUBIAN NYI + '\x2ce0'# -> unI64 11489 + -- COPTIC CAPITAL LETTER OLD NUBIAN WAU + '\x2ce2'# -> unI64 11491 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI + '\x2ceb'# -> unI64 11500 + -- COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA + '\x2ced'# -> unI64 11502 + -- COPTIC CAPITAL LETTER BOHAIRIC KHEI + '\x2cf2'# -> unI64 11507 + -- CYRILLIC CAPITAL LETTER ZEMLYA + '\xa640'# -> unI64 42561 + -- CYRILLIC CAPITAL LETTER DZELO + '\xa642'# -> unI64 42563 + -- CYRILLIC CAPITAL LETTER REVERSED DZE + '\xa644'# -> unI64 42565 + -- CYRILLIC CAPITAL LETTER IOTA + '\xa646'# -> unI64 42567 + -- CYRILLIC CAPITAL LETTER DJERV + '\xa648'# -> unI64 42569 + -- CYRILLIC CAPITAL LETTER MONOGRAPH UK + '\xa64a'# -> unI64 42571 + -- CYRILLIC CAPITAL LETTER BROAD OMEGA + '\xa64c'# -> unI64 42573 + -- CYRILLIC CAPITAL LETTER NEUTRAL YER + '\xa64e'# -> unI64 42575 + -- CYRILLIC CAPITAL LETTER YERU WITH BACK YER + '\xa650'# -> unI64 42577 + -- CYRILLIC CAPITAL LETTER IOTIFIED YAT + '\xa652'# -> unI64 42579 + -- CYRILLIC CAPITAL LETTER REVERSED YU + '\xa654'# -> unI64 42581 + -- CYRILLIC CAPITAL LETTER IOTIFIED A + '\xa656'# -> unI64 42583 + -- CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS + '\xa658'# -> unI64 42585 + -- CYRILLIC CAPITAL LETTER BLENDED YUS + '\xa65a'# -> unI64 42587 + -- CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS + '\xa65c'# -> unI64 42589 + -- CYRILLIC CAPITAL LETTER YN + '\xa65e'# -> unI64 42591 + -- CYRILLIC CAPITAL LETTER REVERSED TSE + '\xa660'# -> unI64 42593 + -- CYRILLIC CAPITAL LETTER SOFT DE + '\xa662'# -> unI64 42595 + -- CYRILLIC CAPITAL LETTER SOFT EL + '\xa664'# -> unI64 42597 + -- CYRILLIC CAPITAL LETTER SOFT EM + '\xa666'# -> unI64 42599 + -- CYRILLIC CAPITAL LETTER MONOCULAR O + '\xa668'# -> unI64 42601 + -- CYRILLIC CAPITAL LETTER BINOCULAR O + '\xa66a'# -> unI64 42603 + -- CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O + '\xa66c'# -> unI64 42605 + -- CYRILLIC CAPITAL LETTER DWE + '\xa680'# -> unI64 42625 + -- CYRILLIC CAPITAL LETTER DZWE + '\xa682'# -> unI64 42627 + -- CYRILLIC CAPITAL LETTER ZHWE + '\xa684'# -> unI64 42629 + -- CYRILLIC CAPITAL LETTER CCHE + '\xa686'# -> unI64 42631 + -- CYRILLIC CAPITAL LETTER DZZE + '\xa688'# -> unI64 42633 + -- CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK + '\xa68a'# -> unI64 42635 + -- CYRILLIC CAPITAL LETTER TWE + '\xa68c'# -> unI64 42637 + -- CYRILLIC CAPITAL LETTER TSWE + '\xa68e'# -> unI64 42639 + -- CYRILLIC CAPITAL LETTER TSSE + '\xa690'# -> unI64 42641 + -- CYRILLIC CAPITAL LETTER TCHE + '\xa692'# -> unI64 42643 + -- CYRILLIC CAPITAL LETTER HWE + '\xa694'# -> unI64 42645 + -- CYRILLIC CAPITAL LETTER SHWE + '\xa696'# -> unI64 42647 + -- CYRILLIC CAPITAL LETTER DOUBLE O + '\xa698'# -> unI64 42649 + -- CYRILLIC CAPITAL LETTER CROSSED O + '\xa69a'# -> unI64 42651 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF + '\xa722'# -> unI64 42787 + -- LATIN CAPITAL LETTER EGYPTOLOGICAL AIN + '\xa724'# -> unI64 42789 + -- LATIN CAPITAL LETTER HENG + '\xa726'# -> unI64 42791 + -- LATIN CAPITAL LETTER TZ + '\xa728'# -> unI64 42793 + -- LATIN CAPITAL LETTER TRESILLO + '\xa72a'# -> unI64 42795 + -- LATIN CAPITAL LETTER CUATRILLO + '\xa72c'# -> unI64 42797 + -- LATIN CAPITAL LETTER CUATRILLO WITH COMMA + '\xa72e'# -> unI64 42799 + -- LATIN CAPITAL LETTER AA + '\xa732'# -> unI64 42803 + -- LATIN CAPITAL LETTER AO + '\xa734'# -> unI64 42805 + -- LATIN CAPITAL LETTER AU + '\xa736'# -> unI64 42807 + -- LATIN CAPITAL LETTER AV + '\xa738'# -> unI64 42809 + -- LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR + '\xa73a'# -> unI64 42811 + -- LATIN CAPITAL LETTER AY + '\xa73c'# -> unI64 42813 + -- LATIN CAPITAL LETTER REVERSED C WITH DOT + '\xa73e'# -> unI64 42815 + -- LATIN CAPITAL LETTER K WITH STROKE + '\xa740'# -> unI64 42817 + -- LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + '\xa742'# -> unI64 42819 + -- LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + '\xa744'# -> unI64 42821 + -- LATIN CAPITAL LETTER BROKEN L + '\xa746'# -> unI64 42823 + -- LATIN CAPITAL LETTER L WITH HIGH STROKE + '\xa748'# -> unI64 42825 + -- LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + '\xa74a'# -> unI64 42827 + -- LATIN CAPITAL LETTER O WITH LOOP + '\xa74c'# -> unI64 42829 + -- LATIN CAPITAL LETTER OO + '\xa74e'# -> unI64 42831 + -- LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + '\xa750'# -> unI64 42833 + -- LATIN CAPITAL LETTER P WITH FLOURISH + '\xa752'# -> unI64 42835 + -- LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + '\xa754'# -> unI64 42837 + -- LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + '\xa756'# -> unI64 42839 + -- LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + '\xa758'# -> unI64 42841 + -- LATIN CAPITAL LETTER R ROTUNDA + '\xa75a'# -> unI64 42843 + -- LATIN CAPITAL LETTER RUM ROTUNDA + '\xa75c'# -> unI64 42845 + -- LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + '\xa75e'# -> unI64 42847 + -- LATIN CAPITAL LETTER VY + '\xa760'# -> unI64 42849 + -- LATIN CAPITAL LETTER VISIGOTHIC Z + '\xa762'# -> unI64 42851 + -- LATIN CAPITAL LETTER THORN WITH STROKE + '\xa764'# -> unI64 42853 + -- LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER + '\xa766'# -> unI64 42855 + -- LATIN CAPITAL LETTER VEND + '\xa768'# -> unI64 42857 + -- LATIN CAPITAL LETTER ET + '\xa76a'# -> unI64 42859 + -- LATIN CAPITAL LETTER IS + '\xa76c'# -> unI64 42861 + -- LATIN CAPITAL LETTER CON + '\xa76e'# -> unI64 42863 + -- LATIN CAPITAL LETTER INSULAR D + '\xa779'# -> unI64 42874 + -- LATIN CAPITAL LETTER INSULAR F + '\xa77b'# -> unI64 42876 + -- LATIN CAPITAL LETTER INSULAR G + '\xa77d'# -> unI64 7545 + -- LATIN CAPITAL LETTER TURNED INSULAR G + '\xa77e'# -> unI64 42879 + -- LATIN CAPITAL LETTER TURNED L + '\xa780'# -> unI64 42881 + -- LATIN CAPITAL LETTER INSULAR R + '\xa782'# -> unI64 42883 + -- LATIN CAPITAL LETTER INSULAR S + '\xa784'# -> unI64 42885 + -- LATIN CAPITAL LETTER INSULAR T + '\xa786'# -> unI64 42887 + -- LATIN CAPITAL LETTER SALTILLO + '\xa78b'# -> unI64 42892 + -- LATIN CAPITAL LETTER TURNED H + '\xa78d'# -> unI64 613 + -- LATIN CAPITAL LETTER N WITH DESCENDER + '\xa790'# -> unI64 42897 + -- LATIN CAPITAL LETTER C WITH BAR + '\xa792'# -> unI64 42899 + -- LATIN CAPITAL LETTER B WITH FLOURISH + '\xa796'# -> unI64 42903 + -- LATIN CAPITAL LETTER F WITH STROKE + '\xa798'# -> unI64 42905 + -- LATIN CAPITAL LETTER VOLAPUK AE + '\xa79a'# -> unI64 42907 + -- LATIN CAPITAL LETTER VOLAPUK OE + '\xa79c'# -> unI64 42909 + -- LATIN CAPITAL LETTER VOLAPUK UE + '\xa79e'# -> unI64 42911 + -- LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + '\xa7a0'# -> unI64 42913 + -- LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + '\xa7a2'# -> unI64 42915 + -- LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + '\xa7a4'# -> unI64 42917 + -- LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + '\xa7a6'# -> unI64 42919 + -- LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + '\xa7a8'# -> unI64 42921 + -- LATIN CAPITAL LETTER H WITH HOOK + '\xa7aa'# -> unI64 614 + -- LATIN CAPITAL LETTER REVERSED OPEN E + '\xa7ab'# -> unI64 604 + -- LATIN CAPITAL LETTER SCRIPT G + '\xa7ac'# -> unI64 609 + -- LATIN CAPITAL LETTER L WITH BELT + '\xa7ad'# -> unI64 620 + -- LATIN CAPITAL LETTER SMALL CAPITAL I + '\xa7ae'# -> unI64 618 + -- LATIN CAPITAL LETTER TURNED K + '\xa7b0'# -> unI64 670 + -- LATIN CAPITAL LETTER TURNED T + '\xa7b1'# -> unI64 647 + -- LATIN CAPITAL LETTER J WITH CROSSED-TAIL + '\xa7b2'# -> unI64 669 + -- LATIN CAPITAL LETTER CHI + '\xa7b3'# -> unI64 43859 + -- LATIN CAPITAL LETTER BETA + '\xa7b4'# -> unI64 42933 + -- LATIN CAPITAL LETTER OMEGA + '\xa7b6'# -> unI64 42935 + -- LATIN CAPITAL LETTER U WITH STROKE + '\xa7b8'# -> unI64 42937 + -- LATIN CAPITAL LETTER GLOTTAL A + '\xa7ba'# -> unI64 42939 + -- LATIN CAPITAL LETTER GLOTTAL I + '\xa7bc'# -> unI64 42941 + -- LATIN CAPITAL LETTER GLOTTAL U + '\xa7be'# -> unI64 42943 + -- LATIN CAPITAL LETTER OLD POLISH O + '\xa7c0'# -> unI64 42945 + -- LATIN CAPITAL LETTER ANGLICANA W + '\xa7c2'# -> unI64 42947 + -- LATIN CAPITAL LETTER C WITH PALATAL HOOK + '\xa7c4'# -> unI64 42900 + -- LATIN CAPITAL LETTER S WITH HOOK + '\xa7c5'# -> unI64 642 + -- LATIN CAPITAL LETTER Z WITH PALATAL HOOK + '\xa7c6'# -> unI64 7566 + -- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + '\xa7c7'# -> unI64 42952 + -- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + '\xa7c9'# -> unI64 42954 + -- LATIN CAPITAL LETTER RAMS HORN + '\xa7cb'# -> unI64 612 + -- LATIN CAPITAL LETTER S WITH DIAGONAL STROKE + '\xa7cc'# -> unI64 42957 + -- LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE + '\xa7ce'# -> unI64 42959 + -- LATIN CAPITAL LETTER CLOSED INSULAR G + '\xa7d0'# -> unI64 42961 + -- LATIN CAPITAL LETTER DOUBLE THORN + '\xa7d2'# -> unI64 42963 + -- LATIN CAPITAL LETTER DOUBLE WYNN + '\xa7d4'# -> unI64 42965 + -- LATIN CAPITAL LETTER MIDDLE SCOTS S + '\xa7d6'# -> unI64 42967 + -- LATIN CAPITAL LETTER SIGMOID S + '\xa7d8'# -> unI64 42969 + -- LATIN CAPITAL LETTER LAMBDA + '\xa7da'# -> unI64 42971 + -- LATIN CAPITAL LETTER LAMBDA WITH STROKE + '\xa7dc'# -> unI64 411 + -- LATIN CAPITAL LETTER REVERSED HALF H + '\xa7f5'# -> unI64 42998 + -- CHEROKEE SMALL LETTER A + '\xab70'# -> unI64 5024 + -- CHEROKEE SMALL LETTER E + '\xab71'# -> unI64 5025 + -- CHEROKEE SMALL LETTER I + '\xab72'# -> unI64 5026 + -- CHEROKEE SMALL LETTER O + '\xab73'# -> unI64 5027 + -- CHEROKEE SMALL LETTER U + '\xab74'# -> unI64 5028 + -- CHEROKEE SMALL LETTER V + '\xab75'# -> unI64 5029 + -- CHEROKEE SMALL LETTER GA + '\xab76'# -> unI64 5030 + -- CHEROKEE SMALL LETTER KA + '\xab77'# -> unI64 5031 + -- CHEROKEE SMALL LETTER GE + '\xab78'# -> unI64 5032 + -- CHEROKEE SMALL LETTER GI + '\xab79'# -> unI64 5033 + -- CHEROKEE SMALL LETTER GO + '\xab7a'# -> unI64 5034 + -- CHEROKEE SMALL LETTER GU + '\xab7b'# -> unI64 5035 + -- CHEROKEE SMALL LETTER GV + '\xab7c'# -> unI64 5036 + -- CHEROKEE SMALL LETTER HA + '\xab7d'# -> unI64 5037 + -- CHEROKEE SMALL LETTER HE + '\xab7e'# -> unI64 5038 + -- CHEROKEE SMALL LETTER HI + '\xab7f'# -> unI64 5039 + -- CHEROKEE SMALL LETTER HO + '\xab80'# -> unI64 5040 + -- CHEROKEE SMALL LETTER HU + '\xab81'# -> unI64 5041 + -- CHEROKEE SMALL LETTER HV + '\xab82'# -> unI64 5042 + -- CHEROKEE SMALL LETTER LA + '\xab83'# -> unI64 5043 + -- CHEROKEE SMALL LETTER LE + '\xab84'# -> unI64 5044 + -- CHEROKEE SMALL LETTER LI + '\xab85'# -> unI64 5045 + -- CHEROKEE SMALL LETTER LO + '\xab86'# -> unI64 5046 + -- CHEROKEE SMALL LETTER LU + '\xab87'# -> unI64 5047 + -- CHEROKEE SMALL LETTER LV + '\xab88'# -> unI64 5048 + -- CHEROKEE SMALL LETTER MA + '\xab89'# -> unI64 5049 + -- CHEROKEE SMALL LETTER ME + '\xab8a'# -> unI64 5050 + -- CHEROKEE SMALL LETTER MI + '\xab8b'# -> unI64 5051 + -- CHEROKEE SMALL LETTER MO + '\xab8c'# -> unI64 5052 + -- CHEROKEE SMALL LETTER MU + '\xab8d'# -> unI64 5053 + -- CHEROKEE SMALL LETTER NA + '\xab8e'# -> unI64 5054 + -- CHEROKEE SMALL LETTER HNA + '\xab8f'# -> unI64 5055 + -- CHEROKEE SMALL LETTER NAH + '\xab90'# -> unI64 5056 + -- CHEROKEE SMALL LETTER NE + '\xab91'# -> unI64 5057 + -- CHEROKEE SMALL LETTER NI + '\xab92'# -> unI64 5058 + -- CHEROKEE SMALL LETTER NO + '\xab93'# -> unI64 5059 + -- CHEROKEE SMALL LETTER NU + '\xab94'# -> unI64 5060 + -- CHEROKEE SMALL LETTER NV + '\xab95'# -> unI64 5061 + -- CHEROKEE SMALL LETTER QUA + '\xab96'# -> unI64 5062 + -- CHEROKEE SMALL LETTER QUE + '\xab97'# -> unI64 5063 + -- CHEROKEE SMALL LETTER QUI + '\xab98'# -> unI64 5064 + -- CHEROKEE SMALL LETTER QUO + '\xab99'# -> unI64 5065 + -- CHEROKEE SMALL LETTER QUU + '\xab9a'# -> unI64 5066 + -- CHEROKEE SMALL LETTER QUV + '\xab9b'# -> unI64 5067 + -- CHEROKEE SMALL LETTER SA + '\xab9c'# -> unI64 5068 + -- CHEROKEE SMALL LETTER S + '\xab9d'# -> unI64 5069 + -- CHEROKEE SMALL LETTER SE + '\xab9e'# -> unI64 5070 + -- CHEROKEE SMALL LETTER SI + '\xab9f'# -> unI64 5071 + -- CHEROKEE SMALL LETTER SO + '\xaba0'# -> unI64 5072 + -- CHEROKEE SMALL LETTER SU + '\xaba1'# -> unI64 5073 + -- CHEROKEE SMALL LETTER SV + '\xaba2'# -> unI64 5074 + -- CHEROKEE SMALL LETTER DA + '\xaba3'# -> unI64 5075 + -- CHEROKEE SMALL LETTER TA + '\xaba4'# -> unI64 5076 + -- CHEROKEE SMALL LETTER DE + '\xaba5'# -> unI64 5077 + -- CHEROKEE SMALL LETTER TE + '\xaba6'# -> unI64 5078 + -- CHEROKEE SMALL LETTER DI + '\xaba7'# -> unI64 5079 + -- CHEROKEE SMALL LETTER TI + '\xaba8'# -> unI64 5080 + -- CHEROKEE SMALL LETTER DO + '\xaba9'# -> unI64 5081 + -- CHEROKEE SMALL LETTER DU + '\xabaa'# -> unI64 5082 + -- CHEROKEE SMALL LETTER DV + '\xabab'# -> unI64 5083 + -- CHEROKEE SMALL LETTER DLA + '\xabac'# -> unI64 5084 + -- CHEROKEE SMALL LETTER TLA + '\xabad'# -> unI64 5085 + -- CHEROKEE SMALL LETTER TLE + '\xabae'# -> unI64 5086 + -- CHEROKEE SMALL LETTER TLI + '\xabaf'# -> unI64 5087 + -- CHEROKEE SMALL LETTER TLO + '\xabb0'# -> unI64 5088 + -- CHEROKEE SMALL LETTER TLU + '\xabb1'# -> unI64 5089 + -- CHEROKEE SMALL LETTER TLV + '\xabb2'# -> unI64 5090 + -- CHEROKEE SMALL LETTER TSA + '\xabb3'# -> unI64 5091 + -- CHEROKEE SMALL LETTER TSE + '\xabb4'# -> unI64 5092 + -- CHEROKEE SMALL LETTER TSI + '\xabb5'# -> unI64 5093 + -- CHEROKEE SMALL LETTER TSO + '\xabb6'# -> unI64 5094 + -- CHEROKEE SMALL LETTER TSU + '\xabb7'# -> unI64 5095 + -- CHEROKEE SMALL LETTER TSV + '\xabb8'# -> unI64 5096 + -- CHEROKEE SMALL LETTER WA + '\xabb9'# -> unI64 5097 + -- CHEROKEE SMALL LETTER WE + '\xabba'# -> unI64 5098 + -- CHEROKEE SMALL LETTER WI + '\xabbb'# -> unI64 5099 + -- CHEROKEE SMALL LETTER WO + '\xabbc'# -> unI64 5100 + -- CHEROKEE SMALL LETTER WU + '\xabbd'# -> unI64 5101 + -- CHEROKEE SMALL LETTER WV + '\xabbe'# -> unI64 5102 + -- CHEROKEE SMALL LETTER YA + '\xabbf'# -> unI64 5103 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909606 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201062 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492518 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575526 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108838 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269747 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269747 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819892 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168308 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751220 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819902 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945524 + -- FULLWIDTH LATIN CAPITAL LETTER A + '\xff21'# -> unI64 65345 + -- FULLWIDTH LATIN CAPITAL LETTER B + '\xff22'# -> unI64 65346 + -- FULLWIDTH LATIN CAPITAL LETTER C + '\xff23'# -> unI64 65347 + -- FULLWIDTH LATIN CAPITAL LETTER D + '\xff24'# -> unI64 65348 + -- FULLWIDTH LATIN CAPITAL LETTER E + '\xff25'# -> unI64 65349 + -- FULLWIDTH LATIN CAPITAL LETTER F + '\xff26'# -> unI64 65350 + -- FULLWIDTH LATIN CAPITAL LETTER G + '\xff27'# -> unI64 65351 + -- FULLWIDTH LATIN CAPITAL LETTER H + '\xff28'# -> unI64 65352 + -- FULLWIDTH LATIN CAPITAL LETTER I + '\xff29'# -> unI64 65353 + -- FULLWIDTH LATIN CAPITAL LETTER J + '\xff2a'# -> unI64 65354 + -- FULLWIDTH LATIN CAPITAL LETTER K + '\xff2b'# -> unI64 65355 + -- FULLWIDTH LATIN CAPITAL LETTER L + '\xff2c'# -> unI64 65356 + -- FULLWIDTH LATIN CAPITAL LETTER M + '\xff2d'# -> unI64 65357 + -- FULLWIDTH LATIN CAPITAL LETTER N + '\xff2e'# -> unI64 65358 + -- FULLWIDTH LATIN CAPITAL LETTER O + '\xff2f'# -> unI64 65359 + -- FULLWIDTH LATIN CAPITAL LETTER P + '\xff30'# -> unI64 65360 + -- FULLWIDTH LATIN CAPITAL LETTER Q + '\xff31'# -> unI64 65361 + -- FULLWIDTH LATIN CAPITAL LETTER R + '\xff32'# -> unI64 65362 + -- FULLWIDTH LATIN CAPITAL LETTER S + '\xff33'# -> unI64 65363 + -- FULLWIDTH LATIN CAPITAL LETTER T + '\xff34'# -> unI64 65364 + -- FULLWIDTH LATIN CAPITAL LETTER U + '\xff35'# -> unI64 65365 + -- FULLWIDTH LATIN CAPITAL LETTER V + '\xff36'# -> unI64 65366 + -- FULLWIDTH LATIN CAPITAL LETTER W + '\xff37'# -> unI64 65367 + -- FULLWIDTH LATIN CAPITAL LETTER X + '\xff38'# -> unI64 65368 + -- FULLWIDTH LATIN CAPITAL LETTER Y + '\xff39'# -> unI64 65369 + -- FULLWIDTH LATIN CAPITAL LETTER Z + '\xff3a'# -> unI64 65370 + -- DESERET CAPITAL LETTER LONG I + '\x10400'# -> unI64 66600 + -- DESERET CAPITAL LETTER LONG E + '\x10401'# -> unI64 66601 + -- DESERET CAPITAL LETTER LONG A + '\x10402'# -> unI64 66602 + -- DESERET CAPITAL LETTER LONG AH + '\x10403'# -> unI64 66603 + -- DESERET CAPITAL LETTER LONG O + '\x10404'# -> unI64 66604 + -- DESERET CAPITAL LETTER LONG OO + '\x10405'# -> unI64 66605 + -- DESERET CAPITAL LETTER SHORT I + '\x10406'# -> unI64 66606 + -- DESERET CAPITAL LETTER SHORT E + '\x10407'# -> unI64 66607 + -- DESERET CAPITAL LETTER SHORT A + '\x10408'# -> unI64 66608 + -- DESERET CAPITAL LETTER SHORT AH + '\x10409'# -> unI64 66609 + -- DESERET CAPITAL LETTER SHORT O + '\x1040a'# -> unI64 66610 + -- DESERET CAPITAL LETTER SHORT OO + '\x1040b'# -> unI64 66611 + -- DESERET CAPITAL LETTER AY + '\x1040c'# -> unI64 66612 + -- DESERET CAPITAL LETTER OW + '\x1040d'# -> unI64 66613 + -- DESERET CAPITAL LETTER WU + '\x1040e'# -> unI64 66614 + -- DESERET CAPITAL LETTER YEE + '\x1040f'# -> unI64 66615 + -- DESERET CAPITAL LETTER H + '\x10410'# -> unI64 66616 + -- DESERET CAPITAL LETTER PEE + '\x10411'# -> unI64 66617 + -- DESERET CAPITAL LETTER BEE + '\x10412'# -> unI64 66618 + -- DESERET CAPITAL LETTER TEE + '\x10413'# -> unI64 66619 + -- DESERET CAPITAL LETTER DEE + '\x10414'# -> unI64 66620 + -- DESERET CAPITAL LETTER CHEE + '\x10415'# -> unI64 66621 + -- DESERET CAPITAL LETTER JEE + '\x10416'# -> unI64 66622 + -- DESERET CAPITAL LETTER KAY + '\x10417'# -> unI64 66623 + -- DESERET CAPITAL LETTER GAY + '\x10418'# -> unI64 66624 + -- DESERET CAPITAL LETTER EF + '\x10419'# -> unI64 66625 + -- DESERET CAPITAL LETTER VEE + '\x1041a'# -> unI64 66626 + -- DESERET CAPITAL LETTER ETH + '\x1041b'# -> unI64 66627 + -- DESERET CAPITAL LETTER THEE + '\x1041c'# -> unI64 66628 + -- DESERET CAPITAL LETTER ES + '\x1041d'# -> unI64 66629 + -- DESERET CAPITAL LETTER ZEE + '\x1041e'# -> unI64 66630 + -- DESERET CAPITAL LETTER ESH + '\x1041f'# -> unI64 66631 + -- DESERET CAPITAL LETTER ZHEE + '\x10420'# -> unI64 66632 + -- DESERET CAPITAL LETTER ER + '\x10421'# -> unI64 66633 + -- DESERET CAPITAL LETTER EL + '\x10422'# -> unI64 66634 + -- DESERET CAPITAL LETTER EM + '\x10423'# -> unI64 66635 + -- DESERET CAPITAL LETTER EN + '\x10424'# -> unI64 66636 + -- DESERET CAPITAL LETTER ENG + '\x10425'# -> unI64 66637 + -- DESERET CAPITAL LETTER OI + '\x10426'# -> unI64 66638 + -- DESERET CAPITAL LETTER EW + '\x10427'# -> unI64 66639 + -- OSAGE CAPITAL LETTER A + '\x104b0'# -> unI64 66776 + -- OSAGE CAPITAL LETTER AI + '\x104b1'# -> unI64 66777 + -- OSAGE CAPITAL LETTER AIN + '\x104b2'# -> unI64 66778 + -- OSAGE CAPITAL LETTER AH + '\x104b3'# -> unI64 66779 + -- OSAGE CAPITAL LETTER BRA + '\x104b4'# -> unI64 66780 + -- OSAGE CAPITAL LETTER CHA + '\x104b5'# -> unI64 66781 + -- OSAGE CAPITAL LETTER EHCHA + '\x104b6'# -> unI64 66782 + -- OSAGE CAPITAL LETTER E + '\x104b7'# -> unI64 66783 + -- OSAGE CAPITAL LETTER EIN + '\x104b8'# -> unI64 66784 + -- OSAGE CAPITAL LETTER HA + '\x104b9'# -> unI64 66785 + -- OSAGE CAPITAL LETTER HYA + '\x104ba'# -> unI64 66786 + -- OSAGE CAPITAL LETTER I + '\x104bb'# -> unI64 66787 + -- OSAGE CAPITAL LETTER KA + '\x104bc'# -> unI64 66788 + -- OSAGE CAPITAL LETTER EHKA + '\x104bd'# -> unI64 66789 + -- OSAGE CAPITAL LETTER KYA + '\x104be'# -> unI64 66790 + -- OSAGE CAPITAL LETTER LA + '\x104bf'# -> unI64 66791 + -- OSAGE CAPITAL LETTER MA + '\x104c0'# -> unI64 66792 + -- OSAGE CAPITAL LETTER NA + '\x104c1'# -> unI64 66793 + -- OSAGE CAPITAL LETTER O + '\x104c2'# -> unI64 66794 + -- OSAGE CAPITAL LETTER OIN + '\x104c3'# -> unI64 66795 + -- OSAGE CAPITAL LETTER PA + '\x104c4'# -> unI64 66796 + -- OSAGE CAPITAL LETTER EHPA + '\x104c5'# -> unI64 66797 + -- OSAGE CAPITAL LETTER SA + '\x104c6'# -> unI64 66798 + -- OSAGE CAPITAL LETTER SHA + '\x104c7'# -> unI64 66799 + -- OSAGE CAPITAL LETTER TA + '\x104c8'# -> unI64 66800 + -- OSAGE CAPITAL LETTER EHTA + '\x104c9'# -> unI64 66801 + -- OSAGE CAPITAL LETTER TSA + '\x104ca'# -> unI64 66802 + -- OSAGE CAPITAL LETTER EHTSA + '\x104cb'# -> unI64 66803 + -- OSAGE CAPITAL LETTER TSHA + '\x104cc'# -> unI64 66804 + -- OSAGE CAPITAL LETTER DHA + '\x104cd'# -> unI64 66805 + -- OSAGE CAPITAL LETTER U + '\x104ce'# -> unI64 66806 + -- OSAGE CAPITAL LETTER WA + '\x104cf'# -> unI64 66807 + -- OSAGE CAPITAL LETTER KHA + '\x104d0'# -> unI64 66808 + -- OSAGE CAPITAL LETTER GHA + '\x104d1'# -> unI64 66809 + -- OSAGE CAPITAL LETTER ZA + '\x104d2'# -> unI64 66810 + -- OSAGE CAPITAL LETTER ZHA + '\x104d3'# -> unI64 66811 + -- VITHKUQI CAPITAL LETTER A + '\x10570'# -> unI64 66967 + -- VITHKUQI CAPITAL LETTER BBE + '\x10571'# -> unI64 66968 + -- VITHKUQI CAPITAL LETTER BE + '\x10572'# -> unI64 66969 + -- VITHKUQI CAPITAL LETTER CE + '\x10573'# -> unI64 66970 + -- VITHKUQI CAPITAL LETTER CHE + '\x10574'# -> unI64 66971 + -- VITHKUQI CAPITAL LETTER DE + '\x10575'# -> unI64 66972 + -- VITHKUQI CAPITAL LETTER DHE + '\x10576'# -> unI64 66973 + -- VITHKUQI CAPITAL LETTER EI + '\x10577'# -> unI64 66974 + -- VITHKUQI CAPITAL LETTER E + '\x10578'# -> unI64 66975 + -- VITHKUQI CAPITAL LETTER FE + '\x10579'# -> unI64 66976 + -- VITHKUQI CAPITAL LETTER GA + '\x1057a'# -> unI64 66977 + -- VITHKUQI CAPITAL LETTER HA + '\x1057c'# -> unI64 66979 + -- VITHKUQI CAPITAL LETTER HHA + '\x1057d'# -> unI64 66980 + -- VITHKUQI CAPITAL LETTER I + '\x1057e'# -> unI64 66981 + -- VITHKUQI CAPITAL LETTER IJE + '\x1057f'# -> unI64 66982 + -- VITHKUQI CAPITAL LETTER JE + '\x10580'# -> unI64 66983 + -- VITHKUQI CAPITAL LETTER KA + '\x10581'# -> unI64 66984 + -- VITHKUQI CAPITAL LETTER LA + '\x10582'# -> unI64 66985 + -- VITHKUQI CAPITAL LETTER LLA + '\x10583'# -> unI64 66986 + -- VITHKUQI CAPITAL LETTER ME + '\x10584'# -> unI64 66987 + -- VITHKUQI CAPITAL LETTER NE + '\x10585'# -> unI64 66988 + -- VITHKUQI CAPITAL LETTER NJE + '\x10586'# -> unI64 66989 + -- VITHKUQI CAPITAL LETTER O + '\x10587'# -> unI64 66990 + -- VITHKUQI CAPITAL LETTER PE + '\x10588'# -> unI64 66991 + -- VITHKUQI CAPITAL LETTER QA + '\x10589'# -> unI64 66992 + -- VITHKUQI CAPITAL LETTER RE + '\x1058a'# -> unI64 66993 + -- VITHKUQI CAPITAL LETTER SE + '\x1058c'# -> unI64 66995 + -- VITHKUQI CAPITAL LETTER SHE + '\x1058d'# -> unI64 66996 + -- VITHKUQI CAPITAL LETTER TE + '\x1058e'# -> unI64 66997 + -- VITHKUQI CAPITAL LETTER THE + '\x1058f'# -> unI64 66998 + -- VITHKUQI CAPITAL LETTER U + '\x10590'# -> unI64 66999 + -- VITHKUQI CAPITAL LETTER VE + '\x10591'# -> unI64 67000 + -- VITHKUQI CAPITAL LETTER XE + '\x10592'# -> unI64 67001 + -- VITHKUQI CAPITAL LETTER Y + '\x10594'# -> unI64 67003 + -- VITHKUQI CAPITAL LETTER ZE + '\x10595'# -> unI64 67004 + -- OLD HUNGARIAN CAPITAL LETTER A + '\x10c80'# -> unI64 68800 + -- OLD HUNGARIAN CAPITAL LETTER AA + '\x10c81'# -> unI64 68801 + -- OLD HUNGARIAN CAPITAL LETTER EB + '\x10c82'# -> unI64 68802 + -- OLD HUNGARIAN CAPITAL LETTER AMB + '\x10c83'# -> unI64 68803 + -- OLD HUNGARIAN CAPITAL LETTER EC + '\x10c84'# -> unI64 68804 + -- OLD HUNGARIAN CAPITAL LETTER ENC + '\x10c85'# -> unI64 68805 + -- OLD HUNGARIAN CAPITAL LETTER ECS + '\x10c86'# -> unI64 68806 + -- OLD HUNGARIAN CAPITAL LETTER ED + '\x10c87'# -> unI64 68807 + -- OLD HUNGARIAN CAPITAL LETTER AND + '\x10c88'# -> unI64 68808 + -- OLD HUNGARIAN CAPITAL LETTER E + '\x10c89'# -> unI64 68809 + -- OLD HUNGARIAN CAPITAL LETTER CLOSE E + '\x10c8a'# -> unI64 68810 + -- OLD HUNGARIAN CAPITAL LETTER EE + '\x10c8b'# -> unI64 68811 + -- OLD HUNGARIAN CAPITAL LETTER EF + '\x10c8c'# -> unI64 68812 + -- OLD HUNGARIAN CAPITAL LETTER EG + '\x10c8d'# -> unI64 68813 + -- OLD HUNGARIAN CAPITAL LETTER EGY + '\x10c8e'# -> unI64 68814 + -- OLD HUNGARIAN CAPITAL LETTER EH + '\x10c8f'# -> unI64 68815 + -- OLD HUNGARIAN CAPITAL LETTER I + '\x10c90'# -> unI64 68816 + -- OLD HUNGARIAN CAPITAL LETTER II + '\x10c91'# -> unI64 68817 + -- OLD HUNGARIAN CAPITAL LETTER EJ + '\x10c92'# -> unI64 68818 + -- OLD HUNGARIAN CAPITAL LETTER EK + '\x10c93'# -> unI64 68819 + -- OLD HUNGARIAN CAPITAL LETTER AK + '\x10c94'# -> unI64 68820 + -- OLD HUNGARIAN CAPITAL LETTER UNK + '\x10c95'# -> unI64 68821 + -- OLD HUNGARIAN CAPITAL LETTER EL + '\x10c96'# -> unI64 68822 + -- OLD HUNGARIAN CAPITAL LETTER ELY + '\x10c97'# -> unI64 68823 + -- OLD HUNGARIAN CAPITAL LETTER EM + '\x10c98'# -> unI64 68824 + -- OLD HUNGARIAN CAPITAL LETTER EN + '\x10c99'# -> unI64 68825 + -- OLD HUNGARIAN CAPITAL LETTER ENY + '\x10c9a'# -> unI64 68826 + -- OLD HUNGARIAN CAPITAL LETTER O + '\x10c9b'# -> unI64 68827 + -- OLD HUNGARIAN CAPITAL LETTER OO + '\x10c9c'# -> unI64 68828 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE + '\x10c9d'# -> unI64 68829 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE + '\x10c9e'# -> unI64 68830 + -- OLD HUNGARIAN CAPITAL LETTER OEE + '\x10c9f'# -> unI64 68831 + -- OLD HUNGARIAN CAPITAL LETTER EP + '\x10ca0'# -> unI64 68832 + -- OLD HUNGARIAN CAPITAL LETTER EMP + '\x10ca1'# -> unI64 68833 + -- OLD HUNGARIAN CAPITAL LETTER ER + '\x10ca2'# -> unI64 68834 + -- OLD HUNGARIAN CAPITAL LETTER SHORT ER + '\x10ca3'# -> unI64 68835 + -- OLD HUNGARIAN CAPITAL LETTER ES + '\x10ca4'# -> unI64 68836 + -- OLD HUNGARIAN CAPITAL LETTER ESZ + '\x10ca5'# -> unI64 68837 + -- OLD HUNGARIAN CAPITAL LETTER ET + '\x10ca6'# -> unI64 68838 + -- OLD HUNGARIAN CAPITAL LETTER ENT + '\x10ca7'# -> unI64 68839 + -- OLD HUNGARIAN CAPITAL LETTER ETY + '\x10ca8'# -> unI64 68840 + -- OLD HUNGARIAN CAPITAL LETTER ECH + '\x10ca9'# -> unI64 68841 + -- OLD HUNGARIAN CAPITAL LETTER U + '\x10caa'# -> unI64 68842 + -- OLD HUNGARIAN CAPITAL LETTER UU + '\x10cab'# -> unI64 68843 + -- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE + '\x10cac'# -> unI64 68844 + -- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE + '\x10cad'# -> unI64 68845 + -- OLD HUNGARIAN CAPITAL LETTER EV + '\x10cae'# -> unI64 68846 + -- OLD HUNGARIAN CAPITAL LETTER EZ + '\x10caf'# -> unI64 68847 + -- OLD HUNGARIAN CAPITAL LETTER EZS + '\x10cb0'# -> unI64 68848 + -- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN + '\x10cb1'# -> unI64 68849 + -- OLD HUNGARIAN CAPITAL LETTER US + '\x10cb2'# -> unI64 68850 + -- GARAY CAPITAL LETTER A + '\x10d50'# -> unI64 68976 + -- GARAY CAPITAL LETTER CA + '\x10d51'# -> unI64 68977 + -- GARAY CAPITAL LETTER MA + '\x10d52'# -> unI64 68978 + -- GARAY CAPITAL LETTER KA + '\x10d53'# -> unI64 68979 + -- GARAY CAPITAL LETTER BA + '\x10d54'# -> unI64 68980 + -- GARAY CAPITAL LETTER JA + '\x10d55'# -> unI64 68981 + -- GARAY CAPITAL LETTER SA + '\x10d56'# -> unI64 68982 + -- GARAY CAPITAL LETTER WA + '\x10d57'# -> unI64 68983 + -- GARAY CAPITAL LETTER LA + '\x10d58'# -> unI64 68984 + -- GARAY CAPITAL LETTER GA + '\x10d59'# -> unI64 68985 + -- GARAY CAPITAL LETTER DA + '\x10d5a'# -> unI64 68986 + -- GARAY CAPITAL LETTER XA + '\x10d5b'# -> unI64 68987 + -- GARAY CAPITAL LETTER YA + '\x10d5c'# -> unI64 68988 + -- GARAY CAPITAL LETTER TA + '\x10d5d'# -> unI64 68989 + -- GARAY CAPITAL LETTER RA + '\x10d5e'# -> unI64 68990 + -- GARAY CAPITAL LETTER NYA + '\x10d5f'# -> unI64 68991 + -- GARAY CAPITAL LETTER FA + '\x10d60'# -> unI64 68992 + -- GARAY CAPITAL LETTER NA + '\x10d61'# -> unI64 68993 + -- GARAY CAPITAL LETTER PA + '\x10d62'# -> unI64 68994 + -- GARAY CAPITAL LETTER HA + '\x10d63'# -> unI64 68995 + -- GARAY CAPITAL LETTER OLD KA + '\x10d64'# -> unI64 68996 + -- GARAY CAPITAL LETTER OLD NA + '\x10d65'# -> unI64 68997 + -- WARANG CITI CAPITAL LETTER NGAA + '\x118a0'# -> unI64 71872 + -- WARANG CITI CAPITAL LETTER A + '\x118a1'# -> unI64 71873 + -- WARANG CITI CAPITAL LETTER WI + '\x118a2'# -> unI64 71874 + -- WARANG CITI CAPITAL LETTER YU + '\x118a3'# -> unI64 71875 + -- WARANG CITI CAPITAL LETTER YA + '\x118a4'# -> unI64 71876 + -- WARANG CITI CAPITAL LETTER YO + '\x118a5'# -> unI64 71877 + -- WARANG CITI CAPITAL LETTER II + '\x118a6'# -> unI64 71878 + -- WARANG CITI CAPITAL LETTER UU + '\x118a7'# -> unI64 71879 + -- WARANG CITI CAPITAL LETTER E + '\x118a8'# -> unI64 71880 + -- WARANG CITI CAPITAL LETTER O + '\x118a9'# -> unI64 71881 + -- WARANG CITI CAPITAL LETTER ANG + '\x118aa'# -> unI64 71882 + -- WARANG CITI CAPITAL LETTER GA + '\x118ab'# -> unI64 71883 + -- WARANG CITI CAPITAL LETTER KO + '\x118ac'# -> unI64 71884 + -- WARANG CITI CAPITAL LETTER ENY + '\x118ad'# -> unI64 71885 + -- WARANG CITI CAPITAL LETTER YUJ + '\x118ae'# -> unI64 71886 + -- WARANG CITI CAPITAL LETTER UC + '\x118af'# -> unI64 71887 + -- WARANG CITI CAPITAL LETTER ENN + '\x118b0'# -> unI64 71888 + -- WARANG CITI CAPITAL LETTER ODD + '\x118b1'# -> unI64 71889 + -- WARANG CITI CAPITAL LETTER TTE + '\x118b2'# -> unI64 71890 + -- WARANG CITI CAPITAL LETTER NUNG + '\x118b3'# -> unI64 71891 + -- WARANG CITI CAPITAL LETTER DA + '\x118b4'# -> unI64 71892 + -- WARANG CITI CAPITAL LETTER AT + '\x118b5'# -> unI64 71893 + -- WARANG CITI CAPITAL LETTER AM + '\x118b6'# -> unI64 71894 + -- WARANG CITI CAPITAL LETTER BU + '\x118b7'# -> unI64 71895 + -- WARANG CITI CAPITAL LETTER PU + '\x118b8'# -> unI64 71896 + -- WARANG CITI CAPITAL LETTER HIYO + '\x118b9'# -> unI64 71897 + -- WARANG CITI CAPITAL LETTER HOLO + '\x118ba'# -> unI64 71898 + -- WARANG CITI CAPITAL LETTER HORR + '\x118bb'# -> unI64 71899 + -- WARANG CITI CAPITAL LETTER HAR + '\x118bc'# -> unI64 71900 + -- WARANG CITI CAPITAL LETTER SSUU + '\x118bd'# -> unI64 71901 + -- WARANG CITI CAPITAL LETTER SII + '\x118be'# -> unI64 71902 + -- WARANG CITI CAPITAL LETTER VIYO + '\x118bf'# -> unI64 71903 + -- MEDEFAIDRIN CAPITAL LETTER M + '\x16e40'# -> unI64 93792 + -- MEDEFAIDRIN CAPITAL LETTER S + '\x16e41'# -> unI64 93793 + -- MEDEFAIDRIN CAPITAL LETTER V + '\x16e42'# -> unI64 93794 + -- MEDEFAIDRIN CAPITAL LETTER W + '\x16e43'# -> unI64 93795 + -- MEDEFAIDRIN CAPITAL LETTER ATIU + '\x16e44'# -> unI64 93796 + -- MEDEFAIDRIN CAPITAL LETTER Z + '\x16e45'# -> unI64 93797 + -- MEDEFAIDRIN CAPITAL LETTER KP + '\x16e46'# -> unI64 93798 + -- MEDEFAIDRIN CAPITAL LETTER P + '\x16e47'# -> unI64 93799 + -- MEDEFAIDRIN CAPITAL LETTER T + '\x16e48'# -> unI64 93800 + -- MEDEFAIDRIN CAPITAL LETTER G + '\x16e49'# -> unI64 93801 + -- MEDEFAIDRIN CAPITAL LETTER F + '\x16e4a'# -> unI64 93802 + -- MEDEFAIDRIN CAPITAL LETTER I + '\x16e4b'# -> unI64 93803 + -- MEDEFAIDRIN CAPITAL LETTER K + '\x16e4c'# -> unI64 93804 + -- MEDEFAIDRIN CAPITAL LETTER A + '\x16e4d'# -> unI64 93805 + -- MEDEFAIDRIN CAPITAL LETTER J + '\x16e4e'# -> unI64 93806 + -- MEDEFAIDRIN CAPITAL LETTER E + '\x16e4f'# -> unI64 93807 + -- MEDEFAIDRIN CAPITAL LETTER B + '\x16e50'# -> unI64 93808 + -- MEDEFAIDRIN CAPITAL LETTER C + '\x16e51'# -> unI64 93809 + -- MEDEFAIDRIN CAPITAL LETTER U + '\x16e52'# -> unI64 93810 + -- MEDEFAIDRIN CAPITAL LETTER YU + '\x16e53'# -> unI64 93811 + -- MEDEFAIDRIN CAPITAL LETTER L + '\x16e54'# -> unI64 93812 + -- MEDEFAIDRIN CAPITAL LETTER Q + '\x16e55'# -> unI64 93813 + -- MEDEFAIDRIN CAPITAL LETTER HP + '\x16e56'# -> unI64 93814 + -- MEDEFAIDRIN CAPITAL LETTER NY + '\x16e57'# -> unI64 93815 + -- MEDEFAIDRIN CAPITAL LETTER X + '\x16e58'# -> unI64 93816 + -- MEDEFAIDRIN CAPITAL LETTER D + '\x16e59'# -> unI64 93817 + -- MEDEFAIDRIN CAPITAL LETTER OE + '\x16e5a'# -> unI64 93818 + -- MEDEFAIDRIN CAPITAL LETTER N + '\x16e5b'# -> unI64 93819 + -- MEDEFAIDRIN CAPITAL LETTER R + '\x16e5c'# -> unI64 93820 + -- MEDEFAIDRIN CAPITAL LETTER O + '\x16e5d'# -> unI64 93821 + -- MEDEFAIDRIN CAPITAL LETTER AI + '\x16e5e'# -> unI64 93822 + -- MEDEFAIDRIN CAPITAL LETTER Y + '\x16e5f'# -> unI64 93823 + -- BERIA ERFE CAPITAL LETTER ARKAB + '\x16ea0'# -> unI64 93883 + -- BERIA ERFE CAPITAL LETTER BASIGNA + '\x16ea1'# -> unI64 93884 + -- BERIA ERFE CAPITAL LETTER DARBAI + '\x16ea2'# -> unI64 93885 + -- BERIA ERFE CAPITAL LETTER EH + '\x16ea3'# -> unI64 93886 + -- BERIA ERFE CAPITAL LETTER FITKO + '\x16ea4'# -> unI64 93887 + -- BERIA ERFE CAPITAL LETTER GOWAY + '\x16ea5'# -> unI64 93888 + -- BERIA ERFE CAPITAL LETTER HIRDEABO + '\x16ea6'# -> unI64 93889 + -- BERIA ERFE CAPITAL LETTER I + '\x16ea7'# -> unI64 93890 + -- BERIA ERFE CAPITAL LETTER DJAI + '\x16ea8'# -> unI64 93891 + -- BERIA ERFE CAPITAL LETTER KOBO + '\x16ea9'# -> unI64 93892 + -- BERIA ERFE CAPITAL LETTER LAKKO + '\x16eaa'# -> unI64 93893 + -- BERIA ERFE CAPITAL LETTER MERI + '\x16eab'# -> unI64 93894 + -- BERIA ERFE CAPITAL LETTER NINI + '\x16eac'# -> unI64 93895 + -- BERIA ERFE CAPITAL LETTER GNA + '\x16ead'# -> unI64 93896 + -- BERIA ERFE CAPITAL LETTER NGAY + '\x16eae'# -> unI64 93897 + -- BERIA ERFE CAPITAL LETTER OI + '\x16eaf'# -> unI64 93898 + -- BERIA ERFE CAPITAL LETTER PI + '\x16eb0'# -> unI64 93899 + -- BERIA ERFE CAPITAL LETTER ERIGO + '\x16eb1'# -> unI64 93900 + -- BERIA ERFE CAPITAL LETTER ERIGO TAMURA + '\x16eb2'# -> unI64 93901 + -- BERIA ERFE CAPITAL LETTER SERI + '\x16eb3'# -> unI64 93902 + -- BERIA ERFE CAPITAL LETTER SHEP + '\x16eb4'# -> unI64 93903 + -- BERIA ERFE CAPITAL LETTER TATASOUE + '\x16eb5'# -> unI64 93904 + -- BERIA ERFE CAPITAL LETTER UI + '\x16eb6'# -> unI64 93905 + -- BERIA ERFE CAPITAL LETTER WASSE + '\x16eb7'# -> unI64 93906 + -- BERIA ERFE CAPITAL LETTER AY + '\x16eb8'# -> unI64 93907 + -- ADLAM CAPITAL LETTER ALIF + '\x1e900'# -> unI64 125218 + -- ADLAM CAPITAL LETTER DAALI + '\x1e901'# -> unI64 125219 + -- ADLAM CAPITAL LETTER LAAM + '\x1e902'# -> unI64 125220 + -- ADLAM CAPITAL LETTER MIIM + '\x1e903'# -> unI64 125221 + -- ADLAM CAPITAL LETTER BA + '\x1e904'# -> unI64 125222 + -- ADLAM CAPITAL LETTER SINNYIIYHE + '\x1e905'# -> unI64 125223 + -- ADLAM CAPITAL LETTER PE + '\x1e906'# -> unI64 125224 + -- ADLAM CAPITAL LETTER BHE + '\x1e907'# -> unI64 125225 + -- ADLAM CAPITAL LETTER RA + '\x1e908'# -> unI64 125226 + -- ADLAM CAPITAL LETTER E + '\x1e909'# -> unI64 125227 + -- ADLAM CAPITAL LETTER FA + '\x1e90a'# -> unI64 125228 + -- ADLAM CAPITAL LETTER I + '\x1e90b'# -> unI64 125229 + -- ADLAM CAPITAL LETTER O + '\x1e90c'# -> unI64 125230 + -- ADLAM CAPITAL LETTER DHA + '\x1e90d'# -> unI64 125231 + -- ADLAM CAPITAL LETTER YHE + '\x1e90e'# -> unI64 125232 + -- ADLAM CAPITAL LETTER WAW + '\x1e90f'# -> unI64 125233 + -- ADLAM CAPITAL LETTER NUN + '\x1e910'# -> unI64 125234 + -- ADLAM CAPITAL LETTER KAF + '\x1e911'# -> unI64 125235 + -- ADLAM CAPITAL LETTER YA + '\x1e912'# -> unI64 125236 + -- ADLAM CAPITAL LETTER U + '\x1e913'# -> unI64 125237 + -- ADLAM CAPITAL LETTER JIIM + '\x1e914'# -> unI64 125238 + -- ADLAM CAPITAL LETTER CHI + '\x1e915'# -> unI64 125239 + -- ADLAM CAPITAL LETTER HA + '\x1e916'# -> unI64 125240 + -- ADLAM CAPITAL LETTER QAAF + '\x1e917'# -> unI64 125241 + -- ADLAM CAPITAL LETTER GA + '\x1e918'# -> unI64 125242 + -- ADLAM CAPITAL LETTER NYA + '\x1e919'# -> unI64 125243 + -- ADLAM CAPITAL LETTER TU + '\x1e91a'# -> unI64 125244 + -- ADLAM CAPITAL LETTER NHA + '\x1e91b'# -> unI64 125245 + -- ADLAM CAPITAL LETTER VA + '\x1e91c'# -> unI64 125246 + -- ADLAM CAPITAL LETTER KHA + '\x1e91d'# -> unI64 125247 + -- ADLAM CAPITAL LETTER GBE + '\x1e91e'# -> unI64 125248 + -- ADLAM CAPITAL LETTER ZAL + '\x1e91f'# -> unI64 125249 + -- ADLAM CAPITAL LETTER KPO + '\x1e920'# -> unI64 125250 + -- ADLAM CAPITAL LETTER SHA + '\x1e921'# -> unI64 125251 + _ -> unI64 0
src/Data/Text/Internal/IO.hs view
@@ -1,306 +1,308 @@-{-# LANGUAGE BangPatterns, RecordWildCards #-}-{-# LANGUAGE MagicHash #-}--- |--- Module : Data.Text.Internal.IO--- Copyright : (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Simon Marlow--- License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ Low-level support for text I\/O.--module Data.Text.Internal.IO- (- hGetLineWith- , readChunk- , hPutStream- , hPutStr- , hPutStrLn- ) where--import qualified Control.Exception as E-import qualified Data.ByteString as B-import Data.ByteString.Builder (hPutBuilder, charUtf8)-import Data.IORef (readIORef, writeIORef)-import Data.Text (Text)-import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder)-import Data.Text.Internal.Fusion (stream, streamLn, unstream)-import Data.Text.Internal.Fusion.Types (Step(..), Stream(..))-import Data.Text.Internal.Fusion.Size (exactSize, maxSize)-import Data.Text.Unsafe (inlinePerformIO)-import Foreign.Storable (peekElemOff)-import GHC.Exts (reallyUnsafePtrEquality#, isTrue#)-import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer,- bufferAdjustL, bufferElems, charSize, emptyBuffer,- isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer,- writeCharBuf)-import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_,- wantWritableHandle)-import GHC.IO.Handle.Text (commitBuffer')-import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..))-import System.IO (Handle, hPutChar, utf8)-import System.IO.Error (isEOFError)-import qualified Data.Text as T---- | Read a single line of input from a handle, constructing a list of--- decoded chunks as we go. When we're done, transform them into the--- destination type.-hGetLineWith :: ([Text] -> t) -> Handle -> IO t-hGetLineWith f h = wantReadableHandle_ "hGetLine" h go- where- go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh []--hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text]-hGetLineLoop hh@Handle__{..} = go where- go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do- let findEOL raw r | r == w = return (False, w)- | otherwise = do- (c,r') <- readCharBuf raw r- if c == '\n'- then return (True, r)- else findEOL raw r'- (eol, off) <- findEOL raw0 r0- (t,r') <- if haInputNL == CRLF- then unpack_nl raw0 r0 off- else do t <- unpack raw0 r0 off- return (t,off)- if eol- then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf)- return $ reverse (t:ts)- else do- let buf1 = bufferAdjustL r' buf- maybe_buf <- maybeFillReadBuffer hh buf1- case maybe_buf of- -- Nothing indicates we caught an EOF, and we may have a- -- partial line to return.- Nothing -> do- -- we reached EOF. There might be a lone \r left- -- in the buffer, so check for that and- -- append it to the line if necessary.- let pre | isEmptyBuffer buf1 = T.empty- | otherwise = T.singleton '\r'- writeIORef haCharBuffer buf1{ bufL=0, bufR=0 }- let str = reverse . filter (not . T.null) $ pre:t:ts- if null str- then ioe_EOF- else return str- Just new_buf -> go (t:ts) new_buf---- This function is lifted almost verbatim from GHC.IO.Handle.Text.-maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer)-maybeFillReadBuffer handle_ buf- = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e ->- if isEOFError e- then return Nothing- else ioError e--unpack :: RawCharBuffer -> Int -> Int -> IO Text-unpack !buf !r !w- | charSize /= 4 = sizeError "unpack"- | r >= w = return T.empty- | otherwise = withRawBuffer buf go- where- go pbuf = return $! unstream (Stream next r (exactSize (w-r)))- where- next !i | i >= w = Done- | otherwise = Yield (ix i) (i+1)- ix i = inlinePerformIO $ peekElemOff pbuf i--unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int)-unpack_nl !buf !r !w- | charSize /= 4 = sizeError "unpack_nl"- | r >= w = return (T.empty, 0)- | otherwise = withRawBuffer buf $ go- where- go pbuf = do- let !t = unstream (Stream next r (maxSize (w-r)))- w' = w - 1- return $ if ix w' == '\r'- then (t,w')- else (t,w)- where- next !i | i >= w = Done- | c == '\r' = let i' = i + 1- in if i' < w- then if ix i' == '\n'- then Yield '\n' (i+2)- else Yield '\n' i'- else Done- | otherwise = Yield c (i+1)- where c = ix i- ix i = inlinePerformIO $ peekElemOff pbuf i---- This function is completely lifted from GHC.IO.Handle.Text.-getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer-getSomeCharacters handle_@Handle__{..} buf@Buffer{..} =- case bufferElems buf of- -- buffer empty: read some more- 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf-- -- if the buffer has a single '\r' in it and we're doing newline- -- translation: read some more- 1 | haInputNL == CRLF -> do- (c,_) <- readCharBuf bufRaw bufL- if c == '\r'- then do -- shuffle the '\r' to the beginning. This is only safe- -- if we're about to call readTextDevice, otherwise it- -- would mess up flushCharBuffer.- -- See [note Buffer Flushing], GHC.IO.Handle.Types- _ <- writeCharBuf bufRaw 0 '\r'- let buf' = buf{ bufL=0, bufR=1 }- readTextDevice handle_ buf'- else do- return buf-- -- buffer has some chars in it already: just return it- _otherwise -> {-# SCC "otherwise" #-} return buf---- | Read a single chunk of strict text from a buffer. Used by both--- the strict and lazy implementations of hGetContents.-readChunk :: Handle__ -> CharBuffer -> IO Text-readChunk hh@Handle__{..} buf = do- buf'@Buffer{..} <- getSomeCharacters hh buf- (t,r) <- if haInputNL == CRLF- then unpack_nl bufRaw bufL bufR- else do t <- unpack bufRaw bufL bufR- return (t,bufR)- writeIORef haCharBuffer (bufferAdjustL r buf')- return t---- | Print a @Stream Char@.-hPutStream :: Handle -> Stream Char -> IO ()-hPutStream h str = hPutStreamOrUtf8 h str Nothing---- | Write a string to a handle.-hPutStr :: Handle -> Text -> IO ()-hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8)- where- putUtf8 = B.hPutStr h (encodeUtf8 t)---- | Write a string to a handle, followed by a newline.-hPutStrLn :: Handle -> Text -> IO ()-hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8)- where- -- Not using B.hPutStrLn because it's not necessarily atomic:- -- https://github.com/haskell/bytestring/issues/200- putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n')---- | 'hPutStream' with an optional special case when the output encoding is--- UTF-8 and without newline conversion.-hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO ()--- This function is modified from GHC.IO.Handle.Text.-hPutStreamOrUtf8 h str mPutUtf8 = do- (buffer_mode, nl, isUtf8) <-- wantWritableHandle "hPutStr" h $ \h_ -> do- bmode <- getSpareBuffer h_- return (bmode, haOutputNL h_, eqUTF8 h_)- case buffer_mode of- _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8- (NoBuffering, _) -> hPutChars h str- (LineBuffering, buf) -> writeLines h nl buf str- (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str-- where- -- If the encoding is UTF-8, it's most likely pointer-equal to- -- 'System.IO.utf8', letting us avoid a String comparison.- -- If it is somehow UTF-8 but not pointer-equal to 'utf8',- -- we will just take a slower branch, but the result is still correct.- eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec-{-# INLINE hPutStreamOrUtf8 #-}--hPutChars :: Handle -> Stream Char -> IO ()-hPutChars h (Stream next0 s0 _len) = loop s0- where- loop !s = case next0 s of- Done -> return ()- Skip s' -> loop s'- Yield x s' -> hPutChar h x >> loop s'---- The following functions are largely lifted from GHC.IO.Handle.Text,--- but adapted to a coinductive stream of data instead of an inductive--- list.------ We have several variations of more or less the same code for--- performance reasons. Splitting the original buffered write--- function into line- and block-oriented versions gave us a 2.1x--- performance improvement. Lifting out the raw/cooked newline--- handling gave a few more percent on top.--writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO ()-writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0- where- outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)- where- inner !s !n =- case next0 s of- Done -> commit n False{-no flush-} True{-release-} >> return ()- Skip s' -> inner s' n- Yield x s'- | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s- | x == '\n' -> do- n' <- if nl == CRLF- then do n1 <- writeCharBuf' raw len n '\r'- writeCharBuf' raw len n1 '\n'- else writeCharBuf' raw len n x- commit n' True{-needs flush-} False >>= outer s'- | otherwise -> writeCharBuf' raw len n x >>= inner s'- commit = commitBuffer h raw len--writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO ()-writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0- where- outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int)- where- inner !s !n =- case next0 s of- Done -> commit n False{-no flush-} True{-release-} >> return ()- Skip s' -> inner s' n- Yield x s'- | isCRLF && x == '\n' && n + 1 < len -> do- n1 <- writeCharBuf' raw len n '\r'- writeCharBuf' raw len n1 '\n' >>= inner s'- | n < len -> writeCharBuf' raw len n x >>= inner s'- | otherwise -> commit n True{-needs flush-} False >>= outer s- commit = commitBuffer h raw len---- | Only modifies the raw buffer and not the buffer attributes-writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int-writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $- writeCharBuf bufRaw n c---- This function is completely lifted from GHC.IO.Handle.Text.-getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer)-getSpareBuffer Handle__{haCharBuffer=ref,- haBuffers=spare_ref,- haBufferMode=mode}- = do- case mode of- NoBuffering -> return (mode, error "no buffer!")- _ -> do- bufs <- readIORef spare_ref- buf <- readIORef ref- case bufs of- BufferListCons b rest -> do- writeIORef spare_ref rest- return ( mode, emptyBuffer b (bufSize buf) WriteBuffer)- BufferListNil -> do- new_buf <- newCharBuffer (bufSize buf) WriteBuffer- return (mode, new_buf)----- This function is modified from GHC.Internal.IO.Handle.Text.-commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool- -> IO CharBuffer-commitBuffer hdl !raw !sz !count flush release =- wantWritableHandle "commitAndReleaseBuffer" hdl $- commitBuffer' raw sz count flush release-{-# INLINE commitBuffer #-}--sizeError :: String -> a-sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"+{-# LANGUAGE BangPatterns, RecordWildCards #-} +{-# LANGUAGE MagicHash #-} +-- | +-- Module : Data.Text.Internal.IO +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Simon Marlow +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- Low-level support for text I\/O. + +module Data.Text.Internal.IO + ( + hGetLineWith + , readChunk + , hPutStream + , hPutStr + , hPutStrLn + ) where + +import qualified Control.Exception as E +import qualified Data.ByteString as B +import Data.ByteString.Builder (hPutBuilder, charUtf8) +import Data.IORef (readIORef, writeIORef) +import Data.Text (Text) +import Data.Text.Encoding (encodeUtf8, encodeUtf8Builder) +import Data.Text.Internal.Fusion (stream, streamLn, unstream) +import Data.Text.Internal.Fusion.Types (Step(..), Stream(..)) +import Data.Text.Internal.Fusion.Size (exactSize, maxSize) +import Data.Text.Unsafe (inlinePerformIO) +import Foreign.Storable (peekElemOff) +import GHC.Exts (reallyUnsafePtrEquality#, isTrue#) +import GHC.IO.Buffer (Buffer(..), BufferState(..), CharBuffer, RawCharBuffer, + bufferAdjustL, bufferElems, charSize, emptyBuffer, + isEmptyBuffer, newCharBuffer, readCharBuf, withRawBuffer, + writeCharBuf) +import GHC.IO.Handle.Internals (ioe_EOF, readTextDevice, wantReadableHandle_, + wantWritableHandle) +import GHC.IO.Handle.Text (commitBuffer') +import GHC.IO.Handle.Types (BufferList(..), BufferMode(..), Handle__(..), Newline(..)) +import System.IO (Handle, hPutChar, utf8) +import System.IO.Error (isEOFError) +import qualified Data.Text as T + +-- | Read a single line of input from a handle, constructing a list of +-- decoded chunks as we go. When we're done, transform them into the +-- destination type. +hGetLineWith :: ([Text] -> t) -> Handle -> IO t +hGetLineWith f h = wantReadableHandle_ "hGetLine" h go + where + go hh@Handle__{..} = readIORef haCharBuffer >>= fmap f . hGetLineLoop hh [] + +hGetLineLoop :: Handle__ -> [Text] -> CharBuffer -> IO [Text] +hGetLineLoop hh@Handle__{..} = go where + go ts buf@Buffer{ bufL=r0, bufR=w, bufRaw=raw0 } = do + let findEOL raw r | r == w = return (False, w) + | otherwise = do + (c,r') <- readCharBuf raw r + if c == '\n' + then return (True, r) + else findEOL raw r' + (eol, off) <- findEOL raw0 r0 + (t,r') <- if haInputNL == CRLF + then unpack_nl raw0 r0 off + else do t <- unpack raw0 r0 off + return (t,off) + if eol + then do writeIORef haCharBuffer (bufferAdjustL (off+1) buf) + return $ reverse (t:ts) + else do + let buf1 = bufferAdjustL r' buf + maybe_buf <- maybeFillReadBuffer hh buf1 + case maybe_buf of + -- Nothing indicates we caught an EOF, and we may have a + -- partial line to return. + Nothing -> do + -- we reached EOF. There might be a lone \r left + -- in the buffer, so check for that and + -- append it to the line if necessary. + let pre | isEmptyBuffer buf1 = T.empty + | otherwise = T.singleton '\r' + writeIORef haCharBuffer buf1{ bufL=0, bufR=0 } + let str = reverse . filter (not . T.null) $ pre:t:ts + if null str + then ioe_EOF + else return str + Just new_buf -> go (t:ts) new_buf + +-- This function is lifted almost verbatim from GHC.IO.Handle.Text. +maybeFillReadBuffer :: Handle__ -> CharBuffer -> IO (Maybe CharBuffer) +maybeFillReadBuffer handle_ buf + = E.catch (Just `fmap` getSomeCharacters handle_ buf) $ \e -> + if isEOFError e + then return Nothing + else ioError e + +unpack :: RawCharBuffer -> Int -> Int -> IO Text +unpack !buf !r !w + | charSize /= 4 = sizeError "unpack" + | r >= w = return T.empty + | otherwise = withRawBuffer buf go + where + go pbuf = return $! unstream (Stream next r (exactSize (w-r))) + where + next !i | i >= w = Done + | otherwise = Yield (ix i) (i+1) + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- Variant of 'unpack' with CRLF decoding. If there is a trailing '\r', leave it in the buffer. +unpack_nl :: RawCharBuffer -> Int -> Int -> IO (Text, Int) +unpack_nl !buf !r !w + | charSize /= 4 = sizeError "unpack_nl" + | r >= w = return (T.empty, 0) + | otherwise = withRawBuffer buf $ go + where + go pbuf = do + let !t = unstream (Stream next r (maxSize (w-r))) + w' = w - 1 + return $ if ix w' == '\r' + then (t,w') + else (t,w) + where + next !i | i >= w = Done + | c == '\r' = let i' = i + 1 + in if i' < w + then if ix i' == '\n' + then Yield '\n' (i+2) + else Yield '\r' i' + else Done + | otherwise = Yield c (i+1) + where c = ix i + ix i = inlinePerformIO $ peekElemOff pbuf i + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSomeCharacters :: Handle__ -> CharBuffer -> IO CharBuffer +getSomeCharacters handle_@Handle__{..} buf@Buffer{..} = + case bufferElems buf of + -- buffer empty: read some more + 0 -> {-# SCC "readTextDevice" #-} readTextDevice handle_ buf + + -- if the buffer has a single '\r' in it and we're doing newline + -- translation: read some more + 1 | haInputNL == CRLF -> do + (c,_) <- readCharBuf bufRaw bufL + if c == '\r' + then do -- shuffle the '\r' to the beginning. This is only safe + -- if we're about to call readTextDevice, otherwise it + -- would mess up flushCharBuffer. + -- See [note Buffer Flushing], GHC.IO.Handle.Types + _ <- writeCharBuf bufRaw 0 '\r' + let buf' = buf{ bufL=0, bufR=1 } + readTextDevice handle_ buf' + else do + return buf + + -- buffer has some chars in it already: just return it + _otherwise -> {-# SCC "otherwise" #-} return buf + +-- | Read a single chunk of strict text from a buffer. Used by both +-- the strict and lazy implementations of hGetContents. +readChunk :: Handle__ -> CharBuffer -> IO Text +readChunk hh@Handle__{..} buf = do + buf'@Buffer{..} <- getSomeCharacters hh buf + (t,r) <- if haInputNL == CRLF + then unpack_nl bufRaw bufL bufR + else do t <- unpack bufRaw bufL bufR + return (t,bufR) + writeIORef haCharBuffer (bufferAdjustL r buf') + return t + +-- | Print a @Stream Char@. +hPutStream :: Handle -> Stream Char -> IO () +hPutStream h str = hPutStreamOrUtf8 h str Nothing + +-- | Write a string to a handle. +hPutStr :: Handle -> Text -> IO () +hPutStr h t = hPutStreamOrUtf8 h (stream t) (Just putUtf8) + where + putUtf8 = B.hPutStr h (encodeUtf8 t) + +-- | Write a string to a handle, followed by a newline. +hPutStrLn :: Handle -> Text -> IO () +hPutStrLn h t = hPutStreamOrUtf8 h (streamLn t) (Just putUtf8) + where + -- Not using B.hPutStrLn because it's not necessarily atomic: + -- https://github.com/haskell/bytestring/issues/200 + putUtf8 = hPutBuilder h (encodeUtf8Builder t <> charUtf8 '\n') + +-- | 'hPutStream' with an optional special case when the output encoding is +-- UTF-8 and without newline conversion. +hPutStreamOrUtf8 :: Handle -> Stream Char -> Maybe (IO ()) -> IO () +-- This function is modified from GHC.IO.Handle.Text. +hPutStreamOrUtf8 h str mPutUtf8 = do + (buffer_mode, nl, isUtf8) <- + wantWritableHandle "hPutStr" h $ \h_ -> do + bmode <- getSpareBuffer h_ + return (bmode, haOutputNL h_, eqUTF8 h_) + case buffer_mode of + _ | Just putUtf8 <- mPutUtf8, nl == LF && isUtf8 -> putUtf8 + (NoBuffering, _) -> hPutChars h str + (LineBuffering, buf) -> writeLines h nl buf str + (BlockBuffering _, buf) -> writeBlocks (nl == CRLF) h buf str + + where + -- If the encoding is UTF-8, it's most likely pointer-equal to + -- 'System.IO.utf8', letting us avoid a String comparison. + -- If it is somehow UTF-8 but not pointer-equal to 'utf8', + -- we will just take a slower branch, but the result is still correct. + eqUTF8 = maybe False (\enc -> isTrue# (reallyUnsafePtrEquality# utf8 enc)) . haCodec +{-# INLINE hPutStreamOrUtf8 #-} + +hPutChars :: Handle -> Stream Char -> IO () +hPutChars h (Stream next0 s0 _len) = loop s0 + where + loop !s = case next0 s of + Done -> return () + Skip s' -> loop s' + Yield x s' -> hPutChar h x >> loop s' + +-- The following functions are largely lifted from GHC.IO.Handle.Text, +-- but adapted to a coinductive stream of data instead of an inductive +-- list. +-- +-- We have several variations of more or less the same code for +-- performance reasons. Splitting the original buffered write +-- function into line- and block-oriented versions gave us a 2.1x +-- performance improvement. Lifting out the raw/cooked newline +-- handling gave a few more percent on top. + +writeLines :: Handle -> Newline -> CharBuffer -> Stream Char -> IO () +writeLines h nl buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' -> do + n' <- if nl == CRLF + then do n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' + else writeCharBuf' raw len n x + commit n' True{-needs flush-} False >>= outer s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +writeBlocks :: Bool -> Handle -> CharBuffer -> Stream Char -> IO () +writeBlocks isCRLF h buf0 (Stream next0 s0 _len) = outer s0 buf0 + where + outer s1 Buffer{bufRaw=raw, bufSize=len} = inner s1 (0::Int) + where + inner !s !n = + case next0 s of + Done -> commit n False{-no flush-} True{-release-} >> return () + Skip s' -> inner s' n + Yield x s' + -- Leave room for two characters for CRLF decoding + | n + 1 >= len -> commit n True{-needs flush-} False >>= outer s + | x == '\n' && isCRLF -> do + n1 <- writeCharBuf' raw len n '\r' + writeCharBuf' raw len n1 '\n' >>= inner s' + | otherwise -> writeCharBuf' raw len n x >>= inner s' + commit = commitBuffer h raw len + +-- | Only modifies the raw buffer and not the buffer attributes +writeCharBuf' :: RawCharBuffer -> Int -> Int -> Char -> IO Int +writeCharBuf' bufRaw bufSize n c = E.assert (n >= 0 && n < bufSize) $ + writeCharBuf bufRaw n c + +-- This function is completely lifted from GHC.IO.Handle.Text. +getSpareBuffer :: Handle__ -> IO (BufferMode, CharBuffer) +getSpareBuffer Handle__{haCharBuffer=ref, + haBuffers=spare_ref, + haBufferMode=mode} + = do + case mode of + NoBuffering -> return (mode, error "no buffer!") + _ -> do + bufs <- readIORef spare_ref + buf <- readIORef ref + case bufs of + BufferListCons b rest -> do + writeIORef spare_ref rest + return ( mode, emptyBuffer b (bufSize buf) WriteBuffer) + BufferListNil -> do + new_buf <- newCharBuffer (bufSize buf) WriteBuffer + return (mode, new_buf) + + +-- This function is modified from GHC.Internal.IO.Handle.Text. +commitBuffer :: Handle -> RawCharBuffer -> Int -> Int -> Bool -> Bool + -> IO CharBuffer +commitBuffer hdl !raw !sz !count flush release = + wantWritableHandle "commitAndReleaseBuffer" hdl $ + commitBuffer' raw sz count flush release +{-# INLINE commitBuffer #-} + +sizeError :: String -> a +sizeError loc = error $ "Data.Text.IO." ++ loc ++ ": bad internal buffer size"
src/Data/Text/Internal/Lazy.hs view
@@ -1,151 +1,151 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}-{-# OPTIONS_HADDOCK not-home #-}---- |--- Module : Data.Text.Internal.Lazy--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ /Warning/: this is an internal module, and does not have a stable--- API or name. Functions in this module may not check or enforce--- preconditions expected by public modules. Use at your own risk!------ A module containing private 'Text' internals. This exposes the--- 'Text' representation and low level construction functions.--- Modules which extend the 'Text' system may need to use this module.--module Data.Text.Internal.Lazy- (- Text(..)- , LazyText- , chunk- , empty- , foldrChunks- , foldlChunks- -- * Data type invariant and abstraction functions-- -- $invariant- , strictInvariant- , lazyInvariant- , showStructure-- -- * Chunk allocation sizes- , defaultChunkSize- , smallChunkSize- , chunkOverhead-- , equal- ) where--import Data.Bits (shiftL)-import Data.Text ()-import Data.Typeable (Typeable)-import Foreign.Storable (sizeOf)-import qualified Data.Text.Array as A-import qualified Data.Text.Internal as T-import qualified Data.Text as T--data Text = Empty- -- ^ Empty text.- --- -- @since 2.1.2- | Chunk {-# UNPACK #-} !T.Text Text- -- ^ Chunks must be non-empty, this invariant is not checked.- deriving (Typeable)---- | Type synonym for the lazy flavour of 'Text'.-type LazyText = Text---- $invariant------ The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or--- consists of non-null 'T.Text's. All functions must preserve this,--- and the QC properties must check this.---- | Check the invariant strictly.-strictInvariant :: Text -> Bool-strictInvariant Empty = True-strictInvariant x@(Chunk (T.Text _ _ len) cs)- | len > 0 = strictInvariant cs- | otherwise = error $ "Data.Text.Lazy: invariant violation: "- ++ showStructure x---- | Check the invariant lazily.-lazyInvariant :: Text -> Text-lazyInvariant Empty = Empty-lazyInvariant x@(Chunk c@(T.Text _ _ len) cs)- | len > 0 = Chunk c (lazyInvariant cs)- | otherwise = error $ "Data.Text.Lazy: invariant violation: "- ++ showStructure x---- | Display the internal structure of a lazy 'Text'.-showStructure :: Text -> String-showStructure Empty = "Empty"-showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"-showStructure (Chunk t ts) =- "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"---- | Smart constructor for 'Chunk'. Guarantees the data type invariant.-chunk :: T.Text -> Text -> Text-{-# INLINE [0] chunk #-}-chunk t ts | T.null t = ts- | otherwise = Chunk t ts--{-# RULES-"TEXT chunk/text" forall arr off len.- chunk (T.text arr off len) = chunk (T.Text arr off len)-"TEXT chunk/empty" forall ts.- chunk T.empty ts = ts-#-}---- | Smart constructor for 'Empty'.-empty :: Text-{-# INLINE [0] empty #-}-empty = Empty---- | Consume the chunks of a lazy 'Text' with a natural right fold.-foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a-foldrChunks f z = go- where go Empty = z- go (Chunk c cs) = f c (go cs)-{-# INLINE foldrChunks #-}---- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,--- accumulating left fold.-foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a-foldlChunks f z = go z- where go !a Empty = a- go !a (Chunk c cs) = go (f a c) cs-{-# INLINE foldlChunks #-}---- | Currently set to 16 KiB, less the memory management overhead.-defaultChunkSize :: Int-defaultChunkSize = 16384 - chunkOverhead-{-# INLINE defaultChunkSize #-}---- | Currently set to 128 bytes, less the memory management overhead.-smallChunkSize :: Int-smallChunkSize = 128 - chunkOverhead-{-# INLINE smallChunkSize #-}---- | The memory management overhead. Currently this is tuned for GHC only.-chunkOverhead :: Int-chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1-{-# INLINE chunkOverhead #-}--equal :: Text -> Text -> Bool-equal Empty Empty = True-equal Empty _ = False-equal _ Empty = False-equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =- case compare lenA lenB of- LT -> A.equal arrA offA arrB offB lenA &&- as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs- EQ -> A.equal arrA offA arrB offB lenA &&- as `equal` bs- GT -> A.equal arrA offA arrB offB lenB &&- Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs+{-# LANGUAGE BangPatterns #-} +{-# OPTIONS_HADDOCK not-home #-} + +-- | +-- Module : Data.Text.Internal.Lazy +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- /Warning/: this is an internal module, and does not have a stable +-- API or name. Functions in this module may not check or enforce +-- preconditions expected by public modules. Use at your own risk! +-- +-- A module containing private 'Text' internals. This exposes the +-- 'Text' representation and low level construction functions. +-- Modules which extend the 'Text' system may need to use this module. + +module Data.Text.Internal.Lazy + ( + Text(..) + , LazyText + , chunk + , empty + , foldrChunks + , foldlChunks + -- * Data type invariant and abstraction functions + + -- $invariant + , strictInvariant + , lazyInvariant + , showStructure + + -- * Chunk allocation sizes + , defaultChunkSize + , smallChunkSize + , chunkOverhead + + , equal + ) where + +import Data.Bits (shiftL) +import Data.Text () +import Foreign.Storable (sizeOf) +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text as T + +data Text = Empty + -- ^ Empty text. + -- + -- @since 2.1.2 + | Chunk {-# UNPACK #-} !T.Text Text + -- ^ Chunks must be non-empty, this invariant is not checked. + +-- | Type synonym for the lazy flavour of 'Text'. +-- +-- @since 2.1.1 +type LazyText = Text + +-- $invariant +-- +-- The data type invariant for lazy 'Text': Every 'Text' is either 'Empty' or +-- consists of non-null 'T.Text's. All functions must preserve this, +-- and the QC properties must check this. + +-- | Check the invariant strictly. +strictInvariant :: Text -> Bool +strictInvariant Empty = True +strictInvariant x@(Chunk (T.Text _ _ len) cs) + | len > 0 = strictInvariant cs + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Check the invariant lazily. +lazyInvariant :: Text -> Text +lazyInvariant Empty = Empty +lazyInvariant x@(Chunk c@(T.Text _ _ len) cs) + | len > 0 = Chunk c (lazyInvariant cs) + | otherwise = error $ "Data.Text.Lazy: invariant violation: " + ++ showStructure x + +-- | Display the internal structure of a lazy 'Text'. +showStructure :: Text -> String +showStructure Empty = "Empty" +showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty" +showStructure (Chunk t ts) = + "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")" + +-- | Smart constructor for 'Chunk'. Guarantees the data type invariant. +chunk :: T.Text -> Text -> Text +{-# INLINE [0] chunk #-} +chunk t ts | T.null t = ts + | otherwise = Chunk t ts + +{-# RULES +"TEXT chunk/text" forall arr off len. + chunk (T.text arr off len) = chunk (T.Text arr off len) +"TEXT chunk/empty" forall ts. + chunk T.empty ts = ts +#-} + +-- | Smart constructor for 'Empty'. +empty :: Text +{-# INLINE [0] empty #-} +empty = Empty + +-- | Consume the chunks of a lazy 'Text' with a natural right fold. +foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a +foldrChunks f z = go + where go Empty = z + go (Chunk c cs) = f c (go cs) +{-# INLINE foldrChunks #-} + +-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive, +-- accumulating left fold. +foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a +foldlChunks f z = go z + where go !a Empty = a + go !a (Chunk c cs) = go (f a c) cs +{-# INLINE foldlChunks #-} + +-- | Currently set to 16 KiB, less the memory management overhead. +defaultChunkSize :: Int +defaultChunkSize = 16384 - chunkOverhead +{-# INLINE defaultChunkSize #-} + +-- | Currently set to 128 bytes, less the memory management overhead. +smallChunkSize :: Int +smallChunkSize = 128 - chunkOverhead +{-# INLINE smallChunkSize #-} + +-- | The memory management overhead. Currently this is tuned for GHC only. +chunkOverhead :: Int +chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 +{-# INLINE chunkOverhead #-} + +equal :: Text -> Text -> Bool +equal Empty Empty = True +equal Empty _ = False +equal _ Empty = False +equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + case compare lenA lenB of + LT -> A.equal arrA offA arrB offB lenA && + as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs + EQ -> A.equal arrA offA arrB offB lenA && + as `equal` bs + GT -> A.equal arrA offA arrB offB lenB && + Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs
src/Data/Text/Internal/Transformation.hs view
@@ -1,340 +1,340 @@-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE PartialTypeSignatures #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE UnboxedTuples #-}-{-# LANGUAGE UnliftedFFITypes #-}--{-# OPTIONS_GHC -fno-warn-orphans #-}-{-# OPTIONS_GHC -Wno-partial-type-signatures #-}---- |--- Module : Data.Text.Internal.Transformation--- Copyright : (c) 2008, 2009 Tom Harper,--- (c) 2009, 2010 Bryan O'Sullivan,--- (c) 2009 Duncan Coutts------ License : BSD-style--- Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : GHC------ This module holds functions shared between the strict and lazy implementations of @Text@ transformations.--module Data.Text.Internal.Transformation- ( mapNonEmpty- , toCaseFoldNonEmpty- , toLowerNonEmpty- , toUpperNonEmpty- , toTitleNonEmpty- , filter_- ) where--import Prelude (Char, Bool(..), Int,- Ord(..),- Monad(..), pure,- (+), (-), ($), (&&), (||), (==),- not, return, otherwise, fromIntegral, (/=), const)-import Data.Bits ((.&.), shiftR, shiftL)-import Data.Char (isLetter, isSpace, ord)-import Control.Monad.ST (ST, runST)-import qualified Data.Text.Array as A-import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4)-import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping)-import Data.Text.Internal (Text(..), safe)-import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8)-import qualified Prelude as P-import Data.Text.Unsafe (Iter(..), iterArray)-import Data.Word (Word8, Word)-import qualified GHC.Exts as Exts-import GHC.Int (Int64(..))---- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@.--- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.-mapNonEmpty :: (Char -> Char) -> Text -> Text-mapNonEmpty f = go- where- go (Text src o l) = runST $ do- marr <- A.new (l + 4)- outer marr (l + 4) o 0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text- outer !dst !dstLen = inner- where- inner !srcOff !dstOff- | srcOff >= l + o = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- | dstOff + 4 > dstLen = do- let !dstLen' = dstLen + (l + o) - srcOff + 4- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff- | otherwise = do- let !(Iter c d) = iterArray src srcOff- d' <- unsafeWrite dst dstOff (safe (f c))- inner (srcOff + d) (dstOff + d')-{-# INLINE mapNonEmpty #-}--caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text-caseConvert ascii remap (Text src o l) = runST $ do- -- Case conversion a single code point may produce up to 3 code-points,- -- each up to 4 bytes, so 12 in total.- dst <- A.new (l + 12)- outer dst l o 0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text- outer !dst !dstLen = inner- where- inner !srcOff !dstOff- | srcOff >= o + l = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- | dstOff + 12 > dstLen = do- -- Ensure to extend the buffer by at least 12 bytes.- let !dstLen' = dstLen + max 12 (l + o - srcOff)- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff- -- If a character is to remain unchanged, no need to decode Char back into UTF8,- -- just copy bytes from input.- | otherwise = do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0- case d of- 1 -> do- A.unsafeWrite dst dstOff (ascii m0)- inner (srcOff + 1) (dstOff + 1)- 2 -> do- let !(Exts.C# c) = chr2 m0 m1- dstOff' <- case I64# (remap c) of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- pure $ dstOff + 2- i -> writeMapping dst i dstOff- inner (srcOff + 2) dstOff'- 3 -> do- let !(Exts.C# c) = chr3 m0 m1 m2- dstOff' <- case I64# (remap c) of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- pure $ dstOff + 3- i -> writeMapping dst i dstOff- inner (srcOff + 3) dstOff'- _ -> do- let !(Exts.C# c) = chr4 m0 m1 m2 m3- dstOff' <- case I64# (remap c) of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- pure $ dstOff + 4- i -> writeMapping dst i dstOff- inner (srcOff + 4) dstOff'--{-# INLINE caseConvert #-}--writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int-writeMapping !_ 0 !dstOff = pure dstOff-writeMapping dst i dstOff = do- let (ch, j) = chopOffChar i- d <- unsafeWrite dst dstOff ch- writeMapping dst j (dstOff + d)--chopOffChar :: Int64 -> (Char, Int64)-chopOffChar ab = (chr a, ab `shiftR` 21)- where- chr (Exts.I# n) = Exts.C# (Exts.chr# n)- mask = (1 `shiftL` 21) - 1- a = P.fromIntegral $ ab .&. mask---- | /O(n)/ Convert a string to folded case.--- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.-toCaseFoldNonEmpty :: Text -> Text-toCaseFoldNonEmpty = \xs -> caseConvert asciiToLower foldMapping xs-{-# INLINE toCaseFoldNonEmpty #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.--- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.-toLowerNonEmpty :: Text -> Text-toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs-{-# INLINE toLowerNonEmpty #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.--- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.-toUpperNonEmpty :: Text -> Text-toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs-{-# INLINE toUpperNonEmpty #-}--asciiToLower :: Word8 -> Word8-asciiToLower w = if w - 65 <= 25 then w + 32 else w--asciiToUpper :: Word8 -> Word8-asciiToUpper w = if w - 97 <= 25 then w - 32 else w--isAsciiLetter :: Word8 -> Bool-isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25--isAsciiSpace :: Word8 -> Bool-isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5)---- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion.--- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty.-toTitleNonEmpty :: Text -> Text-toTitleNonEmpty (Text src o l) = runST $ do- -- Case conversion a single code point may produce up to 3 code-points,- -- each up to 4 bytes, so 12 in total.- dst <- A.new (l + 12)- outer dst l o 0 False- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text- outer !dst !dstLen = inner- where- inner !srcOff !dstOff !mode- | srcOff >= o + l = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return (Text arr 0 dstOff)- | dstOff + 12 > dstLen = do- -- Ensure to extend the buffer by at least 12 bytes.- let !dstLen' = dstLen + max 12 (l + o - srcOff)- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff mode- -- If a character is to remain unchanged, no need to decode Char back into UTF8,- -- just copy bytes from input.- | otherwise = do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0-- case d of- 1 -> do- let (mode', m0') = asciiAdvance mode m0- A.unsafeWrite dst dstOff m0'- inner (srcOff + 1) (dstOff + 1) mode'- 2 -> do- let !(Exts.C# c) = chr2 m0 m1- !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c- dstOff' <- case I64# c' of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- pure $ dstOff + 2- i -> writeMapping dst i dstOff- inner (srcOff + 2) dstOff' mode'- 3 -> do- let !(Exts.C# c) = chr3 m0 m1 m2- isSpace3 ch- = m0 == 0xE1 && m1 == 0x9A && m2 == 0x80- || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F)- || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80- !(# mode', c' #) = advance isSpace3 mode c- dstOff' <- case I64# c' of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- pure $ dstOff + 3- i -> writeMapping dst i dstOff- inner (srcOff + 3) dstOff' mode'- _ -> do- let !(Exts.C# c) = chr4 m0 m1 m2 m3- !(# mode', c' #) = advance (\_ -> False) mode c- dstOff' <- case I64# c' of- 0 -> do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- pure $ dstOff + 4- i -> writeMapping dst i dstOff- inner (srcOff + 4) dstOff' mode'-- asciiAdvance :: Bool -> Word8 -> (Bool, Word8)- asciiAdvance False w = (isAsciiLetter w, asciiToUpper w)- asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w)-- advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #)- advance _ False c = (# isLetter (Exts.C# c), titleMapping c #)- advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #)- {-# INLINE advance #-}---- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@,--- calls the continuation with the @Text@ containing only the characters satisfying the predicate.-filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a-filter_ mkText p = go- where- go (Text src o l) = runST $ do- -- It's tempting to allocate l elements at once and avoid resizing.- -- However, this can be unacceptable in scenarios where a huge array- -- is filtered with a rare predicate, resulting in a much shorter buffer.- let !dstLen = min l 64- dst <- A.new dstLen- outer dst dstLen o 0- where- outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a- outer !dst !dstLen = inner- where- inner !srcOff !dstOff- | srcOff >= o + l = do- A.shrinkM dst dstOff- arr <- A.unsafeFreeze dst- return $ mkText arr 0 dstOff- | dstOff + 4 > dstLen = do- -- Double size of the buffer, unless it becomes longer than- -- source string. Ensure to extend it by least 4 bytes.- let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen)- dst' <- A.resizeM dst dstLen'- outer dst' dstLen' srcOff dstOff- -- In case of success, filter writes exactly the same character- -- it just read (this is not a case for map, for example).- -- We leverage this fact below: no need to decode Char back into UTF8,- -- just copy bytes from input.- | otherwise = do- let m0 = A.unsafeIndex src srcOff- m1 = A.unsafeIndex src (srcOff + 1)- m2 = A.unsafeIndex src (srcOff + 2)- m3 = A.unsafeIndex src (srcOff + 3)- !d = utf8LengthByLeader m0- case d of- 1 -> do- let !c = unsafeChr8 m0- if not (p c) then inner (srcOff + 1) dstOff else do- A.unsafeWrite dst dstOff m0- inner (srcOff + 1) (dstOff + 1)- 2 -> do- let !c = chr2 m0 m1- if not (p c) then inner (srcOff + 2) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- inner (srcOff + 2) (dstOff + 2)- 3 -> do- let !c = chr3 m0 m1 m2- if not (p c) then inner (srcOff + 3) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- inner (srcOff + 3) (dstOff + 3)- _ -> do- let !c = chr4 m0 m1 m2 m3- if not (p c) then inner (srcOff + 4) dstOff else do- A.unsafeWrite dst dstOff m0- A.unsafeWrite dst (dstOff + 1) m1- A.unsafeWrite dst (dstOff + 2) m2- A.unsafeWrite dst (dstOff + 3) m3- inner (srcOff + 4) (dstOff + 4)-{-# INLINE filter_ #-}+{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnboxedTuples #-} +{-# LANGUAGE UnliftedFFITypes #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} + +-- | +-- Module : Data.Text.Internal.Transformation +-- Copyright : (c) 2008, 2009 Tom Harper, +-- (c) 2009, 2010 Bryan O'Sullivan, +-- (c) 2009 Duncan Coutts +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Stability : experimental +-- Portability : GHC +-- +-- This module holds functions shared between the strict and lazy implementations of @Text@ transformations. + +module Data.Text.Internal.Transformation + ( mapNonEmpty + , toCaseFoldNonEmpty + , toLowerNonEmpty + , toUpperNonEmpty + , toTitleNonEmpty + , filter_ + ) where + +import Prelude (Char, Bool(..), Int, + Ord(..), + Monad(..), pure, + (+), (-), ($), (&&), (||), (==), + not, return, otherwise) +import Data.Bits ((.&.), shiftR, shiftL) +import Data.Char (isLetter, isSpace) +import Control.Monad.ST (ST, runST) +import qualified Data.Text.Array as A +import Data.Text.Internal.Encoding.Utf8 (utf8LengthByLeader, chr2, chr3, chr4) +import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, upperMapping, titleMapping) +import Data.Text.Internal (Text(..), safe) +import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) +import qualified Prelude as P +import Data.Text.Unsafe (Iter(..), iterArray) +import Data.Word (Word8) +import qualified GHC.Exts as Exts +import GHC.Int (Int64(..)) + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +mapNonEmpty :: (Char -> Char) -> Text -> Text +mapNonEmpty f = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + | otherwise = do + let !(Iter c d) = iterArray src srcOff + d' <- unsafeWrite dst dstOff (safe (f c)) + inner (srcOff + d) (dstOff + d') +{-# INLINE mapNonEmpty #-} + +caseConvert :: (Word8 -> Word8) -> (Exts.Char# -> _ {- unboxed Int64 -}) -> Text -> Text +caseConvert ascii remap (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff (ascii m0) + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + dstOff' <- case I64# (remap c) of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' + +{-# INLINABLE caseConvert #-} + +writeMapping :: A.MArray s -> Int64 -> Int -> ST s Int +writeMapping !_ 0 !dstOff = pure dstOff +writeMapping dst i dstOff = do + let (ch, j) = chopOffChar i + d <- unsafeWrite dst dstOff ch + writeMapping dst j (dstOff + d) + +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (Exts.I# n) = Exts.C# (Exts.chr# n) + mask = (1 `shiftL` 21) - 1 + a = P.fromIntegral $ ab .&. mask + +-- | /O(n)/ Convert a string to folded case. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toCaseFoldNonEmpty :: Text -> Text +toCaseFoldNonEmpty = \xs -> caseConvert asciiToLower foldMapping xs +{-# INLINE toCaseFoldNonEmpty #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toLowerNonEmpty :: Text -> Text +toLowerNonEmpty = \xs -> caseConvert asciiToLower lowerMapping xs +{-# INLINE toLowerNonEmpty #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toUpperNonEmpty :: Text -> Text +toUpperNonEmpty = \xs -> caseConvert asciiToUpper upperMapping xs +{-# INLINE toUpperNonEmpty #-} + +asciiToLower :: Word8 -> Word8 +asciiToLower w = if w - 65 <= 25 then w + 32 else w + +asciiToUpper :: Word8 -> Word8 +asciiToUpper w = if w - 97 <= 25 then w - 32 else w + +isAsciiLetter :: Word8 -> Bool +isAsciiLetter w = w - 65 <= 25 || w - 97 <= 25 + +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) + +-- | /O(n)/ Convert a string to title case, see 'Data.Text.toTitle' for discussion. +-- Assume that the @Text@ is non-empty. The returned @Text@ is guaranteed to be non-empty. +toTitleNonEmpty :: Text -> Text +toTitleNonEmpty (Text src o l) = runST $ do + -- Case conversion a single code point may produce up to 3 code-points, + -- each up to 4 bytes, so 12 in total. + dst <- A.new (l + 12) + outer dst l o 0 False + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> Bool -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !mode + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 12 > dstLen = do + -- Ensure to extend the buffer by at least 12 bytes. + let !dstLen' = dstLen + max 12 (l + o - srcOff) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff mode + -- If a character is to remain unchanged, no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + + case d of + 1 -> do + let (mode', m0') = asciiAdvance mode m0 + A.unsafeWrite dst dstOff m0' + inner (srcOff + 1) (dstOff + 1) mode' + 2 -> do + let !(Exts.C# c) = chr2 m0 m1 + !(# mode', c' #) = advance (\_ -> m0 == 0xC2 && m1 == 0xA0) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + pure $ dstOff + 2 + i -> writeMapping dst i dstOff + inner (srcOff + 2) dstOff' mode' + 3 -> do + let !(Exts.C# c) = chr3 m0 m1 m2 + isSpace3 ch + = m0 == 0xE1 && m1 == 0x9A && m2 == 0x80 + || m0 == 0xE2 && (m1 == 0x80 && isSpace (Exts.C# ch) || m1 == 0x81 && m2 == 0x9F) + || m0 == 0xE3 && m1 == 0x80 && m2 == 0x80 + !(# mode', c' #) = advance isSpace3 mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + pure $ dstOff + 3 + i -> writeMapping dst i dstOff + inner (srcOff + 3) dstOff' mode' + _ -> do + let !(Exts.C# c) = chr4 m0 m1 m2 m3 + !(# mode', c' #) = advance (\_ -> False) mode c + dstOff' <- case I64# c' of + 0 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + pure $ dstOff + 4 + i -> writeMapping dst i dstOff + inner (srcOff + 4) dstOff' mode' + + asciiAdvance :: Bool -> Word8 -> (Bool, Word8) + asciiAdvance False w = (isAsciiLetter w, asciiToUpper w) + asciiAdvance True w = (not (isAsciiSpace w), asciiToLower w) + + advance :: (Exts.Char# -> Bool) -> Bool -> Exts.Char# -> (# Bool, _ {- unboxed Int64 -} #) + advance _ False c = (# isLetter (Exts.C# c), titleMapping c #) + advance isSpaceChar True c = (# not (isSpaceChar c), lowerMapping c #) + {-# INLINE advance #-} + +-- | /O(n)/ 'filter_', applied to a continuation, a predicate and a @Text@, +-- calls the continuation with the @Text@ containing only the characters satisfying the predicate. +filter_ :: forall a. (A.Array -> Int -> Int -> a) -> (Char -> Bool) -> Text -> a +filter_ mkText p = go + where + go (Text src o l) = runST $ do + -- It's tempting to allocate l elements at once and avoid resizing. + -- However, this can be unacceptable in scenarios where a huge array + -- is filtered with a rare predicate, resulting in a much shorter buffer. + let !dstLen = min l 64 + dst <- A.new dstLen + outer dst dstLen o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s a + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return $ mkText arr 0 dstOff + | dstOff + 4 > dstLen = do + -- Double size of the buffer, unless it becomes longer than + -- source string. Ensure to extend it by least 4 bytes. + let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- In case of success, filter writes exactly the same character + -- it just read (this is not a case for map, for example). + -- We leverage this fact below: no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + let !c = unsafeChr8 m0 + if not (p c) then inner (srcOff + 1) dstOff else do + A.unsafeWrite dst dstOff m0 + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !c = chr2 m0 m1 + if not (p c) then inner (srcOff + 2) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + inner (srcOff + 2) (dstOff + 2) + 3 -> do + let !c = chr3 m0 m1 m2 + if not (p c) then inner (srcOff + 3) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + inner (srcOff + 3) (dstOff + 3) + _ -> do + let !c = chr4 m0 m1 m2 m3 + if not (p c) then inner (srcOff + 4) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + inner (srcOff + 4) (dstOff + 4) +{-# INLINE filter_ #-}
src/Data/Text/Lazy.hs view
@@ -1,1850 +1,1911 @@-{-# OPTIONS_GHC -fno-warn-orphans #-}-{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE TemplateHaskellQuotes #-}-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE PatternSynonyms #-}-{-# LANGUAGE ViewPatterns #-}---- |--- Module : Data.Text.Lazy--- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : GHC------ A time and space-efficient implementation of Unicode text using--- lists of packed arrays.------ /Note/: Read below the synopsis for important notes on the use of--- this module.------ The representation used by this module is suitable for high--- performance use and for streaming large quantities of data. It--- provides a means to manipulate a large body of text without--- requiring that the entire content be resident in memory.------ Some operations, such as 'concat', 'append', 'reverse' and 'cons',--- have better time complexity than their "Data.Text" equivalents, due--- to the underlying representation being a list of chunks. For other--- operations, lazy 'Text's are usually within a few percent of strict--- ones, but often with better heap usage if used in a streaming--- fashion. For data larger than available memory, or if you have--- tight memory constraints, this module will be the only option.------ This module is intended to be imported @qualified@, to avoid name--- clashes with "Prelude" functions. eg.------ > import qualified Data.Text.Lazy as L--module Data.Text.Lazy- (- -- * Fusion- -- $fusion-- -- * Acceptable data- -- $replacement-- -- * Types- Text- , LazyText-- -- * Creation and elimination- , pack- , unpack- , singleton- , empty- , fromChunks- , toChunks- , toStrict- , fromStrict- , foldrChunks- , foldlChunks-- -- * Pattern matching- , pattern Empty- , pattern (:<)- , pattern (:>)-- -- * Basic interface- , cons- , snoc- , append- , uncons- , unsnoc- , head- , last- , tail- , init- , null- , length- , compareLength-- -- * Transformations- , map- , intercalate- , intersperse- , transpose- , reverse- , replace-- -- ** Case conversion- -- $case- , toCaseFold- , toLower- , toUpper- , toTitle-- -- ** Justification- , justifyLeft- , justifyRight- , center-- -- * Folds- , foldl- , foldl'- , foldl1- , foldl1'- , foldr- , foldr1- , foldlM'-- -- ** Special folds- , concat- , concatMap- , any- , all- , maximum- , minimum- , isAscii-- -- * Construction-- -- ** Scans- , scanl- , scanl1- , scanr- , scanr1-- -- ** Accumulating maps- , mapAccumL- , mapAccumR-- -- ** Generation and unfolding- , repeat- , replicate- , cycle- , iterate- , unfoldr- , unfoldrN-- -- * Substrings-- -- ** Breaking strings- , take- , takeEnd- , drop- , dropEnd- , takeWhile- , takeWhileEnd- , dropWhile- , dropWhileEnd- , dropAround- , strip- , stripStart- , stripEnd- , splitAt- , span- , spanM- , spanEndM- , breakOn- , breakOnEnd- , break- , group- , groupBy- , inits- , initsNE- , tails- , tailsNE-- -- ** Breaking into many substrings- -- $split- , splitOn- , split- , chunksOf- -- , breakSubstring-- -- ** Breaking into lines and words- , lines- , words- , unlines- , unwords-- -- * Predicates- , isPrefixOf- , isSuffixOf- , isInfixOf-- -- ** View patterns- , stripPrefix- , stripSuffix- , commonPrefixes-- -- * Searching- , filter- , find- , elem- , breakOnAll- , partition-- -- , findSubstring-- -- * Indexing- , index- , count-- -- * Zipping and unzipping- , zip- , zipWith-- -- * Showing values- , show-- -- -* Ordered text- -- , sort- ) where--import Prelude (Char, Bool(..), Maybe(..), String,- Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec),- Monad(..), pure, (<$>),- (&&), (+), (-), (.), ($), (++),- error, flip, fmap, fromIntegral, not, otherwise, quot)-import qualified Prelude as P-import Control.Arrow (first)-import Control.DeepSeq (NFData(..))-import Data.Bits (finiteBitSize)-import Data.Int (Int64)-import qualified Data.List as L hiding (head, tail)-import Data.Char (isSpace)-import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,- Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))-import Data.Binary (Binary(get, put))-import Data.Binary.Put (putBuilder)-import Data.List.NonEmpty (NonEmpty(..))-import qualified Data.List.NonEmpty as NE-import Data.Monoid (Monoid(..))-import Data.Semigroup (Semigroup(..))-import Data.String (IsString(..))-import qualified Data.Text as T-import qualified Data.Text.Array as A-import qualified Data.Text.Internal as T-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Unsafe as T-import qualified Data.Text.Internal.Lazy.Fusion as S-import Data.Text.Internal.Fusion.Types (PairS(..))-import Data.Text.Internal.Lazy.Fusion (stream, unstream)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,- foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText)-import Data.Text.Internal (firstf, safe, text)-import Data.Text.Internal.Reverse (reverseNonEmpty)-import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_)-import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder)-import Data.Text.Internal.Lazy.Search (indices)-import qualified GHC.CString as GHC-import qualified GHC.Exts as Exts-import GHC.Prim (Addr#)-import GHC.Stack (HasCallStack)-import qualified Language.Haskell.TH.Lib as TH-import qualified Language.Haskell.TH.Syntax as TH-import Text.Printf (PrintfArg, formatArg, formatString)---- $fusion------ Starting from @text-1.3@ fusion is no longer implicit,--- and pipelines of transformations usually allocate intermediate 'Text' values.--- Users, who observe significant changes to performances,--- are encouraged to use fusion framework explicitly, employing--- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common".---- $replacement------ A 'Text' value is a sequence of Unicode scalar values, as defined--- in--- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >.--- As such, a 'Text' cannot contain values in the range U+D800 to--- U+DFFF inclusive. Haskell implementations admit all Unicode code--- points--- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >)--- as 'Char' values, including code points from this invalid range.--- This means that there are some 'Char' values--- (corresponding to 'Data.Char.Surrogate' category) that are not valid--- Unicode scalar values, and the functions in this module must handle--- those cases.------ Within this module, many functions construct a 'Text' from one or--- more 'Char' values. Those functions will substitute 'Char' values--- that are not valid Unicode scalar values with the replacement--- character \"�\" (U+FFFD). Functions that perform this--- inspection and replacement are documented with the phrase--- \"Performs replacement on invalid scalar values\". The functions replace--- invalid scalar values, instead of dropping them, as a security--- measure. For details, see--- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.)---- $setup--- >>> :set -package transformers--- >>> import Control.Monad.Trans.State--- >>> import Data.Text--- >>> import qualified Data.Text as T--- >>> :seti -XOverloadedStrings--instance Eq Text where- (==) = equal- {-# INLINE (==) #-}--instance Ord Text where- compare = compareText--compareText :: Text -> Text -> Ordering-compareText Empty Empty = EQ-compareText Empty _ = LT-compareText _ Empty = GT-compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) =- A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of- LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs)- EQ -> compareText as bs- GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs--- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208),--- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering--- of underlying bytearrays, no decoding is needed.--instance Show Text where- showsPrec p ps r = showsPrec p (unpack ps) r--instance Read Text where- readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]---- | @since 1.2.2.0-instance Semigroup Text where- (<>) = append--instance Monoid Text where- mempty = empty- mappend = (<>)- mconcat = concat---- | Performs replacement on invalid scalar values:------ >>> :set -XOverloadedStrings--- >>> "\55555" :: Data.Text.Lazy.Text--- "\65533"-instance IsString Text where- fromString = pack---- | Performs replacement on invalid scalar values:------ >>> :set -XOverloadedLists--- >>> ['\55555'] :: Data.Text.Lazy.Text--- "\65533"------ @since 1.2.0.0-instance Exts.IsList Text where- type Item Text = Char- fromList = pack- toList = unpack--instance NFData Text where- rnf Empty = ()- rnf (Chunk _ ts) = rnf ts---- | @since 1.2.1.0-instance Binary Text where- put t = do- -- This needs to be in sync with the Binary instance for ByteString- -- in the binary package.- put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t)- putBuilder (encodeUtf8Builder t)- get = do- bs <- get- case decodeUtf8' bs of- P.Left exn -> P.fail (P.show exn)- P.Right a -> P.return a---- | This instance preserves data abstraction at the cost of inefficiency.--- We omit reflection services for the sake of data abstraction.------ This instance was created by copying the updated behavior of--- @"Data.Text".@'Data.Text.Text'-instance Data Text where- gfoldl f z txt = z pack `f` (unpack txt)- toConstr _ = packConstr- gunfold k z c = case constrIndex c of- 1 -> k (z pack)- _ -> error "Data.Text.Lazy.Text.gunfold"- dataTypeOf _ = textDataType---- | @since 1.2.4.0-instance TH.Lift Text where- lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict-#if MIN_VERSION_template_haskell(2,17,0)- liftTyped = TH.unsafeCodeCoerce . TH.lift-#elif MIN_VERSION_template_haskell(2,16,0)- liftTyped = TH.unsafeTExpCoerce . TH.lift-#endif---- | @since 1.2.2.0-instance PrintfArg Text where- formatArg txt = formatString $ unpack txt--packConstr :: Constr-packConstr = mkConstr textDataType "pack" [] Prefix--textDataType :: DataType-textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr]---- | /O(n)/ Convert a 'String' into a 'Text'.------ Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id':------ >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555")--- "\65533"-pack ::-#if defined(ASSERTS)- HasCallStack =>-#endif- String -> Text-pack = unstream . S.streamList . L.map safe-{-# INLINE [1] pack #-}---- | /O(n)/ Convert a 'Text' into a 'String'.-unpack ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> String-unpack t = S.unstreamList (stream t)-{-# INLINE [1] unpack #-}---- | /O(n)/ Convert a literal string into a Text.-unpackCString# :: Addr# -> Text-unpackCString# addr# = unstream (S.streamCString# addr#)-{-# NOINLINE unpackCString# #-}--{-# RULES "TEXT literal" forall a.- unstream (S.streamList (L.map safe (GHC.unpackCString# a)))- = unpackCString# a #-}--{-# RULES "TEXT literal UTF8" forall a.- unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a)))- = unpackCString# a #-}--{-# RULES "LAZY TEXT empty literal"- unstream (S.streamList (L.map safe []))- = Empty #-}--{-# RULES "LAZY TEXT empty literal" forall a.- unstream (S.streamList (L.map safe [a]))- = Chunk (T.singleton a) Empty #-}---- | /O(1)/ Convert a character into a Text.--- Performs replacement on invalid scalar values.-singleton :: Char -> Text-singleton c = Chunk (T.singleton c) Empty-{-# INLINE [1] singleton #-}---- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'.-fromChunks :: [T.Text] -> Text-fromChunks cs = L.foldr chunk Empty cs---- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's.-toChunks :: Text -> [T.Text]-toChunks cs = foldrChunks (:) [] cs---- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'.-toStrict :: LazyText -> T.StrictText-toStrict t = T.concat (toChunks t)-{-# INLINE [1] toStrict #-}---- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'.-fromStrict :: T.StrictText -> LazyText-fromStrict t = chunk t Empty-{-# INLINE [1] fromStrict #-}---- -------------------------------------------------------------------------------- * Basic functions---- | /O(1)/ Adds a character to the front of a 'Text'.-cons :: Char -> Text -> Text-cons c t = Chunk (T.singleton c) t-{-# INLINE [1] cons #-}--infixr 5 `cons`---- | /O(n)/ Adds a character to the end of a 'Text'. This copies the--- entire array in the process.-snoc :: Text -> Char -> Text-snoc t c = foldrChunks Chunk (singleton c) t-{-# INLINE [1] snoc #-}---- | /O(n\/c)/ Appends one 'Text' to another.-append :: Text -> Text -> Text-append xs ys = foldrChunks Chunk ys xs-{-# INLINE [1] append #-}---- | /O(1)/ Returns the first character and rest of a 'Text', or--- 'Nothing' if empty.-uncons :: Text -> Maybe (Char, Text)-uncons Empty = Nothing-uncons (Chunk t ts) = Just (T.unsafeHead t, ts')- where ts' | T.compareLength t 1 == EQ = ts- | otherwise = Chunk (T.unsafeTail t) ts-{-# INLINE uncons #-}---- | /O(1)/ Returns the first character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'uncons' instead.-head :: HasCallStack => Text -> Char-head t = S.head (stream t)-{-# INLINE head #-}---- | /O(1)/ Returns all characters after the head of a 'Text', which--- must be non-empty. This is a partial function, consider using 'uncons' instead.-tail :: HasCallStack => Text -> Text-tail (Chunk t ts) = chunk (T.tail t) ts-tail Empty = emptyError "tail"-{-# INLINE [1] tail #-}---- | /O(n\/c)/ Returns all but the last character of a 'Text', which must--- be non-empty. This is a partial function, consider using 'unsnoc' instead.-init :: HasCallStack => Text -> Text-init (Chunk t0 ts0) = go t0 ts0- where go t (Chunk t' ts) = Chunk t (go t' ts)- go t Empty = chunk (T.init t) Empty-init Empty = emptyError "init"-{-# INLINE [1] init #-}---- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if--- empty.------ * It is no faster than using 'init' and 'last'.------ @since 1.2.3.0-unsnoc :: Text -> Maybe (Text, Char)-unsnoc Empty = Nothing-unsnoc ts@(Chunk _ _) = Just (init ts, last ts)-{-# INLINE unsnoc #-}---- | /O(1)/ Tests whether a 'Text' is empty or not.-null :: Text -> Bool-null Empty = True-null _ = False-{-# INLINE [1] null #-}---- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/),--- to be used together with 'Empty'.------ @since 2.1.2-pattern (:<) :: Char -> Text -> Text-pattern x :< xs <- (uncons -> Just (x, xs)) where- (:<) = cons-infixr 5 :<-{-# COMPLETE Empty, (:<) #-}---- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/)--- to be used together with 'Empty'.------ @since 2.1.2-pattern (:>) :: Text -> Char -> Text-pattern xs :> x <- (unsnoc -> Just (xs, x)) where- (:>) = snoc-infixl 5 :>-{-# COMPLETE Empty, (:>) #-}---- | /O(1)/ Tests whether a 'Text' contains exactly one character.-isSingleton :: Text -> Bool-isSingleton = S.isSingleton . stream-{-# INLINE isSingleton #-}---- | /O(n\/c)/ Returns the last character of a 'Text', which must be--- non-empty. This is a partial function, consider using 'unsnoc' instead.-last :: HasCallStack => Text -> Char-last Empty = emptyError "last"-last (Chunk t ts) = go t ts- where go _ (Chunk t' ts') = go t' ts'- go t' Empty = T.last t'-{-# INLINE [1] last #-}---- | /O(n)/ Returns the number of characters in a 'Text'.-length :: Text -> Int64-length = foldlChunks go 0- where- go :: Int64 -> T.Text -> Int64- go l t = l + intToInt64 (T.length t)-{-# INLINE [1] length #-}--{-# RULES-"TEXT length/map -> length" forall f t.- length (map f t) = length t-"TEXT length/zipWith -> length" forall f t1 t2.- length (zipWith f t1 t2) = min (length t1) (length t2)-"TEXT length/replicate -> n" forall n t.- length (replicate n t) = max 0 n P.* length t-"TEXT length/cons -> length+1" forall c t.- length (cons c t) = 1 + length t-"TEXT length/intersperse -> 2*length-1" forall c t.- length (intersperse c t) = max 0 (2 P.* length t - 1)-"TEXT length/intercalate -> n*length" forall s ts.- length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS)- #-}---- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number.------ @--- 'compareLength' t c = 'P.compare' ('length' t) c--- @------ This function gives the same answer as comparing against the result--- of 'length', but can short circuit if the count of characters is--- greater than the number, and hence be more efficient.-compareLength :: Text -> Int64 -> Ordering-compareLength t c = S.compareLengthI (stream t) c-{-# INLINE [1] compareLength #-}---- We don't apply those otherwise appealing length-to-compareLength--- rewrite rules here, because they can change the strictness--- properties of code.---- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to--- each element of @t@. Performs replacement on--- invalid scalar values.-map :: (Char -> Char) -> Text -> Text-map f = foldrChunks (Chunk . mapNonEmpty f) Empty-{-# INLINE [1] map #-}--{-# RULES-"TEXT map/map -> map" forall f g t.- map f (map g t) = map (f . safe . g) t-#-}---- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of--- 'Text's and concatenates the list after interspersing the first--- argument between each element of the list.-intercalate :: Text -> [Text] -> Text-intercalate t = concat . L.intersperse t-{-# INLINE [1] intercalate #-}---- | /O(n)/ The 'intersperse' function takes a character and places it--- between the characters of a 'Text'. Performs--- replacement on invalid scalar values.-intersperse :: Char -> Text -> Text-intersperse c t = unstream (S.intersperse (safe c) (stream t))-{-# INLINE [1] intersperse #-}---- | /O(n)/ Left-justify a string to the given length, using the--- specified fill character on the right. Performs--- replacement on invalid scalar values.------ Examples:------ > justifyLeft 7 'x' "foo" == "fooxxxx"--- > justifyLeft 3 'x' "foobar" == "foobar"-justifyLeft :: Int64 -> Char -> Text -> Text-justifyLeft k c t- | len >= k = t- | otherwise = t `append` replicateChunk (k-len) (T.singleton c)- where len = length t-{-# INLINE [1] justifyLeft #-}---- | /O(n)/ Right-justify a string to the given length, using the--- specified fill character on the left. Performs replacement on--- invalid scalar values.------ Examples:------ > justifyRight 7 'x' "bar" == "xxxxbar"--- > justifyRight 3 'x' "foobar" == "foobar"-justifyRight :: Int64 -> Char -> Text -> Text-justifyRight k c t- | len >= k = t- | otherwise = replicateChunk (k-len) (T.singleton c) `append` t- where len = length t-{-# INLINE justifyRight #-}---- | /O(n)/ Center a string to the given length, using the specified--- fill character on either side. Performs replacement on invalid--- scalar values.------ Examples:------ > center 8 'x' "HS" = "xxxHSxxx"-center :: Int64 -> Char -> Text -> Text-center k c t- | len >= k = t- | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c)- where len = length t- d = k - len- r = d `quot` 2- l = d - r-{-# INLINE center #-}---- | /O(n)/ The 'transpose' function transposes the rows and columns--- of its 'Text' argument. Note that this function uses 'pack',--- 'unpack', and the list version of transpose, and is thus not very--- efficient.-transpose :: [Text] -> [Text]-transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty)- (L.transpose (L.map unpack ts))--- TODO: make this fast---- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order.-reverse ::-#if defined(ASSERTS)- HasCallStack =>-#endif- Text -> Text-reverse = rev Empty- where rev a Empty = a- rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts---- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in--- @haystack@ with @replacement@.------ This function behaves as though it was defined as follows:------ @--- replace needle replacement haystack =--- 'intercalate' replacement ('splitOn' needle haystack)--- @------ As this suggests, each occurrence is replaced exactly once. So if--- @needle@ occurs in @replacement@, that occurrence will /not/ itself--- be replaced recursively:------ > replace "oo" "foo" "oo" == "foo"------ In cases where several instances of @needle@ overlap, only the--- first one will be replaced:------ > replace "ofo" "bar" "ofofo" == "barfo"------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-replace :: HasCallStack- => Text- -- ^ @needle@ to search for. If this string is empty, an- -- error will occur.- -> Text- -- ^ @replacement@ to replace @needle@ with.- -> Text- -- ^ @haystack@ in which to search.- -> Text-replace s d = intercalate d . splitOn s-{-# INLINE replace #-}---- ------------------------------------------------------------------------------- ** Case conversions (folds)---- $case------ With Unicode text, it is incorrect to use combinators like @map--- toUpper@ to case convert each character of a string individually.--- Instead, use the whole-string case conversion functions from this--- module. For correctness in different writing systems, these--- functions may map one input character to two or three output--- characters.---- | /O(n)/ Convert a string to folded case.------ This function is mainly useful for performing caseless (or case--- insensitive) string comparisons.------ A string @x@ is a caseless match for a string @y@ if and only if:------ @toCaseFold x == toCaseFold y@------ The result string may be longer than the input string, and may--- differ from applying 'toLower' to the input string. For instance,--- the Armenian small ligature men now (U+FB13) is case folded to the--- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is--- case folded to the Greek small letter letter mu (U+03BC) instead of--- itself.-toCaseFold :: Text -> Text-toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty-{-# INLINE toCaseFold #-}---- | /O(n)/ Convert a string to lower case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the Latin capital letter I with dot above (U+0130) maps--- to the sequence Latin small letter i (U+0069) followed by combining--- dot above (U+0307).-toLower :: Text -> Text-toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty-{-# INLINE toLower #-}---- | /O(n)/ Convert a string to upper case, using simple case--- conversion.------ The result string may be longer than the input string. For--- instance, the German eszett (U+00DF) maps to the two-letter--- sequence SS.-toUpper :: Text -> Text-toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty-{-# INLINE toUpper #-}----- | /O(n)/ Convert a string to title case, using simple case--- conversion.------ The first letter (as determined by 'Data.Char.isLetter')--- of the input is converted to title case, as is--- every subsequent letter that immediately follows a non-letter.--- Every letter that immediately follows another letter is converted--- to lower case.------ The result string may be longer than the input string. For example,--- the Latin small ligature fl (U+FB02) is converted to the--- sequence Latin capital letter F (U+0046) followed by Latin small--- letter l (U+006C).------ This function is not idempotent.--- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE).--- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input--- is converted to title case, becoming two letters.--- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter--- and as such is recognised as a letter by 'Data.Char.isLetter',--- so 'T.toTitle' @"ʼN"@ = @"'n"@.------ /Note/: this function does not take language or culture specific--- rules into account. For instance, in English, different style--- guides disagree on whether the book name \"The Hill of the Red--- Fox\" is correctly title cased—but this function will--- capitalize /every/ word.------ @since 1.0.0.0-toTitle :: Text -> Text-toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty-{-# INLINE toTitle #-}---- | /O(n)/ 'foldl', applied to a binary operator, a starting value--- (typically the left-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from left to right.-foldl :: (a -> Char -> a) -> a -> Text -> a-foldl f z t = S.foldl f z (stream t)-{-# INLINE foldl #-}---- | /O(n)/ A strict version of 'foldl'.----foldl' :: (a -> Char -> a) -> a -> Text -> a-foldl' f z t = S.foldl' f z (stream t)-{-# INLINE foldl' #-}---- | /O(n)/ A variant of 'foldl' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1 f t = S.foldl1 f (stream t)-{-# INLINE foldl1 #-}---- | /O(n)/ A strict version of 'foldl1'.-foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldl1' f t = S.foldl1' f (stream t)-{-# INLINE foldl1' #-}---- | /O(n)/ A monadic version of 'foldl''.------ @since 2.1.2-foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a-foldlM' f z t = S.foldlM' f z (stream t)-{-# INLINE foldlM' #-}---- | /O(n)/ 'foldr', applied to a binary operator, a starting value--- (typically the right-identity of the operator), and a 'Text',--- reduces the 'Text' using the binary operator, from right to left.------ 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually--- traverses the 'Text' from left to right, only as far as it needs to.------ For example, 'head' can be defined with /O(1)/ complexity using 'foldr':------ @--- head :: Text -> Char--- head = foldr const (error "head empty")--- @-foldr :: (Char -> a -> a) -> a -> Text -> a-foldr f z t = S.foldr f z (stream t)-{-# INLINE foldr #-}---- | /O(n)/ A variant of 'foldr' that has no starting value argument,--- and thus must be applied to a non-empty 'Text'.-foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char-foldr1 f t = S.foldr1 f (stream t)-{-# INLINE foldr1 #-}---- | /O(n)/ Concatenate a list of 'Text's.-concat :: [Text] -> Text-concat [] = Empty-concat (Empty : css) = concat css-concat (Chunk c Empty : css) = Chunk c (concat css)-concat (Chunk c cs : css) = Chunk c (concat (cs : css))-{-# INLINE concat #-}---- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and--- concatenate the results.-concatMap :: (Char -> Text) -> Text -> Text-concatMap f = concat . foldr ((:) . f) []-{-# INLINE concatMap #-}---- | /O(n)/ 'any' @p@ @t@ determines whether any character in the--- 'Text' @t@ satisfies the predicate @p@.-any :: (Char -> Bool) -> Text -> Bool-any p t = S.any p (stream t)-{-# INLINE any #-}---- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the--- 'Text' @t@ satisfy the predicate @p@.-all :: (Char -> Bool) -> Text -> Bool-all p t = S.all p (stream t)-{-# INLINE all #-}---- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which--- must be non-empty.-maximum :: HasCallStack => Text -> Char-maximum t = S.maximum (stream t)-{-# INLINE maximum #-}---- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which--- must be non-empty.-minimum :: HasCallStack => Text -> Char-minimum t = S.minimum (stream t)-{-# INLINE minimum #-}---- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only--- U+0000 through U+007F).------ This is a more efficient version of @'all' 'Data.Char.isAscii'@.------ >>> isAscii ""--- True------ >>> isAscii "abc\NUL"--- True------ >>> isAscii "abcd€"--- False------ prop> isAscii t == all (< '\x80') t------ @since 2.0.2-isAscii :: Text -> Bool-isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True---- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of--- successive reduced values from the left.--- Performs replacement on invalid scalar values.------ > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]------ Note that------ > last (scanl f z xs) == foldl f z xs.-scanl :: (Char -> Char -> Char) -> Char -> Text -> Text-scanl f z t = unstream (S.scanl g z (stream t))- where g a b = safe (f a b)-{-# INLINE scanl #-}---- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting--- value argument. Performs replacement on invalid scalar values.------ > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...]-scanl1 :: (Char -> Char -> Char) -> Text -> Text-scanl1 f t0 = case uncons t0 of- Nothing -> empty- Just (t,ts) -> scanl f t ts-{-# INLINE scanl1 #-}---- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs--- replacement on invalid scalar values.------ > scanr f v == reverse . scanl (flip f) v . reverse-scanr :: (Char -> Char -> Char) -> Char -> Text -> Text-scanr f v = reverse . scanl g v . reverse- where g a b = safe (f b a)---- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting--- value argument. Performs replacement on invalid scalar values.-scanr1 :: (Char -> Char -> Char) -> Text -> Text-scanr1 f t | null t = empty- | otherwise = scanr f (last t) (init t)---- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a--- function to each element of a 'Text', passing an accumulating--- parameter from left to right, and returns a final 'Text'. Performs--- replacement on invalid scalar values.-mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumL f = go- where- go z (Chunk c cs) = (z'', Chunk c' cs')- where (z', c') = T.mapAccumL f z c- (z'', cs') = go z' cs- go z Empty = (z, Empty)-{-# INLINE mapAccumL #-}---- | The 'mapAccumR' function behaves like a combination of 'map' and--- a strict 'foldr'; it applies a function to each element of a--- 'Text', passing an accumulating parameter from right to left, and--- returning a final value of this accumulator together with the new--- 'Text'. Performs replacement on invalid scalar values.-mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text)-mapAccumR f = go- where- go z (Chunk c cs) = (z'', Chunk c' cs')- where (z'', c') = T.mapAccumR f z' c- (z', cs') = go z cs- go z Empty = (z, Empty)-{-# INLINE mapAccumR #-}---- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every--- element.------ @since 1.2.0.5-repeat :: Char -> Text-repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t- in t---- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input--- @t@ repeated @n@ times.-replicate :: Int64 -> Text -> Text-replicate n- | n <= 0 = P.const Empty- | otherwise = \case- Empty -> Empty- Chunk t Empty -> replicateChunk n t- t -> concat (rep n)- where- rep 0 = []- rep i = t : rep (i - 1)-{-# INLINE [1] replicate #-}--replicateChunk :: Int64 -> T.Text -> Text-replicateChunk !n !t@(T.Text _ _ len)- | n <= 0 = Empty- | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk)- where- perChunk = defaultChunkSize `quot` len- normalChunk = T.replicate perChunk t- (q, r) = n `P.quotRem` intToInt64 perChunk- headChunk = T.replicate (int64ToInt r) t-{-# INLINE replicateChunk #-}---- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or--- equivalently, the infinite repetition of the original 'Text'.------ @since 1.2.0.5-cycle :: HasCallStack => Text -> Text-cycle Empty = emptyError "cycle"-cycle t = let t' = foldrChunks Chunk t' t- in t'---- | @'iterate' f x@ returns an infinite 'Text' of repeated applications--- of @f@ to @x@:------ > iterate f x == [x, f x, f (f x), ...]------ @since 1.2.0.5-iterate :: (Char -> Char) -> Char -> Text-iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))- in t c---- | /O(n)/, where @n@ is the length of the result. The 'unfoldr'--- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a--- 'Text' from a seed value. The function takes the element and--- returns 'Nothing' if it is done producing the 'Text', otherwise--- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the--- string, and @b@ is the seed value for further production.--- Performs replacement on invalid scalar values.-unfoldr :: (a -> Maybe (Char,a)) -> a -> Text-unfoldr f s = unstream (S.unfoldr (firstf safe . f) s)-{-# INLINE unfoldr #-}---- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed--- value. However, the length of the result should be limited by the--- first argument to 'unfoldrN'. This function is more efficient than--- 'unfoldr' when the maximum length of the result is known and--- correct, otherwise its performance is similar to 'unfoldr'.--- Performs replacement on invalid scalar values.-unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text-unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s)-{-# INLINE unfoldrN #-}---- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the--- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than--- the length of the Text.-take :: Int64 -> Text -> Text-take i _ | i <= 0 = Empty-take i t0 = take' i t0- where- take' :: Int64 -> Text -> Text- take' 0 _ = Empty- take' _ Empty = Empty- take' n (Chunk t@(T.Text arr off _) ts)- | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =- if m >= 0- then fromStrict (T.Text arr off m)- else Chunk t (take' (n + intToInt64 m) ts)-- | n < l = Chunk (T.take (int64ToInt n) t) Empty- | otherwise = Chunk t (take' (n - l) ts)- where l = intToInt64 (T.length t)-{-# INLINE [1] take #-}---- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after--- taking @n@ characters from the end of @t@.------ Examples:------ > takeEnd 3 "foobar" == "bar"------ @since 1.1.1.0-takeEnd :: Int64 -> Text -> Text-takeEnd n t0- | n <= 0 = empty- | otherwise = takeChunk n empty . L.reverse . toChunks $ t0- where- takeChunk :: Int64 -> Text -> [T.Text] -> Text- takeChunk _ acc [] = acc- takeChunk i acc (t:ts)- | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc- | otherwise = takeChunk (i-l) (Chunk t acc) ts- where l = intToInt64 (T.length t)---- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the--- 'Text' after the first @n@ characters, or the empty 'Text' if @n@--- is greater than the length of the 'Text'.-drop :: Int64 -> Text -> Text-drop i t0- | i <= 0 = t0- | otherwise = drop' i t0- where- drop' :: Int64 -> Text -> Text- drop' 0 ts = ts- drop' _ Empty = Empty- drop' n (Chunk t@(T.Text arr off len) ts)- | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t =- if m >= 0- then chunk (T.Text arr (off + m) (len - m)) ts- else drop' (n + intToInt64 m) ts-- | n < l = Chunk (T.drop (int64ToInt n) t) ts- | otherwise = drop' (n - l) ts- where l = intToInt64 (T.length t)-{-# INLINE [1] drop #-}---- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after--- dropping @n@ characters from the end of @t@.------ Examples:------ > dropEnd 3 "foobar" == "foo"------ @since 1.1.1.0-dropEnd :: Int64 -> Text -> Text-dropEnd n t0- | n <= 0 = t0- | otherwise = dropChunk n . L.reverse . toChunks $ t0- where- dropChunk :: Int64 -> [T.Text] -> Text- dropChunk _ [] = empty- dropChunk m (t:ts)- | m >= l = dropChunk (m-l) ts- | otherwise = fromChunks . L.reverse $- T.dropEnd (int64ToInt m) t : ts- where l = intToInt64 (T.length t)---- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8'--- values dropped, or the empty 'Text' if @n@ is greater than the--- number of 'Word8' values present.-dropWords :: Int64 -> Text -> Text-dropWords i t0- | i <= 0 = t0- | otherwise = drop' i t0- where- drop' :: Int64 -> Text -> Text- drop' 0 ts = ts- drop' _ Empty = Empty- drop' n (Chunk (T.Text arr off len) ts)- | n < len' = chunk (text arr (off+n') (len-n')) ts- | otherwise = drop' (n - len') ts- where len' = intToInt64 len- n' = int64ToInt n---- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text',--- returns the longest prefix (possibly empty) of elements that--- satisfy @p@.-takeWhile :: (Char -> Bool) -> Text -> Text-takeWhile p t0 = takeWhile' t0- where takeWhile' Empty = Empty- takeWhile' (Chunk t ts) =- case T.findIndex (not . p) t of- Just n | n > 0 -> Chunk (T.take n t) Empty- | otherwise -> Empty- Nothing -> Chunk t (takeWhile' ts)-{-# INLINE [1] takeWhile #-}---- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',--- returns the longest suffix (possibly empty) of elements that--- satisfy @p@.--- Examples:------ > takeWhileEnd (=='o') "foo" == "oo"------ @since 1.2.2.0-takeWhileEnd :: (Char -> Bool) -> Text -> Text-takeWhileEnd p = takeChunk empty . L.reverse . toChunks- where takeChunk acc [] = acc- takeChunk acc (t:ts)- | T.lengthWord8 t' < T.lengthWord8 t- = chunk t' acc- | otherwise = takeChunk (Chunk t' acc) ts- where t' = T.takeWhileEnd p t-{-# INLINE takeWhileEnd #-}---- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after--- 'takeWhile' @p@ @t@.-dropWhile :: (Char -> Bool) -> Text -> Text-dropWhile p t0 = dropWhile' t0- where dropWhile' Empty = Empty- dropWhile' (Chunk t ts) =- case T.findIndex (not . p) t of- Just n -> Chunk (T.drop n t) ts- Nothing -> dropWhile' ts-{-# INLINE [1] dropWhile #-}---- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after--- dropping characters that satisfy the predicate @p@ from the end of--- @t@.------ Examples:------ > dropWhileEnd (=='.') "foo..." == "foo"-dropWhileEnd :: (Char -> Bool) -> Text -> Text-dropWhileEnd p = go- where go Empty = Empty- go (Chunk t Empty) = if T.null t'- then Empty- else Chunk t' Empty- where t' = T.dropWhileEnd p t- go (Chunk t ts) = case go ts of- Empty -> go (Chunk t Empty)- ts' -> Chunk t ts'-{-# INLINE dropWhileEnd #-}---- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after--- dropping characters that satisfy the predicate @p@ from both the--- beginning and end of @t@.-dropAround :: (Char -> Bool) -> Text -> Text-dropAround p = dropWhile p . dropWhileEnd p-{-# INLINE [1] dropAround #-}---- | /O(n)/ Remove leading white space from a string. Equivalent to:------ > dropWhile isSpace-stripStart :: Text -> Text-stripStart = dropWhile isSpace-{-# INLINE stripStart #-}---- | /O(n)/ Remove trailing white space from a string. Equivalent to:------ > dropWhileEnd isSpace-stripEnd :: Text -> Text-stripEnd = dropWhileEnd isSpace-{-# INLINE [1] stripEnd #-}---- | /O(n)/ Remove leading and trailing white space from a string.--- Equivalent to:------ > dropAround isSpace-strip :: Text -> Text-strip = dropAround isSpace-{-# INLINE [1] strip #-}---- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a--- prefix of @t@ of length @n@, and whose second is the remainder of--- the string. It is equivalent to @('take' n t, 'drop' n t)@.-splitAt :: Int64 -> Text -> (Text, Text)-splitAt = loop- where- loop :: Int64 -> Text -> (Text, Text)- loop !_ Empty = (empty, empty)- loop n t | n <= 0 = (empty, t)- loop n (Chunk t ts)- | n < len = let (t',t'') = T.splitAt (int64ToInt n) t- in (Chunk t' Empty, Chunk t'' ts)- | otherwise = let (ts',ts'') = loop (n - len) ts- in (Chunk t ts', ts'')- where len = intToInt64 (T.length t)---- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first--- element is a prefix of @t@ whose chunks contain @n@ 'Word8'--- values, and whose second is the remainder of the string.-splitAtWord :: Int64 -> Text -> PairS Text Text-splitAtWord !_ Empty = empty :*: empty-splitAtWord x (Chunk c@(T.Text arr off len) cs)- | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs- in Chunk c h :*: t- | otherwise = chunk (text arr off y) empty :*:- chunk (text arr (off+y) (len-y)) cs- where y = int64ToInt x---- | /O(n+m)/ Find the first instance of @needle@ (which must be--- non-'null') in @haystack@. The first element of the returned tuple--- is the prefix of @haystack@ before @needle@ is matched. The second--- is the remainder of @haystack@, starting with the match.------ Examples:------ > breakOn "::" "a::b::c" ==> ("a", "::b::c")--- > breakOn "/" "foobar" ==> ("foobar", "")------ Laws:------ > append prefix match == haystack--- > where (prefix, match) = breakOn needle haystack------ If you need to break a string by a substring repeatedly (e.g. you--- want to break on every instance of a substring), use 'breakOnAll'--- instead, as it has lower startup overhead.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-breakOn :: HasCallStack => Text -> Text -> (Text, Text)-breakOn pat src- | null pat = emptyError "breakOn"- | otherwise = case indices pat src of- [] -> (src, empty)- (x:_) -> let h :*: t = splitAtWord x src- in (h, t)---- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.------ The first element of the returned tuple is the prefix of @haystack@--- up to and including the last match of @needle@. The second is the--- remainder of @haystack@, following the match.------ > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")-breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text)-breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)- in (reverse b, reverse a)-{-# INLINE breakOnEnd #-}---- | /O(n+m)/ Find all non-overlapping instances of @needle@ in--- @haystack@. Each element of the returned list consists of a pair:------ * The entire string prior to the /k/th match (i.e. the prefix)------ * The /k/th match, followed by the remainder of the string------ Examples:------ > breakOnAll "::" ""--- > ==> []--- > breakOnAll "/" "a/b/c/"--- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.------ The @needle@ parameter may not be empty.-breakOnAll :: HasCallStack- => Text -- ^ @needle@ to search for- -> Text -- ^ @haystack@ in which to search- -> [(Text, Text)]-breakOnAll pat src- | null pat = emptyError "breakOnAll"- | otherwise = go 0 empty src (indices pat src)- where- go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s- h' = append p h- in (h',t) : go x h' t xs- go _ _ _ _ = []---- | /O(n)/ 'break' is like 'span', but the prefix returned is over--- elements that fail the predicate @p@.------ >>> T.break (=='c') "180cm"--- ("180","cm")-break :: (Char -> Bool) -> Text -> (Text, Text)-break p t0 = break' t0- where break' Empty = (empty, empty)- break' c@(Chunk t ts) =- case T.findIndex p t of- Nothing -> let (ts', ts'') = break' ts- in (Chunk t ts', ts'')- Just n | n == 0 -> (Empty, c)- | otherwise -> let (a,b) = T.splitAt n t- in (Chunk a Empty, Chunk b ts)---- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns--- a pair whose first element is the longest prefix (possibly empty)--- of @t@ of elements that satisfy @p@, and whose second is the--- remainder of the text.------ >>> T.span (=='0') "000AB"--- ("000","AB")-span :: (Char -> Bool) -> Text -> (Text, Text)-span p = break (not . p)-{-# INLINE span #-}---- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@,--- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of--- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text.------ >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97--- (("abc","efg"),101)------ 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity':------ @--- -- for all p :: Char -> Bool--- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p)--- @------ @since 2.0.1-spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)-spanM p t0 = go t0- where- go Empty = pure (empty, empty)- go (Chunk t ts) = do- (t1, t2) <- T.spanM p t- if T.null t2 then first (chunk t) <$> go ts- else pure (chunk t1 empty, Chunk t2 ts)-{-# INLINE spanM #-}---- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@,--- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of--- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text.------ >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122--- (("tuv","xyz"),118)------ @--- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p--- @------ @since 2.0.1-spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text)-spanEndM p t0 = go t0- where- go Empty = pure (empty, empty)- go (Chunk t ts) = do- (t3, t4) <- go ts- if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t- else pure (Chunk t t3, t4)-{-# INLINE spanEndM #-}---- | The 'group' function takes a 'Text' and returns a list of 'Text's--- such that the concatenation of the result is equal to the argument.--- Moreover, each sublist in the result contains only equal elements.--- For example,------ > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"]------ It is a special case of 'groupBy', which allows the programmer to--- supply their own equality test.-group :: Text -> [Text]-group = groupBy (==)-{-# INLINE group #-}---- | The 'groupBy' function is the non-overloaded version of 'group'.-groupBy :: (Char -> Char -> Bool) -> Text -> [Text]-groupBy _ Empty = []-groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs- where (ys,zs) = span (eq x) xs- x = T.unsafeHead t- xs = chunk (T.unsafeTail t) ts---- | /O(n²)/ Return all initial segments of the given 'Text',--- shortest first.-inits :: Text -> [Text]-inits = (NE.toList P.$!) . initsNE---- | /O(n²)/ Return all initial segments of the given 'Text',--- shortest first.------ @since 2.1.2-initsNE :: Text -> NonEmpty Text-initsNE ts0 = Empty NE.:| inits' 0 ts0- where- inits' :: Int64 -- Number of previous chunks i- -> Text -- The remainder after dropping i chunks from ts0- -> [Text] -- Prefixes longer than the first i chunks of ts0.- inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t))- ++ inits' (i + 1) ts- inits' _ Empty = []--takeChunks :: Int64 -> Text -> T.Text -> Text-takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk)-takeChunks _ _ lastChunk = Chunk lastChunk Empty---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.-tails :: Text -> [Text]-tails = (NE.toList P.$!) . tailsNE---- | /O(n)/ Return all final segments of the given 'Text', longest--- first.------ @since 2.1.2-tailsNE :: Text -> NonEmpty Text-tailsNE Empty = Empty :| []-tailsNE ts@(Chunk t ts')- | T.length t == 1 = ts :| tails ts'- | otherwise = ts :| tails (Chunk (T.unsafeTail t) ts')---- $split------ Splitting functions in this library do not perform character-wise--- copies to create substrings; they just construct new 'Text's that--- are slices of the original.---- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'--- argument (which cannot be an empty string), consuming the--- delimiter. An empty delimiter is invalid, and will cause an error--- to be raised.------ Examples:------ > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]--- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]--- > splitOn "x" "x" == ["",""]------ and------ > intercalate s . splitOn s == id--- > splitOn (singleton c) == split (==c)------ (Note: the string @s@ to split on above cannot be empty.)------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-splitOn :: HasCallStack- => Text- -- ^ String to split on. If this string is empty, an error- -- will occur.- -> Text- -- ^ Input text.- -> [Text]-splitOn pat src- | null pat = emptyError "splitOn"- | isSingleton pat = split (== head pat) src- | otherwise = go 0 (indices pat src) src- where- go _ [] cs = [cs]- go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs- in h : go (x+l) xs (dropWords l t)- l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat-{-# INLINE [1] splitOn #-}--{-# RULES-"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.- splitOn (singleton c) t = split (==c) t- #-}---- | /O(n)/ Splits a 'Text' into components delimited by separators,--- where the predicate returns True for a separator element. The--- resulting components do not contain the separators. Two adjacent--- separators result in an empty component in the output. eg.------ > split (=='a') "aabbaca" == ["","","bb","c",""]--- > split (=='a') [] == [""]-split :: (Char -> Bool) -> Text -> [Text]-split _ Empty = [Empty]-split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0- where comb acc (s:[]) Empty = revChunks (s:acc) : []- comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts- comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts- comb _ [] _ = impossibleError "split"-{-# INLINE split #-}---- | /O(n)/ Splits a 'Text' into components of length @k@. The last--- element may be shorter than the other chunks, depending on the--- length of the input. Examples:------ > chunksOf 3 "foobarbaz" == ["foo","bar","baz"]--- > chunksOf 4 "haskell.org" == ["hask","ell.","org"]-chunksOf :: Int64 -> Text -> [Text]-chunksOf k = go- where- go t = case splitAt k t of- (a,b) | null a -> []- | otherwise -> a : go b-{-# INLINE chunksOf #-}---- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters--- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines.------ 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character.-lines :: Text -> [Text]-lines Empty = []-lines t = NE.toList $ go t- where- go :: Text -> NonEmpty Text- go Empty = Empty :| []- go (Chunk x xs)- -- x is non-empty, so T.lines x is non-empty as well- | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs- | otherwise = case unsnocList (T.lines x) of- Nothing -> impossibleError "lines"- Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls--prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text-prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial!--unsnocList :: [a] -> Maybe ([a], a)-unsnocList [] = Nothing-unsnocList (x : xs) = Just $ go x xs- where- go y [] = ([], y)- go y (z : zs) = first (y :) (go z zs)--hasNlEnd :: T.Text -> Bool-hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A---- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's--- representing white space.-words :: Text -> [Text]-words = L.filter (not . null) . split isSpace-{-# INLINE words #-}---- | /O(n)/ Joins lines, after appending a terminating newline to--- each.-unlines :: [Text] -> Text-unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) []-{-# INLINE unlines #-}---- | /O(n)/ Joins words using single space characters.-unwords :: [Text] -> Text-unwords = intercalate (singleton ' ')-{-# INLINE unwords #-}---- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns--- 'True' if and only if the first is a prefix of the second.-isPrefixOf :: Text -> Text -> Bool-isPrefixOf Empty _ = True-isPrefixOf _ Empty = False-isPrefixOf (Chunk x xs) (Chunk y ys)- | lx == ly = x == y && isPrefixOf xs ys- | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys)- | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys- where (xh,xt) = T.splitAt ly x- (yh,yt) = T.splitAt lx y- lx = T.length x- ly = T.length y---- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns--- 'True' if and only if the first is a suffix of the second.-isSuffixOf :: Text -> Text -> Bool-isSuffixOf x y = reverse x `isPrefixOf` reverse y-{-# INLINE isSuffixOf #-}--- TODO: a better implementation---- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns--- 'True' if and only if the first is contained, wholly and intact, anywhere--- within the second.------ This function is strict in its first argument, and lazy in its--- second.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-isInfixOf :: Text -> Text -> Bool-isInfixOf needle haystack- | null needle = True- | isSingleton needle = S.elem (head needle) . S.stream $ haystack- | otherwise = not . L.null . indices needle $ haystack-{-# INLINE [1] isInfixOf #-}------------------------------------------------------------------------------------ * View patterns---- | /O(n)/ Return the suffix of the second string if its prefix--- matches the entire first string.------ Examples:------ > stripPrefix "foo" "foobar" == Just "bar"--- > stripPrefix "" "baz" == Just "baz"--- > stripPrefix "foo" "quux" == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > fnordLength :: Text -> Int--- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf--- > fnordLength _ = -1-stripPrefix :: Text -> Text -> Maybe Text-stripPrefix p t- | null p = Just t- | otherwise = case commonPrefixes p t of- Just (_,c,r) | null c -> Just r- _ -> Nothing---- | /O(n)/ Find the longest non-empty common prefix of two strings--- and return it, along with the suffixes of each string at which they--- no longer match.------ If the strings do not have a common prefix or either one is empty,--- this function returns 'Nothing'.------ Examples:------ > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux")--- > commonPrefixes "veeble" "fetzer" == Nothing--- > commonPrefixes "" "baz" == Nothing-commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text)-commonPrefixes Empty _ = Nothing-commonPrefixes _ Empty = Nothing-commonPrefixes a0 b0 = Just (go a0 b0 [])- where- go t0@(Chunk x xs) t1@(Chunk y ys) ps- = case T.commonPrefixes x y of- Just (p,a,b)- | T.null a -> go xs (chunk b ys) (p:ps)- | T.null b -> go (chunk a xs) ys (p:ps)- | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys)- Nothing -> (fromChunks (L.reverse ps),t0,t1)- go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1)---- | /O(n)/ Return the prefix of the second string if its suffix--- matches the entire first string.------ Examples:------ > stripSuffix "bar" "foobar" == Just "foo"--- > stripSuffix "" "baz" == Just "baz"--- > stripSuffix "foo" "quux" == Nothing------ This is particularly useful with the @ViewPatterns@ extension to--- GHC, as follows:------ > {-# LANGUAGE ViewPatterns #-}--- > import Data.Text.Lazy as T--- >--- > quuxLength :: Text -> Int--- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre--- > quuxLength _ = -1-stripSuffix :: Text -> Text -> Maybe Text-stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t)---- | /O(n)/ 'filter', applied to a predicate and a 'Text',--- returns a 'Text' containing those characters that satisfy the--- predicate.-filter :: (Char -> Bool) -> Text -> Text-filter p = foldrChunks (chunk . filter_ T.Text p) Empty-{-# INLINE [1] filter #-}--{-# RULES-"TEXT filter/filter -> filter" forall p q t.- filter p (filter q t) = filter (\c -> q c && p c) t-#-}---- | /O(n)/ The 'find' function takes a predicate and a 'Text', and--- returns the first element in matching the predicate, or 'Nothing'--- if there is no such element.-find :: (Char -> Bool) -> Text -> Maybe Char-find p t = S.findBy p (stream t)-{-# INLINE find #-}---- | /O(n)/ The 'elem' function takes a character and a 'Text', and--- returns 'True' if the element is found in the given 'Text', or--- 'False' otherwise.-elem :: Char -> Text -> Bool-elem c t = S.any (== c) (stream t)-{-# INLINE elem #-}---- | /O(n)/ The 'partition' function takes a predicate and a 'Text',--- and returns the pair of 'Text's with elements which do and do not--- satisfy the predicate, respectively; i.e.------ > partition p t == (filter p t, filter (not . p) t)-partition :: (Char -> Bool) -> Text -> (Text, Text)-partition p t = (filter p t, filter (not . p) t)-{-# INLINE partition #-}---- | /O(n)/ 'Text' index (subscript) operator, starting from 0.-index :: HasCallStack => Text -> Int64 -> Char-index t n = S.index (stream t) n-{-# INLINE index #-}---- | /O(n+m)/ The 'count' function returns the number of times the--- query string appears in the given 'Text'. An empty query string is--- invalid, and will cause an error to be raised.------ In (unlikely) bad cases, this function's time complexity degrades--- towards /O(n*m)/.-count :: HasCallStack => Text -> Text -> Int64-count pat- | null pat = emptyError "count"- | otherwise = go 0 . indices pat- where go !n [] = n- go !n (_:xs) = go (n+1) xs-{-# INLINE [1] count #-}--{-# RULES-"LAZY TEXT count/singleton -> countChar" [~1] forall c t.- count (singleton c) t = countChar c t- #-}---- | /O(n)/ The 'countChar' function returns the number of times the--- query element appears in the given 'Text'.-countChar :: Char -> Text -> Int64-countChar c t = S.countChar c (stream t)---- | /O(n)/ 'zip' takes two 'Text's and returns a list of--- corresponding pairs of bytes. If one input 'Text' is short,--- excess elements of the longer 'Text' are discarded. This is--- equivalent to a pair of 'unpack' operations.-zip :: Text -> Text -> [(Char,Char)]-zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)-{-# INLINE [0] zip #-}---- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function--- given as the first argument, instead of a tupling function.--- Performs replacement on invalid scalar values.-zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text-zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))- where g a b = safe (f a b)-{-# INLINE [0] zipWith #-}---- | Convert a value to lazy 'Text'.------ @since 2.1.2-show :: Show a => a -> Text-show = pack . P.show--revChunks :: [T.Text] -> Text-revChunks = L.foldl' (flip chunk) Empty--emptyError :: HasCallStack => String -> a-emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input")--impossibleError :: HasCallStack => String -> a-impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case")--intToInt64 :: Exts.Int -> Int64-intToInt64 = fromIntegral--int64ToInt :: Int64 -> Exts.Int-int64ToInt = fromIntegral+{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy +-- Copyright : (c) 2009, 2010, 2012 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : GHC +-- +-- A time and space-efficient implementation of Unicode text using +-- lists of packed arrays. +-- +-- /Note/: Read below the synopsis for important notes on the use of +-- this module. +-- +-- The representation used by this module is suitable for high +-- performance use and for streaming large quantities of data. It +-- provides a means to manipulate a large body of text without +-- requiring that the entire content be resident in memory. +-- +-- Some operations, such as 'concat', 'append', 'reverse' and 'cons', +-- have better time complexity than their "Data.Text" equivalents, due +-- to the underlying representation being a list of chunks. For other +-- operations, lazy 'Text's are usually within a few percent of strict +-- ones, but often with better heap usage if used in a streaming +-- fashion. For data larger than available memory, or if you have +-- tight memory constraints, this module will be the only option. +-- +-- This module is intended to be imported @qualified@, to avoid name +-- clashes with "Prelude" functions. eg. +-- +-- > import qualified Data.Text.Lazy as L + +module Data.Text.Lazy + ( + -- * Fusion + -- $fusion + + -- * Acceptable data + -- $replacement + + -- * Types + Text + , LazyText + + -- * Creation and elimination + , pack + , unpack + , singleton + , empty + , fromChunks + , toChunks + , toStrict + , fromStrict + , foldrChunks + , foldlChunks + + -- * Pattern matching + , pattern Empty + , pattern (:<) + , pattern (:>) + + -- * Basic interface + , cons + , snoc + , append + , uncons + , unsnoc + , head + , last + , tail + , init + , null + , length + , compareLength + + -- * Transformations + , map + , intercalate + , intersperse + , transpose + , reverse + , replace + + -- ** Case conversion + -- $case + , toCaseFold + , toLower + , toUpper + , toTitle + + -- ** Justification + , justifyLeft + , justifyRight + , center + + -- * Folds + , foldl + , foldl' + , foldl1 + , foldl1' + , foldr + , foldr1 + , foldlM' + + -- ** Special folds + , concat + , concatMap + , any + , all + , maximum + , minimum + , isAscii + + -- * Construction + + -- ** Scans + , scanl + , scanl1 + , scanr + , scanr1 + + -- ** Accumulating maps + , mapAccumL + , mapAccumR + + -- ** Generation and unfolding + , repeat + , replicate + , cycle + , iterate + , unfoldr + , unfoldrN + + -- * Substrings + + -- ** Breaking strings + , take + , takeEnd + , drop + , dropEnd + , takeWhile + , takeWhileEnd + , dropWhile + , dropWhileEnd + , dropAround + , strip + , stripStart + , stripEnd + , splitAt + , span + , spanM + , spanEndM + , breakOn + , breakOnEnd + , break + , group + , groupBy + , inits + , initsNE + , tails + , tailsNE + + -- ** Breaking into many substrings + -- $split + , splitOn + , split + , chunksOf + -- , breakSubstring + + -- ** Breaking into lines and words + , lines + , words + , unlines + , unwords + + -- * Predicates + , isPrefixOf + , isSuffixOf + , isInfixOf + + -- ** View patterns + , stripPrefix + , stripSuffix + , commonPrefixes + + -- * Searching + , filter + , find + , elem + , breakOnAll + , partition + + -- , findSubstring + + -- * Indexing + , index + , count + + -- * Zipping and unzipping + , zip + , zipWith + + -- * Showing values + , show + + -- -* Ordered text + -- , sort + ) where + +import Prelude (Char, Bool(..), Maybe(..), String, + Eq, (==), Ord(..), Ordering(..), Read(..), Show(showsPrec), + Monad(..), pure, (<$>), + (&&), (+), (-), (.), ($), (++), + error, flip, fmap, fromIntegral, not, otherwise, quot) +import qualified Prelude as P +import Control.Arrow (first) +import Control.DeepSeq (NFData(..)) +import Data.Bits (finiteBitSize, toIntegralSized) +import Data.Int (Int64) +import qualified Data.List as L hiding (head, tail) +import Data.Char (isSpace) +import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, + Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) +import Data.Binary (Binary(get, put)) +import Data.Binary.Put (putBuilder) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE +import Data.Monoid (Monoid(..)) +import Data.Semigroup (Semigroup(..)) +import Data.String (IsString(..)) +import qualified Data.Text as T +import qualified Data.Text.Array as A +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Unsafe as T +import qualified Data.Text.Internal.Lazy.Fusion as S +import Data.Text.Internal.Fusion.Types (PairS(..)) +import Data.Text.Internal.Lazy.Fusion (stream, unstream) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, + foldrChunks, smallChunkSize, defaultChunkSize, equal, LazyText) +import Data.Text.Internal (firstf, safe, text) +import Data.Text.Internal.Reverse (reverseNonEmpty) +import Data.Text.Internal.Transformation (mapNonEmpty, toCaseFoldNonEmpty, toLowerNonEmpty, toUpperNonEmpty, filter_) +import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8Builder) +import Data.Text.Internal.Lazy.Search (indices) +import qualified GHC.CString as GHC +import qualified GHC.Exts as Exts +import GHC.Prim (Addr#) +import GHC.Stack (HasCallStack) +#if __GLASGOW_HASKELL__ >= 914 +import qualified Language.Haskell.TH.Lift as TH +#else +import qualified Language.Haskell.TH.Syntax as TH +import qualified Language.Haskell.TH.Lib as TH +#endif +import Text.Printf (PrintfArg, formatArg, formatString) + +-- $fusion +-- +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transformations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". + +-- $replacement +-- +-- A 'Text' value is a sequence of Unicode scalar values, as defined +-- in +-- <http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=35 §3.9, definition D76 of the Unicode 5.2 standard >. +-- As such, a 'Text' cannot contain values in the range U+D800 to +-- U+DFFF inclusive. Haskell implementations admit all Unicode code +-- points +-- (<http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#page=13 §3.4, definition D10 >) +-- as 'Char' values, including code points from this invalid range. +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid +-- Unicode scalar values, and the functions in this module must handle +-- those cases. +-- +-- Within this module, many functions construct a 'Text' from one or +-- more 'Char' values. Those functions will substitute 'Char' values +-- that are not valid Unicode scalar values with the replacement +-- character \"�\" (U+FFFD). Functions that perform this +-- inspection and replacement are documented with the phrase +-- \"Performs replacement on invalid scalar values\". The functions replace +-- invalid scalar values, instead of dropping them, as a security +-- measure. For details, see +-- <http://unicode.org/reports/tr36/#Deletion_of_Noncharacters Unicode Technical Report 36, §3.5 >.) + +-- $setup +-- >>> :set -package transformers +-- >>> import Control.Monad.Trans.State +-- >>> import Data.Text +-- >>> import qualified Data.Text as T +-- >>> :seti -XOverloadedStrings + +instance Eq Text where + (==) = equal + {-# INLINE (==) #-} + +instance Ord Text where + compare = compareText + +compareText :: Text -> Text -> Ordering +compareText Empty Empty = EQ +compareText Empty _ = LT +compareText _ Empty = GT +compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of + LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) + EQ -> compareText as bs + GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. + +instance Show Text where + showsPrec p ps r = showsPrec p (unpack ps) r + +instance Read Text where + readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str] + +-- | @since 1.2.2.0 +instance Semigroup Text where + (<>) = append + stimes n _ | n < 0 = P.error "Data.Text.Lazy.stimes: given number is negative!" + stimes n a = + let nInt64 = fromIntegral n :: Int64 + len = if n == fromIntegral nInt64 && nInt64 >= 0 then nInt64 else P.maxBound + -- We clamp the length to maxBound :: Int64. + -- To tell the difference, the caller would have to skip through 2^63 chunks. + in replicate len a + +instance Monoid Text where + mempty = empty + mappend = (<>) + mconcat = concat + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedStrings +-- >>> "\55555" :: Data.Text.Lazy.Text +-- "\65533" +instance IsString Text where + fromString = pack + +-- | Performs replacement on invalid scalar values: +-- +-- >>> :set -XOverloadedLists +-- >>> ['\55555'] :: Data.Text.Lazy.Text +-- "\65533" +-- +-- @since 1.2.0.0 +instance Exts.IsList Text where + type Item Text = Char + fromList = pack + toList = unpack + +instance NFData Text where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts + +-- | @since 1.2.1.0 +instance Binary Text where + put t = do + -- This needs to be in sync with the Binary instance for ByteString + -- in the binary package. + put (foldlChunks (\n c -> n + T.lengthWord8 c) 0 t) + putBuilder (encodeUtf8Builder t) + get = do + bs <- get + case decodeUtf8' bs of + P.Left exn -> P.fail (P.show exn) + P.Right a -> P.return a + +-- | This instance preserves data abstraction at the cost of inefficiency. +-- We omit reflection services for the sake of data abstraction. +-- +-- This instance was created by copying the updated behavior of +-- @"Data.Text".@'Data.Text.Text' +instance Data Text where + gfoldl f z txt = z pack `f` (unpack txt) + toConstr _ = packConstr + gunfold k z c = case constrIndex c of + 1 -> k (z pack) + _ -> error "Data.Text.Lazy.Text.gunfold" + dataTypeOf _ = textDataType + +-- | @since 1.2.4.0 +instance TH.Lift Text where +#if __GLASGOW_HASKELL__ >= 900 + lift x = [| fromStrict $(TH.lift . toStrict $ x) |] +#else + lift = TH.appE (TH.varE 'fromStrict) . TH.lift . toStrict +#endif +#if __GLASGOW_HASKELL__ >= 914 + liftTyped = TH.defaultLiftTyped +#elif __GLASGOW_HASKELL__ >= 900 + liftTyped = TH.unsafeCodeCoerce . TH.lift +#elif __GLASGOW_HASKELL__ >= 810 + liftTyped = TH.unsafeTExpCoerce . TH.lift +#endif + +-- | @since 1.2.2.0 +instance PrintfArg Text where + formatArg txt = formatString $ unpack txt + +packConstr :: Constr +packConstr = mkConstr textDataType "pack" [] Prefix + +textDataType :: DataType +textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr] + +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- +-- Performs replacement on invalid scalar values, so @'unpack' . 'pack'@ is not 'id': +-- +-- >>> Data.Text.Lazy.unpack (Data.Text.Lazy.pack "\55555") +-- "\65533" +pack :: +#if defined(ASSERTS) + HasCallStack => +#endif + String -> Text +pack = unstream . S.streamList . L.map safe +{-# INLINE [1] pack #-} + +-- | /O(n)/ Convert a 'Text' into a 'String'. +unpack :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> String +unpack t = S.unstreamList (stream t) +{-# NOINLINE unpack #-} + +foldrFB :: (Char -> b -> b) -> b -> Text -> b +foldrFB = foldr +{-# INLINE [0] foldrFB #-} + +-- List fusion rules for `unpack`: +-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build` +-- fuses if `foldr` is applied to it. +-- * If it doesn't fuse: In phase 1, `build` inlines to give us `foldrFB (:) []` +-- and we rewrite that back to `unpack`. +-- * If it fuses: In phase 0, `foldrFB` inlines and `foldr` inlines. GHC +-- optimizes the fused code. +{-# RULES +"Text.Lazy.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrFB lcons lnil t) +"Text.Lazy.unpackBack" [1] foldrFB (:) [] = unpack + #-} + +-- | /O(n)/ Convert a literal string into a Text. +unpackCString# :: Addr# -> Text +unpackCString# addr# = unstream (S.streamCString# addr#) +{-# NOINLINE unpackCString# #-} + +{-# RULES "TEXT literal" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCString# a))) + = unpackCString# a #-} + +{-# RULES "TEXT literal UTF8" forall a. + unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a))) + = unpackCString# a #-} + +{-# RULES "LAZY TEXT empty literal" + unstream (S.streamList (L.map safe [])) + = Empty #-} + +{-# RULES "LAZY TEXT empty literal" forall a. + unstream (S.streamList (L.map safe [a])) + = Chunk (T.singleton a) Empty #-} + +-- | /O(1)/ Convert a character into a Text. +-- Performs replacement on invalid scalar values. +singleton :: Char -> Text +singleton c = Chunk (T.singleton c) Empty +{-# INLINE [1] singleton #-} + +-- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. +fromChunks :: [T.Text] -> Text +fromChunks cs = L.foldr chunk Empty cs + +-- | /O(n)/ Convert a lazy 'Text' into a list of strict 'T.Text's. +toChunks :: Text -> [T.Text] +toChunks cs = foldrChunks (:) [] cs + +-- | /O(n)/ Convert a lazy 'Text' into a strict 'T.Text'. +toStrict :: LazyText -> T.StrictText +toStrict t = T.concat (toChunks t) +{-# INLINE [1] toStrict #-} + +-- | /O(c)/ Convert a strict 'T.Text' into a lazy 'Text'. +fromStrict :: T.StrictText -> LazyText +fromStrict t = chunk t Empty +{-# INLINE [1] fromStrict #-} + +-- ----------------------------------------------------------------------------- +-- * Basic functions + +-- | /O(1)/ Adds a character to the front of a 'Text'. +cons :: Char -> Text -> Text +cons c t = Chunk (T.singleton c) t +{-# INLINE [1] cons #-} + +infixr 5 `cons` + +-- | /O(n)/ Adds a character to the end of a 'Text'. This copies the +-- entire array in the process. +snoc :: Text -> Char -> Text +snoc t c = foldrChunks Chunk (singleton c) t +{-# INLINE [1] snoc #-} + +-- | /O(n\/c)/ Appends one 'Text' to another. +append :: Text -> Text -> Text +append xs ys = foldrChunks Chunk ys xs +{-# INLINE [1] append #-} + +-- | /O(1)/ Returns the first character and rest of a 'Text', or +-- 'Nothing' if empty. +uncons :: Text -> Maybe (Char, Text) +uncons Empty = Nothing +uncons (Chunk t ts) = Just (T.unsafeHead t, ts') + where ts' | T.compareLength t 1 == EQ = ts + | otherwise = Chunk (T.unsafeTail t) ts +{-# INLINE uncons #-} + +-- | /O(1)/ Returns the first character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char +head t = S.head (stream t) +{-# INLINE head #-} + +-- | /O(1)/ Returns all characters after the head of a 'Text', which +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text +tail (Chunk t ts) = chunk (T.tail t) ts +tail Empty = emptyError "tail" +{-# INLINE [1] tail #-} + +-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init (Chunk t0 ts0) = go t0 ts0 + where go t (Chunk t' ts) = Chunk t (go t' ts) + go t Empty = chunk (T.init t) Empty +init Empty = emptyError "init" +{-# INLINE [1] init #-} + +-- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if +-- empty. +-- +-- * It is no faster than using 'init' and 'last'. +-- +-- @since 1.2.3.0 +unsnoc :: Text -> Maybe (Text, Char) +unsnoc Empty = Nothing +unsnoc ts@(Chunk _ _) = Just (init ts, last ts) +{-# INLINE unsnoc #-} + +-- | /O(1)/ Tests whether a 'Text' is empty or not. +null :: Text -> Bool +null Empty = True +null _ = False +{-# INLINE [1] null #-} + +-- | Bidirectional pattern synonym for 'cons' (/O(n)/) and 'uncons' (/O(1)/), +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:<) :: Char -> Text -> Text +pattern x :< xs <- (uncons -> Just (x, xs)) where + (:<) = cons +infixr 5 :< +{-# COMPLETE Empty, (:<) #-} + +-- | Bidirectional pattern synonym for 'snoc' (/O(n)/) and 'unsnoc' (/O(1)/) +-- to be used together with 'Empty'. +-- +-- @since 2.1.2 +pattern (:>) :: Text -> Char -> Text +pattern xs :> x <- (unsnoc -> Just (xs, x)) where + (:>) = snoc +infixl 5 :> +{-# COMPLETE Empty, (:>) #-} + +-- | /O(1)/ Tests whether a 'Text' contains exactly one character. +isSingleton :: Text -> Bool +isSingleton = S.isSingleton . stream +{-# INLINE isSingleton #-} + +-- | /O(n\/c)/ Returns the last character of a 'Text', which must be +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last Empty = emptyError "last" +last (Chunk t ts) = go t ts + where go _ (Chunk t' ts') = go t' ts' + go t' Empty = T.last t' +{-# INLINE [1] last #-} + +-- | /O(n)/ Returns the number of characters in a 'Text'. +length :: Text -> Int64 +length = foldlChunks go 0 + where + go :: Int64 -> T.Text -> Int64 + go l t = l + intToInt64 (T.length t) +{-# INLINE [1] length #-} + +{-# RULES +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = max 0 n P.* length t +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (2 P.* length t - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} + +-- | /O(min(n,c))/ Compare the count of characters in a 'Text' to a number. +-- +-- @ +-- 'compareLength' t c = 'P.compare' ('length' t) c +-- @ +-- +-- This function gives the same answer as comparing against the result +-- of 'length', but can short circuit if the count of characters is +-- greater than the number, and hence be more efficient. +compareLength :: Text -> Int64 -> Ordering +compareLength t c = S.compareLengthI (stream t) c +{-# INLINE [1] compareLength #-} + +-- We don't apply those otherwise appealing length-to-compareLength +-- rewrite rules here, because they can change the strictness +-- properties of code. + +-- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to +-- each element of @t@. Performs replacement on +-- invalid scalar values. +map :: (Char -> Char) -> Text -> Text +map f = foldrChunks (Chunk . mapNonEmpty f) Empty +{-# INLINE [1] map #-} + +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + +-- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of +-- 'Text's and concatenates the list after interspersing the first +-- argument between each element of the list. +intercalate :: Text -> [Text] -> Text +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} + +-- | /O(n)/ The 'intersperse' function takes a character and places it +-- between the characters of a 'Text'. Performs +-- replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t = unstream (S.intersperse (safe c) (stream t)) +{-# INLINE [1] intersperse #-} + +-- | /O(n)/ Left-justify a string to the given length, using the +-- specified fill character on the right. Performs +-- replacement on invalid scalar values. +-- +-- Examples: +-- +-- > justifyLeft 7 'x' "foo" == "fooxxxx" +-- > justifyLeft 3 'x' "foobar" == "foobar" +justifyLeft :: Int64 -> Char -> Text -> Text +justifyLeft k c t + | len >= k = t + | otherwise = t `append` replicateChunk (k-len) (T.singleton c) + where len = length t +{-# INLINE [1] justifyLeft #-} + +-- | /O(n)/ Right-justify a string to the given length, using the +-- specified fill character on the left. Performs replacement on +-- invalid scalar values. +-- +-- Examples: +-- +-- > justifyRight 7 'x' "bar" == "xxxxbar" +-- > justifyRight 3 'x' "foobar" == "foobar" +justifyRight :: Int64 -> Char -> Text -> Text +justifyRight k c t + | len >= k = t + | otherwise = replicateChunk (k-len) (T.singleton c) `append` t + where len = length t +{-# INLINE justifyRight #-} + +-- | /O(n)/ Center a string to the given length, using the specified +-- fill character on either side. Performs replacement on invalid +-- scalar values. +-- +-- Examples: +-- +-- > center 8 'x' "HS" = "xxxHSxxx" +center :: Int64 -> Char -> Text -> Text +center k c t + | len >= k = t + | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) + where len = length t + d = k - len + r = d `quot` 2 + l = d - r +{-# INLINE center #-} + +-- | /O(n)/ The 'transpose' function transposes the rows and columns +-- of its 'Text' argument. Note that this function uses 'pack', +-- 'unpack', and the list version of transpose, and is thus not very +-- efficient. +transpose :: [Text] -> [Text] +transpose ts = L.map (\ss -> Chunk (T.pack ss) Empty) + (L.transpose (L.map unpack ts)) +-- TODO: make this fast + +-- | /O(n)/ 'reverse' @t@ returns the elements of @t@ in reverse order. +reverse :: +#if defined(ASSERTS) + HasCallStack => +#endif + Text -> Text +reverse = rev Empty + where rev a Empty = a + rev a (Chunk t ts) = rev (Chunk (reverseNonEmpty t) a) ts + +-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in +-- @haystack@ with @replacement@. +-- +-- This function behaves as though it was defined as follows: +-- +-- @ +-- replace needle replacement haystack = +-- 'intercalate' replacement ('splitOn' needle haystack) +-- @ +-- +-- As this suggests, each occurrence is replaced exactly once. So if +-- @needle@ occurs in @replacement@, that occurrence will /not/ itself +-- be replaced recursively: +-- +-- > replace "oo" "foo" "oo" == "foo" +-- +-- In cases where several instances of @needle@ overlap, only the +-- first one will be replaced: +-- +-- > replace "ofo" "bar" "ofofo" == "barfo" +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +replace :: HasCallStack + => Text + -- ^ @needle@ to search for. If this string is empty, an + -- error will occur. + -> Text + -- ^ @replacement@ to replace @needle@ with. + -> Text + -- ^ @haystack@ in which to search. + -> Text +replace s d = intercalate d . splitOn s +{-# INLINE replace #-} + +-- ---------------------------------------------------------------------------- +-- ** Case conversions (folds) + +-- $case +-- +-- With Unicode text, it is incorrect to use combinators like @map +-- toUpper@ to case convert each character of a string individually. +-- Instead, use the whole-string case conversion functions from this +-- module. For correctness in different writing systems, these +-- functions may map one input character to two or three output +-- characters. + +-- | /O(n)/ Convert a string to folded case. +-- +-- This function is mainly useful for performing caseless (or case +-- insensitive) string comparisons. +-- +-- A string @x@ is a caseless match for a string @y@ if and only if: +-- +-- @toCaseFold x == toCaseFold y@ +-- +-- The result string may be longer than the input string, and may +-- differ from applying 'toLower' to the input string. For instance, +-- the Armenian small ligature men now (U+FB13) is case folded to the +-- bigram men now (U+0574 U+0576), while the micro sign (U+00B5) is +-- case folded to the Greek small letter letter mu (U+03BC) instead of +-- itself. +toCaseFold :: Text -> Text +toCaseFold = foldrChunks (\chnk acc -> Chunk (toCaseFoldNonEmpty chnk) acc) Empty +{-# INLINE toCaseFold #-} + +-- | /O(n)/ Convert a string to lower case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the Latin capital letter I with dot above (U+0130) maps +-- to the sequence Latin small letter i (U+0069) followed by combining +-- dot above (U+0307). +toLower :: Text -> Text +toLower = foldrChunks (\chnk acc -> Chunk (toLowerNonEmpty chnk) acc) Empty +{-# INLINE toLower #-} + +-- | /O(n)/ Convert a string to upper case, using simple case +-- conversion. +-- +-- The result string may be longer than the input string. For +-- instance, the German eszett (U+00DF) maps to the two-letter +-- sequence SS. +toUpper :: Text -> Text +toUpper = foldrChunks (\chnk acc -> Chunk (toUpperNonEmpty chnk) acc) Empty +{-# INLINE toUpper #-} + + +-- | /O(n)/ Convert a string to title case, using simple case +-- conversion. +-- +-- The first letter (as determined by 'Data.Char.isLetter') +-- of the input is converted to title case, as is +-- every subsequent letter that immediately follows a non-letter. +-- Every letter that immediately follows another letter is converted +-- to lower case. +-- +-- The result string may be longer than the input string. For example, +-- the Latin small ligature fl (U+FB02) is converted to the +-- sequence Latin capital letter F (U+0046) followed by Latin small +-- letter l (U+006C). +-- +-- This function is not idempotent. +-- Consider lower-case letter @ʼn@ (U+0149 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE). +-- Then 'T.toTitle' @"ʼn"@ = @"ʼN"@: the first (and the only) letter of the input +-- is converted to title case, becoming two letters. +-- Now @ʼ@ (U+02BC MODIFIER LETTER APOSTROPHE) is a modifier letter +-- and as such is recognised as a letter by 'Data.Char.isLetter', +-- so 'T.toTitle' @"ʼN"@ = @"'n"@. +-- +-- /Note/: this function does not take language or culture specific +-- rules into account. For instance, in English, different style +-- guides disagree on whether the book name \"The Hill of the Red +-- Fox\" is correctly title cased—but this function will +-- capitalize /every/ word. +-- +-- @since 1.0.0.0 +toTitle :: Text -> Text +toTitle = foldrChunks (\chnk acc -> Chunk (T.toTitle chnk) acc) Empty +{-# INLINE toTitle #-} + +-- | /O(n)/ 'foldl', applied to a binary operator, a starting value +-- (typically the left-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from left to right. +foldl :: (a -> Char -> a) -> a -> Text -> a +foldl f z t = S.foldl f z (stream t) +{-# INLINE foldl #-} + +-- | /O(n)/ A strict version of 'foldl'. +-- +foldl' :: (a -> Char -> a) -> a -> Text -> a +foldl' f z t = S.foldl' f z (stream t) +{-# INLINE foldl' #-} + +-- | /O(n)/ A variant of 'foldl' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1 f t = S.foldl1 f (stream t) +{-# INLINE foldl1 #-} + +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldl1' f t = S.foldl1' f (stream t) +{-# INLINE foldl1' #-} + +-- | /O(n)/ A monadic version of 'foldl''. +-- +-- @since 2.1.2 +foldlM' :: Monad m => (a -> Char -> m a) -> a -> Text -> m a +foldlM' f z t = S.foldlM' f z (stream t) +{-# INLINE foldlM' #-} + +-- | /O(n)/ 'foldr', applied to a binary operator, a starting value +-- (typically the right-identity of the operator), and a 'Text', +-- reduces the 'Text' using the binary operator, from right to left. +-- +-- 'foldr' is lazy like 'Data.List.foldr' for lists: evaluation actually +-- traverses the 'Text' from left to right, only as far as it needs to. +-- +-- For example, 'head' can be defined with /O(1)/ complexity using 'foldr': +-- +-- @ +-- head :: Text -> Char +-- head = foldr const (error "head empty") +-- @ +foldr :: (Char -> a -> a) -> a -> Text -> a +foldr f z t = S.foldr f z (stream t) +{-# INLINE foldr #-} + +-- | /O(n)/ A variant of 'foldr' that has no starting value argument, +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char +foldr1 f t = S.foldr1 f (stream t) +{-# INLINE foldr1 #-} + +-- | /O(n)/ Concatenate a list of 'Text's. +concat :: [Text] -> Text +concat [] = Empty +concat (Empty : css) = concat css +concat (Chunk c Empty : css) = Chunk c (concat css) +concat (Chunk c cs : css) = Chunk c (concat (cs : css)) +{-# INLINE concat #-} + +-- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and +-- concatenate the results. +concatMap :: (Char -> Text) -> Text -> Text +concatMap f = concat . foldr ((:) . f) [] +{-# INLINE concatMap #-} + +-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the +-- 'Text' @t@ satisfies the predicate @p@. +any :: (Char -> Bool) -> Text -> Bool +any p t = S.any p (stream t) +{-# INLINE any #-} + +-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the +-- 'Text' @t@ satisfy the predicate @p@. +all :: (Char -> Bool) -> Text -> Bool +all p t = S.all p (stream t) +{-# INLINE all #-} + +-- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which +-- must be non-empty. +maximum :: HasCallStack => Text -> Char +maximum t = S.maximum (stream t) +{-# INLINE maximum #-} + +-- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which +-- must be non-empty. +minimum :: HasCallStack => Text -> Char +minimum t = S.minimum (stream t) +{-# INLINE minimum #-} + +-- | \O(n)\ Test whether 'Text' contains only ASCII code-points (i.e. only +-- U+0000 through U+007F). +-- +-- This is a more efficient version of @'all' 'Data.Char.isAscii'@. +-- +-- >>> isAscii "" +-- True +-- +-- >>> isAscii "abc\NUL" +-- True +-- +-- >>> isAscii "abcd€" +-- False +-- +-- prop> isAscii t == all (< '\x80') t +-- +-- @since 2.0.2 +isAscii :: Text -> Bool +isAscii = foldrChunks (\chnk acc -> T.isAscii chnk && acc) True + +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of +-- successive reduced values from the left. +-- Performs replacement on invalid scalar values. +-- +-- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] +-- +-- Note that +-- +-- > last (scanl f z xs) == foldl f z xs. +scanl :: (Char -> Char -> Char) -> Char -> Text -> Text +scanl f z t = cons z $ P.snd $ + mapAccumL (\acc c -> let c' = f acc c in (c', c')) (safe z) t +-- This is a bit suboptimal: we could have used +-- Data.Text.scanl for the first chunk and mapAccumL +-- for subsequent ones, but but I doubt anyone cares +-- about the performance of 'scanl' much. +{-# INLINE scanl #-} + +-- | /O(n)/ 'scanl1' is a variant of 'scanl' that has no starting +-- value argument. Performs replacement on invalid scalar values. +-- +-- > scanl1 f [x1, x2, ...] == [x1, x1 `f` x2, ...] +scanl1 :: (Char -> Char -> Char) -> Text -> Text +scanl1 f t0 = case uncons t0 of + Nothing -> empty + Just (t,ts) -> scanl f t ts +{-# INLINE scanl1 #-} + +-- | /O(n)/ 'scanr' is the right-to-left dual of 'scanl'. Performs +-- replacement on invalid scalar values. +-- +-- > scanr f v == reverse . scanl (flip f) v . reverse +scanr :: (Char -> Char -> Char) -> Char -> Text -> Text +scanr f z t = (`snoc` z) $ P.snd $ + mapAccumR (\acc c -> let c' = f c acc in (c', c')) (safe z) t +-- See the comment for 'scanl' above. +{-# INLINE scanr #-} + +-- | /O(n)/ 'scanr1' is a variant of 'scanr' that has no starting +-- value argument. Performs replacement on invalid scalar values. +scanr1 :: (Char -> Char -> Char) -> Text -> Text +scanr1 f t | null t = empty + | otherwise = scanr f (last t) (init t) + +-- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a +-- function to each element of a 'Text', passing an accumulating +-- parameter from left to right, and returns a final 'Text'. Performs +-- replacement on invalid scalar values. +mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumL f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z', c') = T.mapAccumL f z c + (z'', cs') = go z' cs + go z Empty = (z, Empty) +{-# INLINE mapAccumL #-} + +-- | The 'mapAccumR' function behaves like a combination of 'map' and +-- a strict 'foldr'; it applies a function to each element of a +-- 'Text', passing an accumulating parameter from right to left, and +-- returning a final value of this accumulator together with the new +-- 'Text'. Performs replacement on invalid scalar values. +mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) +mapAccumR f = go + where + go z (Chunk c cs) = (z'', Chunk c' cs') + where (z'', c') = T.mapAccumR f z' c + (z', cs') = go z cs + go z Empty = (z, Empty) +{-# INLINE mapAccumR #-} + +-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every +-- element. +-- +-- @since 1.2.0.5 +repeat :: Char -> Text +repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t + in t + +-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input +-- @t@ repeated @n@ times. +replicate :: Int64 -> Text -> Text +replicate n + | n <= 0 = P.const Empty + | otherwise = \case + Empty -> Empty + Chunk t Empty -> replicateChunk n t + t -> concat (rep n) + where + rep 0 = [] + rep i = t : rep (i - 1) +{-# INLINE [1] replicate #-} + +replicateChunk :: Int64 -> T.Text -> Text +replicateChunk !n !t@(T.Text _ _ len) + | n <= 0 = Empty + | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) + where + perChunk = defaultChunkSize `quot` len + normalChunk = T.replicate perChunk t + (q, r) = n `P.quotRem` intToInt64 perChunk + headChunk = T.replicate (int64ToInt r) t +{-# INLINE replicateChunk #-} + +-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or +-- equivalently, the infinite repetition of the original 'Text'. +-- +-- @since 1.2.0.5 +cycle :: HasCallStack => Text -> Text +cycle Empty = emptyError "cycle" +cycle t = let t' = foldrChunks Chunk t' t + in t' + +-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications +-- of @f@ to @x@: +-- +-- > iterate f x == [x, f x, f (f x), ...] +-- +-- @since 1.2.0.5 +iterate :: (Char -> Char) -> Char -> Text +iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) + in t c + +-- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' +-- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a +-- 'Text' from a seed value. The function takes the element and +-- returns 'Nothing' if it is done producing the 'Text', otherwise +-- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. +unfoldr :: (a -> Maybe (Char,a)) -> a -> Text +unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) +{-# INLINE unfoldr #-} + +-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a 'Text' from a seed +-- value. However, the length of the result should be limited by the +-- first argument to 'unfoldrN'. This function is more efficient than +-- 'unfoldr' when the maximum length of the result is known and +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. +unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text +unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) +{-# INLINE unfoldrN #-} + +-- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the +-- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than +-- the length of the Text. +take :: Int64 -> Text -> Text +take i _ | i <= 0 = Empty +take i t0 = take' i t0 + where + take' :: Int64 -> Text -> Text + take' 0 _ = Empty + take' _ Empty = Empty + take' n (Chunk t@(T.Text arr off _) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then fromStrict (T.Text arr off m) + else Chunk t (take' (n + intToInt64 m) ts) + + | n < l = Chunk (T.take (int64ToInt n) t) Empty + | otherwise = Chunk t (take' (n - l) ts) + where l = intToInt64 (T.length t) +{-# INLINE [1] take #-} + +-- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after +-- taking @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > takeEnd 3 "foobar" == "bar" +-- +-- @since 1.1.1.0 +takeEnd :: Int64 -> Text -> Text +takeEnd n t0 + | n <= 0 = empty + | otherwise = takeChunk n empty . L.reverse . toChunks $ t0 + where + takeChunk :: Int64 -> Text -> [T.Text] -> Text + takeChunk _ acc [] = acc + takeChunk i acc (t:ts) + | i <= l = chunk (T.takeEnd (int64ToInt i) t) acc + | otherwise = takeChunk (i-l) (Chunk t acc) ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the +-- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ +-- is greater than the length of the 'Text'. +drop :: Int64 -> Text -> Text +drop i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk t@(T.Text arr off len) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then chunk (T.Text arr (off + m) (len - m)) ts + else drop' (n + intToInt64 m) ts + + | n < l = Chunk (T.drop (int64ToInt n) t) ts + | otherwise = drop' (n - l) ts + where l = intToInt64 (T.length t) +{-# INLINE [1] drop #-} + +-- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after +-- dropping @n@ characters from the end of @t@. +-- +-- Examples: +-- +-- > dropEnd 3 "foobar" == "foo" +-- +-- @since 1.1.1.0 +dropEnd :: Int64 -> Text -> Text +dropEnd n t0 + | n <= 0 = t0 + | otherwise = dropChunk n . L.reverse . toChunks $ t0 + where + dropChunk :: Int64 -> [T.Text] -> Text + dropChunk _ [] = empty + dropChunk m (t:ts) + | m >= l = dropChunk (m-l) ts + | otherwise = fromChunks . L.reverse $ + T.dropEnd (int64ToInt m) t : ts + where l = intToInt64 (T.length t) + +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' +-- values dropped, or the empty 'Text' if @n@ is greater than the +-- number of 'Word8' values present. +dropWords :: Int64 -> Text -> Text +dropWords i t0 + | i <= 0 = t0 + | otherwise = drop' i t0 + where + drop' :: Int64 -> Text -> Text + drop' 0 ts = ts + drop' _ Empty = Empty + drop' n (Chunk (T.Text arr off len) ts) + | n < len' = chunk (text arr (off+n') (len-n')) ts + | otherwise = drop' (n - len') ts + where len' = intToInt64 len + n' = int64ToInt n + +-- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', +-- returns the longest prefix (possibly empty) of elements that +-- satisfy @p@. +takeWhile :: (Char -> Bool) -> Text -> Text +takeWhile p t0 = takeWhile' t0 + where takeWhile' Empty = Empty + takeWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n | n > 0 -> Chunk (T.take n t) Empty + | otherwise -> Empty + Nothing -> Chunk t (takeWhile' ts) +{-# INLINE [1] takeWhile #-} + +-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', +-- returns the longest suffix (possibly empty) of elements that +-- satisfy @p@. +-- Examples: +-- +-- > takeWhileEnd (=='o') "foo" == "oo" +-- +-- @since 1.2.2.0 +takeWhileEnd :: (Char -> Bool) -> Text -> Text +takeWhileEnd p = takeChunk empty . L.reverse . toChunks + where takeChunk acc [] = acc + takeChunk acc (t:ts) + | T.lengthWord8 t' < T.lengthWord8 t + = chunk t' acc + | otherwise = takeChunk (Chunk t' acc) ts + where t' = T.takeWhileEnd p t +{-# INLINE takeWhileEnd #-} + +-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after +-- 'takeWhile' @p@ @t@. +dropWhile :: (Char -> Bool) -> Text -> Text +dropWhile p t0 = dropWhile' t0 + where dropWhile' Empty = Empty + dropWhile' (Chunk t ts) = + case T.findIndex (not . p) t of + Just n -> Chunk (T.drop n t) ts + Nothing -> dropWhile' ts +{-# INLINE [1] dropWhile #-} + +-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after +-- dropping characters that satisfy the predicate @p@ from the end of +-- @t@. +-- +-- Examples: +-- +-- > dropWhileEnd (=='.') "foo..." == "foo" +dropWhileEnd :: (Char -> Bool) -> Text -> Text +dropWhileEnd p = go + where go Empty = Empty + go (Chunk t Empty) = if T.null t' + then Empty + else Chunk t' Empty + where t' = T.dropWhileEnd p t + go (Chunk t ts) = case go ts of + Empty -> go (Chunk t Empty) + ts' -> Chunk t ts' +{-# INLINE dropWhileEnd #-} + +-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after +-- dropping characters that satisfy the predicate @p@ from both the +-- beginning and end of @t@. +dropAround :: (Char -> Bool) -> Text -> Text +dropAround p = dropWhile p . dropWhileEnd p +{-# INLINE [1] dropAround #-} + +-- | /O(n)/ Remove leading white space from a string. Equivalent to: +-- +-- > dropWhile isSpace +stripStart :: Text -> Text +stripStart = dropWhile isSpace +{-# INLINE stripStart #-} + +-- | /O(n)/ Remove trailing white space from a string. Equivalent to: +-- +-- > dropWhileEnd isSpace +stripEnd :: Text -> Text +stripEnd = dropWhileEnd isSpace +{-# INLINE [1] stripEnd #-} + +-- | /O(n)/ Remove leading and trailing white space from a string. +-- Equivalent to: +-- +-- > dropAround isSpace +strip :: Text -> Text +strip = dropAround isSpace +{-# INLINE [1] strip #-} + +-- | /O(n)/ 'splitAt' @n t@ returns a pair whose first element is a +-- prefix of @t@ of length @n@, and whose second is the remainder of +-- the string. It is equivalent to @('take' n t, 'drop' n t)@. +splitAt :: Int64 -> Text -> (Text, Text) +splitAt = loop + where + loop :: Int64 -> Text -> (Text, Text) + loop !_ Empty = (empty, empty) + loop n t | n <= 0 = (empty, t) + loop n (Chunk t@(T.Text arr off len) ts) + | n > mx = let (ts', ts'') = loop (n - intToInt64 (T.length t)) ts + in (Chunk t ts', ts'') + | m > 0, m >= len = (Chunk t Empty, ts) + | m > 0 = let t' = T.Text arr off m + t'' = T.Text arr (off+m) (len-m) + in (Chunk t' Empty, Chunk t'' ts) + | otherwise = let (ts', ts'') = loop (n + intToInt64 m) ts + in (Chunk t ts', ts'') + where + mx = intToInt64 P.maxBound + m = T.measureOff (int64ToInt n) t + + +-- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first +-- element is a prefix of @t@ whose chunks contain @n@ 'Word8' +-- values, and whose second is the remainder of the string. +splitAtWord :: Int64 -> Text -> PairS Text Text +splitAtWord !_ Empty = empty :*: empty +splitAtWord x (Chunk c@(T.Text arr off len) cs) + | y >= len = let h :*: t = splitAtWord (x-intToInt64 len) cs + in Chunk c h :*: t + | otherwise = chunk (text arr off y) empty :*: + chunk (text arr (off+y) (len-y)) cs + where y = int64ToInt x + +-- | /O(n+m)/ Find the first instance of @needle@ (which must be +-- non-'null') in @haystack@. The first element of the returned tuple +-- is the prefix of @haystack@ before @needle@ is matched. The second +-- is the remainder of @haystack@, starting with the match. +-- +-- Examples: +-- +-- > breakOn "::" "a::b::c" ==> ("a", "::b::c") +-- > breakOn "/" "foobar" ==> ("foobar", "") +-- +-- Laws: +-- +-- > append prefix match == haystack +-- > where (prefix, match) = breakOn needle haystack +-- +-- If you need to break a string by a substring repeatedly (e.g. you +-- want to break on every instance of a substring), use 'breakOnAll' +-- instead, as it has lower startup overhead. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +breakOn :: HasCallStack => Text -> Text -> (Text, Text) +breakOn pat src + | null pat = emptyError "breakOn" + | otherwise = case indices pat src of + [] -> (src, empty) + (x:_) -> let h :*: t = splitAtWord x src + in (h, t) + +-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string. +-- +-- The first element of the returned tuple is the prefix of @haystack@ +-- up to and including the last match of @needle@. The second is the +-- remainder of @haystack@, following the match. +-- +-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) +breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) + in (reverse b, reverse a) +{-# INLINE breakOnEnd #-} + +-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in +-- @haystack@. Each element of the returned list consists of a pair: +-- +-- * The entire string prior to the /k/th match (i.e. the prefix) +-- +-- * The /k/th match, followed by the remainder of the string +-- +-- Examples: +-- +-- > breakOnAll "::" "" +-- > ==> [] +-- > breakOnAll "/" "a/b/c/" +-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")] +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +-- +-- The @needle@ parameter may not be empty. +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for + -> Text -- ^ @haystack@ in which to search + -> [(Text, Text)] +breakOnAll pat src + | null pat = emptyError "breakOnAll" + | otherwise = go 0 empty src (indices pat src) + where + go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s + h' = append p h + in (h',t) : go x h' t xs + go _ _ _ _ = [] + +-- | /O(n)/ 'break' is like 'span', but the prefix returned is over +-- elements that fail the predicate @p@. +-- +-- >>> T.break (=='c') "180cm" +-- ("180","cm") +break :: (Char -> Bool) -> Text -> (Text, Text) +break p t0 = break' t0 + where break' Empty = (empty, empty) + break' c@(Chunk t ts) = + case T.findIndex p t of + Nothing -> let (ts', ts'') = break' ts + in (Chunk t ts', ts'') + Just n | n == 0 -> (Empty, c) + | otherwise -> let (a,b) = T.splitAt n t + in (Chunk a Empty, Chunk b ts) + +-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns +-- a pair whose first element is the longest prefix (possibly empty) +-- of @t@ of elements that satisfy @p@, and whose second is the +-- remainder of the text. +-- +-- >>> T.span (=='0') "000AB" +-- ("000","AB") +span :: (Char -> Bool) -> Text -> (Text, Text) +span p = break (not . p) +{-# INLINE span #-} + +-- | /O(length of prefix)/ 'spanM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t1@ is the longest prefix of +-- @t@ whose elements satisfy @p@, and @t2@ is the remainder of the text. +-- +-- >>> T.spanM (\c -> state $ \i -> (fromEnum c == i, i+1)) "abcefg" `runState` 97 +-- (("abc","efg"),101) +-- +-- 'span' is 'spanM' specialized to 'Data.Functor.Identity.Identity': +-- +-- @ +-- -- for all p :: Char -> Bool +-- 'span' p = 'Data.Functor.Identity.runIdentity' . 'spanM' ('pure' . p) +-- @ +-- +-- @since 2.0.1 +spanM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t1, t2) <- T.spanM p t + if T.null t2 then first (chunk t) <$> go ts + else pure (chunk t1 empty, Chunk t2 ts) +{-# INLINE spanM #-} + +-- | /O(length of suffix)/ 'spanEndM', applied to a monadic predicate @p@, +-- a text @t@, returns a pair @(t1, t2)@ where @t2@ is the longest suffix of +-- @t@ whose elements satisfy @p@, and @t1@ is the remainder of the text. +-- +-- >>> T.spanEndM (\c -> state $ \i -> (fromEnum c == i, i-1)) "tuvxyz" `runState` 122 +-- (("tuv","xyz"),118) +-- +-- @ +-- 'spanEndM' p . 'reverse' = fmap ('Data.Bifunctor.bimap' 'reverse' 'reverse') . 'spanM' p +-- @ +-- +-- @since 2.0.1 +spanEndM :: Monad m => (Char -> m Bool) -> Text -> m (Text, Text) +spanEndM p t0 = go t0 + where + go Empty = pure (empty, empty) + go (Chunk t ts) = do + (t3, t4) <- go ts + if null t3 then (\(t1, t2) -> (chunk t1 empty, chunk t2 ts)) <$> T.spanEndM p t + else pure (Chunk t t3, t4) +{-# INLINE spanEndM #-} + +-- | The 'group' function takes a 'Text' and returns a list of 'Text's +-- such that the concatenation of the result is equal to the argument. +-- Moreover, each sublist in the result contains only equal elements. +-- For example, +-- +-- > group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"] +-- +-- It is a special case of 'groupBy', which allows the programmer to +-- supply their own equality test. +group :: Text -> [Text] +group = groupBy (==) +{-# INLINE group #-} + +-- | The 'groupBy' function is the non-overloaded version of 'group'. +groupBy :: (Char -> Char -> Bool) -> Text -> [Text] +groupBy _ Empty = [] +groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs + where (ys,zs) = span (eq x) xs + x = T.unsafeHead t + xs = chunk (T.unsafeTail t) ts + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +inits :: Text -> [Text] +inits = (NE.toList P.$!) . initsNE + +-- | /O(n²)/ Return all initial segments of the given 'Text', +-- shortest first. +-- +-- @since 2.1.2 +initsNE :: Text -> NonEmpty Text +initsNE ts0 = Empty NE.:| inits' 0 ts0 + where + inits' :: Int64 -- Number of previous chunks i + -> Text -- The remainder after dropping i chunks from ts0 + -> [Text] -- Prefixes longer than the first i chunks of ts0. + inits' !i (Chunk t ts) = L.map (takeChunks i ts0) (NE.tail (T.initsNE t)) + ++ inits' (i + 1) ts + inits' _ Empty = [] + +takeChunks :: Int64 -> Text -> T.Text -> Text +takeChunks !i (Chunk t ts) lastChunk | i > 0 = Chunk t (takeChunks (i - 1) ts lastChunk) +takeChunks _ _ lastChunk = Chunk lastChunk Empty + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +tails :: Text -> [Text] +tails = (NE.toList P.$!) . tailsNE + +-- | /O(n)/ Return all final segments of the given 'Text', longest +-- first. +-- +-- @since 2.1.2 +tailsNE :: Text -> NonEmpty Text +tailsNE Empty = Empty :| [] +tailsNE ts@(Chunk t ts') + | T.length t == 1 = ts :| tails ts' + | otherwise = ts :| tails (Chunk (T.unsafeTail t) ts') + +-- $split +-- +-- Splitting functions in this library do not perform character-wise +-- copies to create substrings; they just construct new 'Text's that +-- are slices of the original. + +-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text' +-- argument (which cannot be an empty string), consuming the +-- delimiter. An empty delimiter is invalid, and will cause an error +-- to be raised. +-- +-- Examples: +-- +-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"] +-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""] +-- > splitOn "x" "x" == ["",""] +-- +-- and +-- +-- > intercalate s . splitOn s == id +-- > splitOn (singleton c) == split (==c) +-- +-- (Note: the string @s@ to split on above cannot be empty.) +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +splitOn :: HasCallStack + => Text + -- ^ String to split on. If this string is empty, an error + -- will occur. + -> Text + -- ^ Input text. + -> [Text] +splitOn pat src + | null pat = emptyError "splitOn" + | isSingleton pat = split (== head pat) src + | otherwise = go 0 (indices pat src) src + where + go _ [] cs = [cs] + go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs + in h : go (x+l) xs (dropWords l t) + l = foldlChunks (\a (T.Text _ _ b) -> a + intToInt64 b) 0 pat +{-# INLINE [1] splitOn #-} + +{-# RULES +"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t. + splitOn (singleton c) t = split (==c) t + #-} + +-- | /O(n)/ Splits a 'Text' into components delimited by separators, +-- where the predicate returns True for a separator element. The +-- resulting components do not contain the separators. Two adjacent +-- separators result in an empty component in the output. eg. +-- +-- > split (=='a') "aabbaca" == ["","","bb","c",""] +-- > split (=='a') [] == [""] +split :: (Char -> Bool) -> Text -> [Text] +split _ Empty = [Empty] +split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0 + where comb acc (s:[]) Empty = revChunks (s:acc) : [] + comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts + comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts + comb _ [] _ = impossibleError "split" +{-# INLINE split #-} + +-- | /O(n)/ Splits a 'Text' into components of length @k@. The last +-- element may be shorter than the other chunks, depending on the +-- length of the input. Examples: +-- +-- > chunksOf 3 "foobarbaz" == ["foo","bar","baz"] +-- > chunksOf 4 "haskell.org" == ["hask","ell.","org"] +chunksOf :: Int64 -> Text -> [Text] +chunksOf k = go + where + go t = case splitAt k t of + (a,b) | null a -> [] + | otherwise -> a : go b +{-# INLINE chunksOf #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. +lines :: Text -> [Text] +lines Empty = [] +lines t = NE.toList $ go t + where + go :: Text -> NonEmpty Text + go Empty = Empty :| [] + go (Chunk x xs) + -- x is non-empty, so T.lines x is non-empty as well + | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs + | otherwise = case unsnocList (T.lines x) of + Nothing -> impossibleError "lines" + Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls + +prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text +prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! + +unsnocList :: [a] -> Maybe ([a], a) +unsnocList [] = Nothing +unsnocList (x : xs) = Just $ go x xs + where + go y [] = ([], y) + go y (z : zs) = first (y :) (go z zs) + +hasNlEnd :: T.Text -> Bool +hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A + +-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's +-- representing white space. +words :: Text -> [Text] +words = L.filter (not . null) . split isSpace +{-# INLINE words #-} + +-- | /O(n)/ Joins lines, after appending a terminating newline to +-- each. +unlines :: [Text] -> Text +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] +{-# INLINE unlines #-} + +-- | /O(n)/ Joins words using single space characters. +unwords :: [Text] -> Text +unwords = intercalate (singleton ' ') +{-# INLINE unwords #-} + +-- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a prefix of the second. +isPrefixOf :: Text -> Text -> Bool +isPrefixOf Empty _ = True +isPrefixOf _ Empty = False +isPrefixOf (Chunk x xs) (Chunk y ys) + | lx == ly = x == y && isPrefixOf xs ys + | lx < ly = x == yh && isPrefixOf xs (Chunk yt ys) + | otherwise = xh == y && isPrefixOf (Chunk xt xs) ys + where (xh,xt) = T.splitAt ly x + (yh,yt) = T.splitAt lx y + lx = T.length x + ly = T.length y + +-- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is a suffix of the second. +isSuffixOf :: Text -> Text -> Bool +isSuffixOf x y = reverse x `isPrefixOf` reverse y +{-# INLINE isSuffixOf #-} +-- TODO: a better implementation + +-- | /O(n+m)/ The 'isInfixOf' function takes two 'Text's and returns +-- 'True' if and only if the first is contained, wholly and intact, anywhere +-- within the second. +-- +-- This function is strict in its first argument, and lazy in its +-- second. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +isInfixOf :: Text -> Text -> Bool +isInfixOf needle haystack + | null needle = True + | isSingleton needle = S.elem (head needle) . S.stream $ haystack + | otherwise = not . L.null . indices needle $ haystack +{-# INLINE [1] isInfixOf #-} + +------------------------------------------------------------------------------- +-- * View patterns + +-- | /O(n)/ Return the suffix of the second string if its prefix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripPrefix "foo" "foobar" == Just "bar" +-- > stripPrefix "" "baz" == Just "baz" +-- > stripPrefix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > fnordLength :: Text -> Int +-- > fnordLength (stripPrefix "fnord" -> Just suf) = T.length suf +-- > fnordLength _ = -1 +stripPrefix :: Text -> Text -> Maybe Text +stripPrefix p t + | null p = Just t + | otherwise = case commonPrefixes p t of + Just (_,c,r) | null c -> Just r + _ -> Nothing + +-- | /O(n)/ Find the longest non-empty common prefix of two strings +-- and return it, along with the suffixes of each string at which they +-- no longer match. +-- +-- If the strings do not have a common prefix or either one is empty, +-- this function returns 'Nothing'. +-- +-- Examples: +-- +-- > commonPrefixes "foobar" "fooquux" == Just ("foo","bar","quux") +-- > commonPrefixes "veeble" "fetzer" == Nothing +-- > commonPrefixes "" "baz" == Nothing +commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) +commonPrefixes Empty _ = Nothing +commonPrefixes _ Empty = Nothing +commonPrefixes a0 b0 = Just (go a0 b0 []) + where + go t0@(Chunk x xs) t1@(Chunk y ys) ps + = case T.commonPrefixes x y of + Just (p,a,b) + | T.null a -> go xs (chunk b ys) (p:ps) + | T.null b -> go (chunk a xs) ys (p:ps) + | otherwise -> (fromChunks (L.reverse (p:ps)),chunk a xs, chunk b ys) + Nothing -> (fromChunks (L.reverse ps),t0,t1) + go t0 t1 ps = (fromChunks (L.reverse ps),t0,t1) + +-- | /O(n)/ Return the prefix of the second string if its suffix +-- matches the entire first string. +-- +-- Examples: +-- +-- > stripSuffix "bar" "foobar" == Just "foo" +-- > stripSuffix "" "baz" == Just "baz" +-- > stripSuffix "foo" "quux" == Nothing +-- +-- This is particularly useful with the @ViewPatterns@ extension to +-- GHC, as follows: +-- +-- > {-# LANGUAGE ViewPatterns #-} +-- > import Data.Text.Lazy as T +-- > +-- > quuxLength :: Text -> Int +-- > quuxLength (stripSuffix "quux" -> Just pre) = T.length pre +-- > quuxLength _ = -1 +stripSuffix :: Text -> Text -> Maybe Text +stripSuffix p t = reverse `fmap` stripPrefix (reverse p) (reverse t) + +-- | /O(n)/ 'filter', applied to a predicate and a 'Text', +-- returns a 'Text' containing those characters that satisfy the +-- predicate. +filter :: (Char -> Bool) -> Text -> Text +filter p = foldrChunks (chunk . filter_ T.Text p) Empty +{-# INLINE [1] filter #-} + +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> q c && p c) t +#-} + +-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and +-- returns the first element in matching the predicate, or 'Nothing' +-- if there is no such element. +find :: (Char -> Bool) -> Text -> Maybe Char +find p t = S.findBy p (stream t) +{-# INLINE find #-} + +-- | /O(n)/ The 'elem' function takes a character and a 'Text', and +-- returns 'True' if the element is found in the given 'Text', or +-- 'False' otherwise. +elem :: Char -> Text -> Bool +elem c t = S.any (== c) (stream t) +{-# INLINE elem #-} + +-- | /O(n)/ The 'partition' function takes a predicate and a 'Text', +-- and returns the pair of 'Text's with elements which do and do not +-- satisfy the predicate, respectively; i.e. +-- +-- > partition p t == (filter p t, filter (not . p) t) +partition :: (Char -> Bool) -> Text -> (Text, Text) +partition p t = (filter p t, filter (not . p) t) +{-# INLINE partition #-} + +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int64 -> Char +index lazyText ix + | ix < 0 = P.error $ "Data.Text.Lazy.index: negative index " ++ P.show ix + | otherwise = go lazyText ix + where + go :: Text -> Int64 -> Char + go Empty _ = P.error $ "Data.Text.index: index " ++ P.show ix ++ " is too large" + go (Chunk t@(T.Text _ _ lenInBytes) ts) n = case toIntegralSized n of + Nothing -> + go ts (n - fromIntegral (T.length t)) + Just n' + | off < 0 -> go ts (n + fromIntegral off) + | off == lenInBytes -> go ts 0 + | otherwise -> ch + where + off = T.measureOff n' t + T.Iter ch _ = T.iter t off +{-# INLINE index #-} + +-- | /O(n+m)/ The 'count' function returns the number of times the +-- query string appears in the given 'Text'. An empty query string is +-- invalid, and will cause an error to be raised. +-- +-- In (unlikely) bad cases, this function's time complexity degrades +-- towards /O(n*m)/. +count :: HasCallStack => Text -> Text -> Int64 +count pat + | null pat = emptyError "count" + | otherwise = go 0 . indices pat + where go !n [] = n + go !n (_:xs) = go (n+1) xs +{-# INLINE [1] count #-} + +{-# RULES +"LAZY TEXT count/singleton -> countChar" [~1] forall c t. + count (singleton c) t = countChar c t + #-} + +-- | /O(n)/ The 'countChar' function returns the number of times the +-- query element appears in the given 'Text'. +countChar :: Char -> Text -> Int64 +countChar c t = S.countChar c (stream t) + +-- | /O(n)/ 'zip' takes two 'Text's and returns a list of +-- corresponding pairs of bytes. If one input 'Text' is short, +-- excess elements of the longer 'Text' are discarded. This is +-- equivalent to a pair of 'unpack' operations. +zip :: Text -> Text -> [(Char,Char)] +zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b) +{-# INLINE [0] zip #-} + +-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function +-- given as the first argument, instead of a tupling function. +-- Performs replacement on invalid scalar values. +zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text +zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) + where g a b = safe (f a b) +{-# INLINE [0] zipWith #-} + +-- | Convert a value to lazy 'Text'. +-- +-- @since 2.1.2 +show :: Show a => a -> Text +show = pack . P.show + +revChunks :: [T.Text] -> Text +revChunks = L.foldl' (flip chunk) Empty + +emptyError :: HasCallStack => String -> a +emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") + +impossibleError :: HasCallStack => String -> a +impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") + +intToInt64 :: Exts.Int -> Int64 +intToInt64 = fromIntegral + +int64ToInt :: Int64 -> Exts.Int +int64ToInt = fromIntegral
src/Data/Text/Lazy/Builder/RealFloat.hs view
@@ -1,6 +1,9 @@ {-# LANGUAGE CPP, OverloadedStrings #-} {-# LANGUAGE Trustworthy #-} +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}+ -- | -- Module: Data.Text.Lazy.Builder.RealFloat -- Copyright: (c) The University of Glasgow 1994-2002
src/Data/Text/Lazy/Encoding.hs view
@@ -1,244 +1,254 @@-{-# LANGUAGE BangPatterns,CPP #-}-{-# LANGUAGE Trustworthy #-}-{-# LANGUAGE ViewPatterns #-}---- |--- Module : Data.Text.Lazy.Encoding--- Copyright : (c) 2009, 2010 Bryan O'Sullivan------ License : BSD-style--- Maintainer : bos@serpentine.com--- Portability : portable------ Functions for converting lazy 'Text' values to and from lazy--- 'ByteString', using several standard encodings.------ To gain access to a much larger family of encodings, use the--- <http://hackage.haskell.org/package/text-icu text-icu package>.--module Data.Text.Lazy.Encoding- (- -- * Decoding ByteStrings to Text- -- $strict-- -- ** Total Functions #total#- -- $total- decodeLatin1-- -- *** Catchable failure- , decodeUtf8'-- -- *** Controllable error handling- , decodeUtf8With- , decodeUtf16LEWith- , decodeUtf16BEWith- , decodeUtf32LEWith- , decodeUtf32BEWith-- -- ** Partial Functions- -- $partial- , decodeASCII- , decodeUtf8- , decodeUtf16LE- , decodeUtf16BE- , decodeUtf32LE- , decodeUtf32BE-- -- * Encoding Text to ByteStrings- , encodeUtf8- , encodeUtf16LE- , encodeUtf16BE- , encodeUtf32LE- , encodeUtf32BE-- -- * Encoding Text using ByteString Builders- , encodeUtf8Builder- , encodeUtf8BuilderEscaped- ) where--import Control.Exception (evaluate, try)-import Data.Monoid (Monoid(..))-import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)-import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)-import Data.Word (Word8)-import qualified Data.ByteString.Builder as B-import qualified Data.ByteString.Builder.Prim as BP-import qualified Data.ByteString.Lazy as B-import qualified Data.ByteString.Lazy.Internal as B-import qualified Data.Text.Encoding as TE-import qualified Data.Text.Internal.Encoding as TE-import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E-import qualified Data.Text.Internal.Lazy.Fusion as F-import qualified Data.Text.Internal.StrictBuilder as SB-import Data.Text.Unsafe (unsafeDupablePerformIO)---- $strict------ All of the single-parameter functions for decoding bytestrings--- encoded in one of the Unicode Transformation Formats (UTF) operate--- in a /strict/ mode: each will throw an exception if given invalid--- input.------ Each function has a variant, whose name is suffixed with -'With',--- that gives greater control over the handling of decoding errors.--- For instance, 'decodeUtf8' will throw an exception, but--- 'decodeUtf8With' allows the programmer to determine what to do on a--- decoding error.---- $total------ These functions facilitate total decoding and should be preferred--- over their partial counterparts.---- $partial------ These functions are partial and should only be used with great caution--- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better--- solutions.---- | Decode a 'ByteString' containing 7-bit ASCII--- encoded text.-decodeASCII :: B.ByteString -> Text-decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks---- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text.-decodeLatin1 :: B.ByteString -> Text-decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks---- | Decode a 'ByteString' containing UTF-8 encoded text.-decodeUtf8With :: OnDecodeError -> B.ByteString -> Text-decodeUtf8With onErr = loop TE.startUtf8State- where- chunkb builder t | SB.sbLength builder == 0 = t- | otherwise = Chunk (TE.strictBuilderToText builder) t- loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of- (builder, _, s') -> chunkb builder (loop s' bs)- loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty- msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream"---- | Decode a 'ByteString' containing UTF-8 encoded text that is known--- to be valid.------ If the input contains any invalid UTF-8 data, an exception will be--- thrown that cannot be caught in pure code. For more control over--- the handling of invalid data, use 'decodeUtf8'' or--- 'decodeUtf8With'.-decodeUtf8 :: B.ByteString -> Text-decodeUtf8 = decodeUtf8With strictDecode-{-# INLINE[0] decodeUtf8 #-}---- | Decode a 'ByteString' containing UTF-8 encoded text..------ If the input contains any invalid UTF-8 data, the relevant--- exception will be returned, otherwise the decoded text.------ /Note/: this function is /not/ lazy, as it must decode its entire--- input before it can return a result. If you need lazy (streaming)--- decoding, use 'decodeUtf8With' in lenient mode.-decodeUtf8' :: B.ByteString -> Either UnicodeException Text-decodeUtf8' bs = unsafeDupablePerformIO $ do- let t = decodeUtf8 bs- try (evaluate (rnf t `seq` t))- where- rnf Empty = ()- rnf (Chunk _ ts) = rnf ts-{-# INLINE decodeUtf8' #-}---- | Encode text using UTF-8 encoding.-encodeUtf8 :: Text -> B.ByteString-encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty---- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.------ @since 1.1.0.0-encodeUtf8Builder :: Text -> B.Builder-encodeUtf8Builder =- foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty---- | Encode text using UTF-8 encoding and escape the ASCII characters using--- a 'BP.BoundedPrim'.------ Use this function is to implement efficient encoders for text-based formats--- like JSON or HTML.------ @since 1.1.0.0-{-# INLINE encodeUtf8BuilderEscaped #-}-encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder-encodeUtf8BuilderEscaped prim =- foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty---- | Decode text from little endian UTF-16 encoding.-decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)-{-# INLINE decodeUtf16LEWith #-}---- | Decode text from little endian UTF-16 encoding.------ If the input contains any invalid little endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16LEWith'.-decodeUtf16LE :: B.ByteString -> Text-decodeUtf16LE = decodeUtf16LEWith strictDecode-{-# INLINE decodeUtf16LE #-}---- | Decode text from big endian UTF-16 encoding.-decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)-{-# INLINE decodeUtf16BEWith #-}---- | Decode text from big endian UTF-16 encoding.------ If the input contains any invalid big endian UTF-16 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf16BEWith'.-decodeUtf16BE :: B.ByteString -> Text-decodeUtf16BE = decodeUtf16BEWith strictDecode-{-# INLINE decodeUtf16BE #-}---- | Encode text using little endian UTF-16 encoding.-encodeUtf16LE :: Text -> B.ByteString-encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt)-{-# INLINE encodeUtf16LE #-}---- | Encode text using big endian UTF-16 encoding.-encodeUtf16BE :: Text -> B.ByteString-encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt)-{-# INLINE encodeUtf16BE #-}---- | Decode text from little endian UTF-32 encoding.-decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)-{-# INLINE decodeUtf32LEWith #-}---- | Decode text from little endian UTF-32 encoding.------ If the input contains any invalid little endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32LEWith'.-decodeUtf32LE :: B.ByteString -> Text-decodeUtf32LE = decodeUtf32LEWith strictDecode-{-# INLINE decodeUtf32LE #-}---- | Decode text from big endian UTF-32 encoding.-decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text-decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)-{-# INLINE decodeUtf32BEWith #-}---- | Decode text from big endian UTF-32 encoding.------ If the input contains any invalid big endian UTF-32 data, an--- exception will be thrown. For more control over the handling of--- invalid data, use 'decodeUtf32BEWith'.-decodeUtf32BE :: B.ByteString -> Text-decodeUtf32BE = decodeUtf32BEWith strictDecode-{-# INLINE decodeUtf32BE #-}---- | Encode text using little endian UTF-32 encoding.-encodeUtf32LE :: Text -> B.ByteString-encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt)-{-# INLINE encodeUtf32LE #-}---- | Encode text using big endian UTF-32 encoding.-encodeUtf32BE :: Text -> B.ByteString-encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt)-{-# INLINE encodeUtf32BE #-}+{-# LANGUAGE BangPatterns,CPP #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module : Data.Text.Lazy.Encoding +-- Copyright : (c) 2009, 2010 Bryan O'Sullivan +-- +-- License : BSD-style +-- Maintainer : bos@serpentine.com +-- Portability : portable +-- +-- Functions for converting lazy 'Text' values to and from lazy +-- 'ByteString', using several standard encodings. +-- +-- To gain access to a much larger family of encodings, use the +-- <http://hackage.haskell.org/package/text-icu text-icu package>. + +module Data.Text.Lazy.Encoding + ( + -- * Decoding ByteStrings to Text + -- $strict + + -- ** Total Functions #total# + -- $total + decodeLatin1 + , decodeUtf8Lenient + + -- *** Catchable failure + , decodeUtf8' + + -- *** Controllable error handling + , decodeUtf8With + , decodeUtf16LEWith + , decodeUtf16BEWith + , decodeUtf32LEWith + , decodeUtf32BEWith + + -- ** Partial Functions + -- $partial + , decodeASCII + , decodeUtf8 + , decodeUtf16LE + , decodeUtf16BE + , decodeUtf32LE + , decodeUtf32BE + + -- * Encoding Text to ByteStrings + , encodeUtf8 + , encodeUtf16LE + , encodeUtf16BE + , encodeUtf32LE + , encodeUtf32BE + + -- * Encoding Text using ByteString Builders + , encodeUtf8Builder + , encodeUtf8BuilderEscaped + ) where + +import Control.Exception (evaluate, try) +import Data.Monoid (Monoid(..)) +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks) +import Data.Word (Word8) +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Lazy as B +import qualified Data.ByteString.Lazy.Internal as B +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Internal.Encoding as TE +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as F +import qualified Data.Text.Internal.StrictBuilder as SB +import Data.Text.Unsafe (unsafeDupablePerformIO) + +-- $strict +-- +-- All of the single-parameter functions for decoding bytestrings +-- encoded in one of the Unicode Transformation Formats (UTF) operate +-- in a /strict/ mode: each will throw an exception if given invalid +-- input. +-- +-- Each function has a variant, whose name is suffixed with -'With', +-- that gives greater control over the handling of decoding errors. +-- For instance, 'decodeUtf8' will throw an exception, but +-- 'decodeUtf8With' allows the programmer to determine what to do on a +-- decoding error. + +-- $total +-- +-- These functions facilitate total decoding and should be preferred +-- over their partial counterparts. + +-- $partial +-- +-- These functions are partial and should only be used with great caution +-- (preferably not at all). See "Data.Text.Lazy.Encoding#g:total" for better +-- solutions. + +-- | Decode a 'ByteString' containing 7-bit ASCII +-- encoded text. +decodeASCII :: B.ByteString -> Text +decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks + +-- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. +decodeLatin1 :: B.ByteString -> Text +decodeLatin1 = foldr (chunk . TE.decodeLatin1) empty . B.toChunks + +-- | Decode a 'ByteString' containing UTF-8 encoded text. +decodeUtf8With :: OnDecodeError -> B.ByteString -> Text +decodeUtf8With onErr = loop TE.startUtf8State + where + chunkb builder t | SB.sbLength builder == 0 = t + | otherwise = Chunk (TE.strictBuilderToText builder) t + loop s (B.Chunk b bs) = case TE.decodeUtf8With2 onErr msg s b of + (builder, _, s') -> chunkb builder (loop s' bs) + loop s B.Empty = chunkb (TE.skipIncomplete onErr msg s) Empty + msg = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" + +-- | Decode a 'ByteString' containing UTF-8 encoded text that is known +-- to be valid. +-- +-- If the input contains any invalid UTF-8 data, an exception will be +-- thrown that cannot be caught in pure code. For more control over +-- the handling of invalid data, use 'decodeUtf8'' or +-- 'decodeUtf8With'. +decodeUtf8 :: B.ByteString -> Text +decodeUtf8 = decodeUtf8With strictDecode +{-# INLINE[0] decodeUtf8 #-} + +-- | Decode a 'ByteString' containing UTF-8 encoded text.. +-- +-- If the input contains any invalid UTF-8 data, the relevant +-- exception will be returned, otherwise the decoded text. +-- +-- /Note/: this function is /not/ lazy, as it must decode its entire +-- input before it can return a result. If you need lazy (streaming) +-- decoding, use 'decodeUtf8With' in lenient mode. +decodeUtf8' :: B.ByteString -> Either UnicodeException Text +decodeUtf8' bs = unsafeDupablePerformIO $ do + let t = decodeUtf8 bs + try (evaluate (rnf t `seq` t)) + where + rnf Empty = () + rnf (Chunk _ ts) = rnf ts +{-# INLINE decodeUtf8' #-} + +-- | Decode a lazy 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +-- +-- @since 2.1.4 +decodeUtf8Lenient :: B.ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + +-- | Encode text using UTF-8 encoding. +encodeUtf8 :: Text -> B.ByteString +encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty + +-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. +-- +-- @since 1.1.0.0 +encodeUtf8Builder :: Text -> B.Builder +encodeUtf8Builder = + foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty + +-- | Encode text using UTF-8 encoding and escape the ASCII characters using +-- a 'BP.BoundedPrim'. +-- +-- Use this function is to implement efficient encoders for text-based formats +-- like JSON or HTML. +-- +-- @since 1.1.0.0 +{-# INLINE encodeUtf8BuilderEscaped #-} +encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder +encodeUtf8BuilderEscaped prim = + foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty + +-- | Decode text from little endian UTF-16 encoding. +decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs) +{-# INLINE decodeUtf16LEWith #-} + +-- | Decode text from little endian UTF-16 encoding. +-- +-- If the input contains any invalid little endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16LEWith'. +decodeUtf16LE :: B.ByteString -> Text +decodeUtf16LE = decodeUtf16LEWith strictDecode +{-# INLINE decodeUtf16LE #-} + +-- | Decode text from big endian UTF-16 encoding. +decodeUtf16BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs) +{-# INLINE decodeUtf16BEWith #-} + +-- | Decode text from big endian UTF-16 encoding. +-- +-- If the input contains any invalid big endian UTF-16 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf16BEWith'. +decodeUtf16BE :: B.ByteString -> Text +decodeUtf16BE = decodeUtf16BEWith strictDecode +{-# INLINE decodeUtf16BE #-} + +-- | Encode text using little endian UTF-16 encoding. +encodeUtf16LE :: Text -> B.ByteString +encodeUtf16LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16LE) [] txt) +{-# INLINE encodeUtf16LE #-} + +-- | Encode text using big endian UTF-16 encoding. +encodeUtf16BE :: Text -> B.ByteString +encodeUtf16BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf16BE) [] txt) +{-# INLINE encodeUtf16BE #-} + +-- | Decode text from little endian UTF-32 encoding. +decodeUtf32LEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs) +{-# INLINE decodeUtf32LEWith #-} + +-- | Decode text from little endian UTF-32 encoding. +-- +-- If the input contains any invalid little endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32LEWith'. +decodeUtf32LE :: B.ByteString -> Text +decodeUtf32LE = decodeUtf32LEWith strictDecode +{-# INLINE decodeUtf32LE #-} + +-- | Decode text from big endian UTF-32 encoding. +decodeUtf32BEWith :: OnDecodeError -> B.ByteString -> Text +decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs) +{-# INLINE decodeUtf32BEWith #-} + +-- | Decode text from big endian UTF-32 encoding. +-- +-- If the input contains any invalid big endian UTF-32 data, an +-- exception will be thrown. For more control over the handling of +-- invalid data, use 'decodeUtf32BEWith'. +decodeUtf32BE :: B.ByteString -> Text +decodeUtf32BE = decodeUtf32BEWith strictDecode +{-# INLINE decodeUtf32BE #-} + +-- | Encode text using little endian UTF-32 encoding. +encodeUtf32LE :: Text -> B.ByteString +encodeUtf32LE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32LE) [] txt) +{-# INLINE encodeUtf32LE #-} + +-- | Encode text using big endian UTF-32 encoding. +encodeUtf32BE :: Text -> B.ByteString +encodeUtf32BE txt = B.fromChunks (foldrChunks ((:) . TE.encodeUtf32BE) [] txt) +{-# INLINE encodeUtf32BE #-}
src/Data/Text/Lazy/Internal.hs view
@@ -1,4 +1,3 @@-{-# LANGUAGE BangPatterns, DeriveDataTypeable #-} -- | -- Module : Data.Text.Lazy.Internal -- Copyright : (c) 2013 Bryan O'Sullivan
src/Data/Text/Show.hs view
@@ -31,6 +31,7 @@ import Data.Text.Internal.Unsafe.Char (unsafeWrite) import Data.Text.Unsafe (Iter(..), iterArray) import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#)+import qualified GHC.Exts as Exts import GHC.Word (Word8(..)) import qualified Data.Text.Array as A #if !MIN_VERSION_ghc_prim(0,7,0)@@ -53,12 +54,32 @@ HasCallStack => #endif Text -> String-unpack (Text arr off len) = go off+unpack t = foldrText (:) [] t+{-# NOINLINE unpack #-}++foldrText :: (Char -> b -> b) -> b -> Text -> b+foldrText f z (Text arr off len) = go off where go !i- | i >= off + len = []- | otherwise = let !(Iter c l) = iterArray arr i in c : go (i + l)-{-# INLINE [1] unpack #-}+ | i >= off + len = z+ | otherwise = let !(Iter c l) = iterArray arr i in f c (go (i + l))+{-# INLINE foldrText #-}++foldrTextFB :: (Char -> b -> b) -> b -> Text -> b+foldrTextFB = foldrText+{-# INLINE [0] foldrTextFB #-}++-- List fusion rules for `unpack`:+-- * `unpack` rewrites to `build` up till (but not including) phase 1. `build`+-- fuses if `foldr` is applied to it.+-- * If it doesn't fuse: In phase 1, `build` inlines to give us+-- `foldrTextFB (:) []` and we rewrite that back to `unpack`.+-- * If it fuses: In phase 0, `foldrTextFB` inlines and `foldrText` inlines. GHC+-- optimizes the fused code.+{-# RULES+"Text.unpack" [~1] forall t. unpack t = Exts.build (\lcons lnil -> foldrTextFB lcons lnil t)+"Text.unpackBack" [1] foldrTextFB (:) [] = unpack+ #-} -- | /O(n)/ Convert a null-terminated -- <https://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 modified UTF-8>
tests/Tests/Lift.hs view
@@ -1,34 +1,39 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TemplateHaskell #-}-module Tests.Lift- ( tests- )- where--import qualified Data.Text as S-import qualified Data.Text.Lazy as L-import Language.Haskell.TH.Syntax (lift)-import Test.Tasty.HUnit (testCase, assertEqual)-import Test.Tasty (TestTree, testGroup)--tests :: TestTree-tests = testGroup "TH lifting Text"- [ testCase "strict" $ assertEqual "strict"- $(lift ("foo" :: S.Text))- ("foo" :: S.Text)- , testCase "strict0" $ assertEqual "strict0"- $(lift ("f\0o\1o\2" :: S.Text))- ("f\0o\1o\2" :: S.Text)- , testCase "strict-nihao" $ assertEqual "strict-nihao"- $(lift ("\20320\22909" :: S.Text))- ("\20320\22909" :: S.Text)- , testCase "lazy" $ assertEqual "lazy"- $(lift ("foo" :: L.Text))- ("foo" :: L.Text)- , testCase "lazy0" $ assertEqual "lazy0"- $(lift ("f\0o\1o\2" :: L.Text))- ("f\0o\1o\2" :: L.Text)- , testCase "lazy-nihao" $ assertEqual "lazy-nihao"- $(lift ("\20320\22909" :: L.Text))- ("\20320\22909" :: L.Text)- ]+{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +module Tests.Lift + ( tests + ) + where + +import qualified Data.Text as S +import qualified Data.Text.Lazy as L +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) + +tests :: TestTree +tests = testGroup "TH lifting Text" + [ testCase "strict" $ assertEqual "strict" + $(lift ("foo" :: S.Text)) + ("foo" :: S.Text) + , testCase "strict0" $ assertEqual "strict0" + $(lift ("f\0o\1o\2" :: S.Text)) + ("f\0o\1o\2" :: S.Text) + , testCase "strict-nihao" $ assertEqual "strict-nihao" + $(lift ("\20320\22909" :: S.Text)) + ("\20320\22909" :: S.Text) + , testCase "lazy" $ assertEqual "lazy" + $(lift ("foo" :: L.Text)) + ("foo" :: L.Text) + , testCase "lazy0" $ assertEqual "lazy0" + $(lift ("f\0o\1o\2" :: L.Text)) + ("f\0o\1o\2" :: L.Text) + , testCase "lazy-nihao" $ assertEqual "lazy-nihao" + $(lift ("\20320\22909" :: L.Text)) + ("\20320\22909" :: L.Text) + ]
tests/Tests/Properties.hs view
@@ -17,6 +17,7 @@ import Tests.Properties.Text (testText) import Tests.Properties.Transcoding (testTranscoding) import Tests.Properties.Validate (testValidate)+import Tests.Properties.CornerCases (testCornerCases) tests :: TestTree tests =@@ -30,5 +31,6 @@ testBuilder, testLowLevel, testRead,+ testCornerCases, testValidate ]
+ tests/Tests/Properties/CornerCases.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}++-- | Check that the definitions that are partial crash in the expected ways or+-- return sensible defaults.+module Tests.Properties.CornerCases (testCornerCases) where++import Control.Exception+import Data.Either+import Data.Semigroup+import Data.Text+import Test.QuickCheck+import Test.Tasty (TestTree, testGroup)+import Test.Tasty.QuickCheck (testProperty)+import Tests.QuickCheckUtils ()++testCornerCases :: TestTree+testCornerCases =+ testGroup+ "corner cases"+ [ testGroup+ "stimes"+ $ let specimen = stimes :: Integer -> Text -> Text+ in [ testProperty+ "given a negative number, evaluate to error call"+ $ \(Negative number) text ->+ (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+ specimen+ (fromIntegral (number :: Int))+ text+ , testProperty+ "given a number that does not fit into Int, evaluate to error call"+ $ \(NonNegative number) text ->+ (ioProperty . fmap isLeft . try @ErrorCall . evaluate) $+ specimen+ (fromIntegral (number :: Int) + fromIntegral (maxBound :: Int) + 1)+ text+ ]+ ]
tests/Tests/Properties/Folds.hs view
@@ -1,342 +1,357 @@--- | Test folds, scans, and unfolds--{-# LANGUAGE CPP #-}-{-# LANGUAGE ViewPatterns #-}--{-# OPTIONS_GHC -fno-warn-missing-signatures #-}--#ifdef MIN_VERSION_tasty_inspection_testing-{-# LANGUAGE TemplateHaskell #-}-{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-}-#endif--module Tests.Properties.Folds- ( testFolds- ) where--import Control.Arrow (second)-import Control.Exception (ErrorCall, evaluate, try)-import Data.Functor.Identity (Identity(..))-import Control.Monad.Trans.State (runState, state)-import Data.Word (Word8, Word16)-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.HUnit (testCase, assertFailure, assertBool)-import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2)-import Tests.QuickCheckUtils-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Lazy as TL-import qualified Data.Char as Char--#ifdef MIN_VERSION_tasty_inspection_testing-import Test.Tasty.Inspection (inspectTest, (==~))-import GHC.Exts (inline)-#endif---- Folds--sf_foldl (applyFun -> p) (applyFun2 -> f) z =- (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z)- where _types = f :: Char -> Char -> Char-tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z)- where _types = f :: Char -> Char -> Char-sf_foldl' (applyFun -> p) (applyFun2 -> f) z =- (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z- where _types = f :: Char -> Char -> Char-tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z- where _types = f :: Char -> Char -> Char-sf_foldl1 (applyFun -> p) (applyFun2 -> f) =- (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p)-t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f-tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f-sf_foldl1' (applyFun -> p) (applyFun2 -> f) =- (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p)-t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f-tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f-sf_foldr (applyFun -> p) (applyFun2 -> f) z =- (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p)- where _types = f :: Char -> Char -> Char-t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z- where _types = f :: Char -> Char -> Char-t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z- where _types = f :: Char -> Char -> Char-tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z- where _types = f :: Char -> Char -> Char-sf_foldr1 (applyFun -> p) (applyFun2 -> f) =- (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p)-t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f-tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f---- Distinguish foldl/foldr from foldl'/foldr'--fold_apart :: IO ()-fold_apart = do- ok (T.foldr f () (T.pack "az"))- ko (T.foldr' f () (T.pack "az"))- ok (T.foldl (flip f) () (T.pack "za"))- ko (T.foldl' (flip f) () (T.pack "za"))- where- f c _ = if c == 'z' then error "catchme" else ()- ok = evaluate- ko t = do- x <- try (evaluate t)- case x :: Either ErrorCall () of- Left _ -> pure ()- Right _ -> assertFailure "test should have failed but didn't"---- Special folds--s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt)-sf_concat (applyFun -> p)- = (L.concat . map (L.filter p) . unSqrt) `eq`- (unpackS . S.concat . map (S.filter p . packS) . unSqrt)-t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt)-tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt)-sf_concatMap (applyFun -> p) (applyFun -> f) =- (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p)-t_concatMap (applyFun -> f)- = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f))-tl_concatMap (applyFun -> f)- = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f))-sf_any (applyFun -> q) (applyFun -> p)- = (L.any p . L.filter q) `eqP` (S.any p . S.filter q)-t_any (applyFun -> p)- = L.any p `eqP` T.any p-tl_any (applyFun -> p)- = L.any p `eqP` TL.any p-sf_all (applyFun -> q) (applyFun -> p)- = (L.all p . L.filter q) `eqP` (S.all p . S.filter q)-t_all (applyFun -> p)- = L.all p `eqP` T.all p-tl_all (applyFun -> p)- = L.all p `eqP` TL.all p-sf_maximum (applyFun -> p)- = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p)-t_maximum = L.maximum `eqP` T.maximum-tl_maximum = L.maximum `eqP` TL.maximum-sf_minimum (applyFun -> p)- = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p)-t_minimum = L.minimum `eqP` T.minimum-tl_minimum = L.minimum `eqP` TL.minimum-t_isAscii = L.all Char.isAscii `eqP` T.isAscii-tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii---- Scans--sf_scanl (applyFun -> p) (applyFun2 -> f) z =- (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p)-t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z)-tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z)-t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f)-tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f)-t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z)-tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z)-t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f)-tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f)--t_mapAccumL_char c t =- snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)-t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z)- where _types = f :: Int -> Char -> (Int,Char)-tl_mapAccumL_char c t =- snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)-tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z)- where _types = f :: Int -> Char -> (Int,Char)-t_mapAccumR_char c t =- snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c)-t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)- where _types = f :: Int -> Char -> (Int,Char)-tl_mapAccumR_char c t =- snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c)-tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)- where _types = f :: Int -> Char -> (Int,Char)---- Unfolds--tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat)--s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS)--t_replicate_char (Small n) c =- L.replicate n c === T.unpack (T.replicate n (T.singleton c))-tl_replicate_char (Small n) c =- L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c))-t_length_replicate_char (Small n) c =- L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c))-tl_length_replicate_char (Small n) c =- L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c))--t_replicate (Small n) =- (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n)-tl_replicate (Small n) =- (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n))-t_length_replicate (Small n) =- (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n)-tl_length_replicate (Small n) =- (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n))--tl_cycle n = (L.take m . L.cycle) `eq`- (unpackS . TL.take (fromIntegral m) . TL.cycle . packS)- where m = fromIntegral (n :: Word8)--tl_iterate (applyFun -> f) n- = (L.take m . L.iterate f) `eq`- (unpackS . TL.take (fromIntegral m) . TL.iterate f)- where m = fromIntegral (n :: Word8)--unf :: Int -> Char -> Maybe (Char, Char)-unf n c | fromEnum c * 100 > n = Nothing- | otherwise = Just (c, succ c)--t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m))- where m = fromIntegral (n :: Word16)-tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m))- where m = fromIntegral (n :: Word16)-t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq`- (unpackS . T.unfoldrN i (unf j))- where i = fromIntegral (n :: Word16)- j = fromIntegral (m :: Word16)-tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq`- (unpackS . TL.unfoldrN (fromIntegral i) (unf j))- where i = fromIntegral (n :: Word16)- j = fromIntegral (m :: Word16)---- Monadic folds---- Parametric polymorphism allows us to only test foldlM' specialized to--- one function in the state monad (called @logger@ in the following tests)--- that just logs the arguments it was applied to and produces a fresh--- accumulator. That alone determines the general behavior of foldlM' with an--- arbitrary function in any monad.--- Reference: "Testing Polymorphic Properties" by Bernardy et al.--- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf--t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0)- where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order-tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0)- where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order--#ifdef MIN_VERSION_tasty_inspection_testing--- As a sanity check for performance, the simplified Core--- foldlM' specialized to Identity is the same as foldl'.--_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a-_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x--_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a-_S_foldl' = inline S.foldl'-#endif--isAscii_border :: IO ()-isAscii_border = do- let text = T.drop 2 $ T.pack "XX1234五"- assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text--testFolds :: TestTree-testFolds =- testGroup "folds-unfolds" [- testGroup "folds" [- testProperty "sf_foldl" sf_foldl,- testProperty "t_foldl" t_foldl,- testProperty "tl_foldl" tl_foldl,- testProperty "sf_foldl'" sf_foldl',- testProperty "t_foldl'" t_foldl',- testProperty "tl_foldl'" tl_foldl',- testProperty "sf_foldl1" sf_foldl1,- testProperty "t_foldl1" t_foldl1,- testProperty "tl_foldl1" tl_foldl1,- testProperty "t_foldl1'" t_foldl1',- testProperty "sf_foldl1'" sf_foldl1',- testProperty "tl_foldl1'" tl_foldl1',- testProperty "sf_foldr" sf_foldr,- testProperty "t_foldr" t_foldr,- testProperty "t_foldr'" t_foldr',- testProperty "tl_foldr" tl_foldr,- testProperty "sf_foldr1" sf_foldr1,- testProperty "t_foldr1" t_foldr1,- testProperty "tl_foldr1" tl_foldr1,- testProperty "t_foldlM'" t_foldlM',- testProperty "tl_foldlM'" tl_foldlM',-#ifdef MIN_VERSION_tasty_inspection_testing- $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')),-#endif- testCase "fold_apart" fold_apart,-- testGroup "special" [- testProperty "s_concat_s" s_concat_s,- testProperty "sf_concat" sf_concat,- testProperty "t_concat" t_concat,- testProperty "tl_concat" tl_concat,- testProperty "sf_concatMap" sf_concatMap,- testProperty "t_concatMap" t_concatMap,- testProperty "tl_concatMap" tl_concatMap,- testProperty "sf_any" sf_any,- testProperty "t_any" t_any,- testProperty "tl_any" tl_any,- testProperty "sf_all" sf_all,- testProperty "t_all" t_all,- testProperty "tl_all" tl_all,- testProperty "sf_maximum" sf_maximum,- testProperty "t_maximum" t_maximum,- testProperty "tl_maximum" tl_maximum,- testProperty "sf_minimum" sf_minimum,- testProperty "t_minimum" t_minimum,- testProperty "tl_minimum" tl_minimum,- testProperty "t_isAscii " t_isAscii,- testProperty "tl_isAscii " tl_isAscii,- testCase "isAscii_border" isAscii_border- ]- ],-- testGroup "scans" [- testProperty "sf_scanl" sf_scanl,- testProperty "t_scanl" t_scanl,- testProperty "tl_scanl" tl_scanl,- testProperty "t_scanl1" t_scanl1,- testProperty "tl_scanl1" tl_scanl1,- testProperty "t_scanr" t_scanr,- testProperty "tl_scanr" tl_scanr,- testProperty "t_scanr1" t_scanr1,- testProperty "tl_scanr1" tl_scanr1- ],-- testGroup "mapAccum" [- testProperty "t_mapAccumL_char" t_mapAccumL_char,- testProperty "t_mapAccumL" t_mapAccumL,- testProperty "tl_mapAccumL_char" tl_mapAccumL_char,- testProperty "tl_mapAccumL" tl_mapAccumL,- testProperty "t_mapAccumR_char" t_mapAccumR_char,- testProperty "t_mapAccumR" t_mapAccumR,- testProperty "tl_mapAccumR_char" tl_mapAccumR_char,- testProperty "tl_mapAccumR" tl_mapAccumR- ],-- testGroup "unfolds" [- testProperty "tl_cycle" tl_cycle,- testProperty "tl_iterate" tl_iterate,- testProperty "t_unfoldr" t_unfoldr,- testProperty "tl_unfoldr" tl_unfoldr,- testProperty "t_unfoldrN" t_unfoldrN,- testProperty "tl_unfoldrN" tl_unfoldrN- ],-- testGroup "replicate" [- testProperty "tl_repeat" tl_repeat,- testProperty "s_replicate" s_replicate,- testProperty "t_replicate_char" t_replicate_char,- testProperty "tl_replicate_char" tl_replicate_char,- testProperty "t_length_replicate_char" t_length_replicate_char,- testProperty "tl_length_replicate_char" tl_length_replicate_char,- testProperty "t_replicate" t_replicate,- testProperty "tl_replicate" tl_replicate,- testProperty "t_length_replicate" t_length_replicate,- testProperty "tl_length_replicate" tl_length_replicate- ]-- ]+-- | Test folds, scans, and unfolds + +{-# LANGUAGE CPP #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.Folds + ( testFolds + ) where + +import Control.Arrow (second) +import Control.Exception (ErrorCall, evaluate, try) +import Data.Functor.Identity (Identity(..)) +import Control.Monad.Trans.State (runState, state) +import Data.Word (Word8, Word16) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertFailure, assertBool) +import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) +import Tests.QuickCheckUtils +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Lazy as TL +import qualified Data.Char as Char + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectTest, (==~)) +import GHC.Exts (inline) +#endif + +-- Folds + +sf_foldl (applyFun -> p) (applyFun2 -> f) z = + (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) + where _types = f :: Char -> Char -> Char +tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) + where _types = f :: Char -> Char -> Char +sf_foldl' (applyFun -> p) (applyFun2 -> f) z = + (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z + where _types = f :: Char -> Char -> Char +tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z + where _types = f :: Char -> Char -> Char +sf_foldl1 (applyFun -> p) (applyFun2 -> f) = + (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' (applyFun -> p) (applyFun2 -> f) = + (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr (applyFun -> p) (applyFun2 -> f) z = + (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) + where _types = f :: Char -> Char -> Char +t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z + where _types = f :: Char -> Char -> Char +t_foldr' (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr' f z + where _types = f :: Char -> Char -> Char +tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z + where _types = f :: Char -> Char -> Char +sf_foldr1 (applyFun -> p) (applyFun2 -> f) = + (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) +t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f + +-- Distinguish foldl/foldr from foldl'/foldr' + +fold_apart :: IO () +fold_apart = do + ok (T.foldr f () (T.pack "az")) + ko (T.foldr' f () (T.pack "az")) + ok (T.foldl (flip f) () (T.pack "za")) + ko (T.foldl' (flip f) () (T.pack "za")) + where + f c _ = if c == 'z' then error "catchme" else () + ok = evaluate + ko t = do + x <- try (evaluate t) + case x :: Either ErrorCall () of + Left _ -> pure () + Right _ -> assertFailure "test should have failed but didn't" + +-- Special folds + +s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) +sf_concat (applyFun -> p) + = (L.concat . map (L.filter p) . unSqrt) `eq` + (unpackS . S.concat . map (S.filter p . packS) . unSqrt) +t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) +tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) +sf_concatMap (applyFun -> p) (applyFun -> f) = + (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) +t_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) +tl_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) +sf_any (applyFun -> q) (applyFun -> p) + = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any (applyFun -> p) + = L.any p `eqP` T.any p +tl_any (applyFun -> p) + = L.any p `eqP` TL.any p +sf_all (applyFun -> q) (applyFun -> p) + = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all (applyFun -> p) + = L.all p `eqP` T.all p +tl_all (applyFun -> p) + = L.all p `eqP` TL.all p +sf_maximum (applyFun -> p) + = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +t_maximum = L.maximum `eqP` T.maximum +tl_maximum = L.maximum `eqP` TL.maximum +sf_minimum (applyFun -> p) + = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +t_minimum = L.minimum `eqP` T.minimum +tl_minimum = L.minimum `eqP` TL.minimum +t_isAscii = L.all Char.isAscii `eqP` T.isAscii +tl_isAscii = L.all Char.isAscii `eqP` TL.isAscii + +-- Scans + +sf_scanl (applyFun -> p) (applyFun2 -> f) z = + (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) + +t_scanl_is_safe = let c = '\55296' in + T.scanl undefined c mempty === T.singleton c +tl_scanl_is_safe = let c = '\55296' in + TL.scanl undefined c mempty === TL.singleton c +t_scanr_is_safe = let c = '\55296' in + T.scanr undefined c mempty === T.singleton c +tl_scanr_is_safe = let c = '\55296' in + TL.scanr undefined c mempty === TL.singleton c + +t_mapAccumL_char c t = + snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumL_char c t = + snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) + where _types = f :: Int -> Char -> (Int,Char) +t_mapAccumR_char c t = + snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) +tl_mapAccumR_char c t = + snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) + where _types = f :: Int -> Char -> (Int,Char) + +-- Unfolds + +tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) + +s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) + +t_replicate_char (Small n) c = + L.replicate n c === T.unpack (T.replicate n (T.singleton c)) +tl_replicate_char (Small n) c = + L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) +t_length_replicate_char (Small n) c = + L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) +tl_length_replicate_char (Small n) c = + L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) + +t_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) +tl_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) +t_length_replicate (Small n) = + (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) +tl_length_replicate (Small n) = + (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) + +tl_cycle n = (L.take m . L.cycle) `eq` + (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) + where m = fromIntegral (n :: Word8) + +tl_iterate (applyFun -> f) n + = (L.take m . L.iterate f) `eq` + (unpackS . TL.take (fromIntegral m) . TL.iterate f) + where m = fromIntegral (n :: Word8) + +unf :: Int -> Char -> Maybe (Char, Char) +unf n c | fromEnum c * 100 > n = Nothing + | otherwise = Just (c, succ c) + +t_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . T.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +tl_unfoldr n = L.unfoldr (unf m) `eq` (unpackS . TL.unfoldr (unf m)) + where m = fromIntegral (n :: Word16) +t_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . T.unfoldrN i (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) +tl_unfoldrN n m = (L.take i . L.unfoldr (unf j)) `eq` + (unpackS . TL.unfoldrN (fromIntegral i) (unf j)) + where i = fromIntegral (n :: Word16) + j = fromIntegral (m :: Word16) + +-- Monadic folds + +-- Parametric polymorphism allows us to only test foldlM' specialized to +-- one function in the state monad (called @logger@ in the following tests) +-- that just logs the arguments it was applied to and produces a fresh +-- accumulator. That alone determines the general behavior of foldlM' with an +-- arbitrary function in any monad. +-- Reference: "Testing Polymorphic Properties" by Bernardy et al. +-- https://publications.lib.chalmers.se/records/fulltext/local_99387.pdf + +t_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . T.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order +tl_foldlM' = (\l -> (length l, zip [0 ..] l)) `eqP` (fmap reverse . (`runState` []) . TL.foldlM' logger 0) + where logger i c = state (\cs -> (length cs + 1, (i, c) : cs)) -- list in reverse order + +#ifdef MIN_VERSION_tasty_inspection_testing +-- As a sanity check for performance, the simplified Core +-- foldlM' specialized to Identity is the same as foldl'. + +_S_foldl'_from_foldlM' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl'_from_foldlM' f x = runIdentity . S.foldlM' (\i c -> Identity (f i c)) x + +_S_foldl' :: (a -> Char -> a) -> a -> S.Stream Char -> a +_S_foldl' = inline S.foldl' +#endif + +isAscii_border :: IO () +isAscii_border = do + let text = T.drop 2 $ T.pack "XX1234五" + assertBool "UTF-8 string with ASCII prefix ending at last position incorrectly detected as ASCII" $ not $ T.isAscii text + +testFolds :: TestTree +testFolds = + testGroup "folds-unfolds" [ + testGroup "folds" [ + testProperty "sf_foldl" sf_foldl, + testProperty "t_foldl" t_foldl, + testProperty "tl_foldl" tl_foldl, + testProperty "sf_foldl'" sf_foldl', + testProperty "t_foldl'" t_foldl', + testProperty "tl_foldl'" tl_foldl', + testProperty "sf_foldl1" sf_foldl1, + testProperty "t_foldl1" t_foldl1, + testProperty "tl_foldl1" tl_foldl1, + testProperty "t_foldl1'" t_foldl1', + testProperty "sf_foldl1'" sf_foldl1', + testProperty "tl_foldl1'" tl_foldl1', + testProperty "sf_foldr" sf_foldr, + testProperty "t_foldr" t_foldr, + testProperty "t_foldr'" t_foldr', + testProperty "tl_foldr" tl_foldr, + testProperty "sf_foldr1" sf_foldr1, + testProperty "t_foldr1" t_foldr1, + testProperty "tl_foldr1" tl_foldr1, + testProperty "t_foldlM'" t_foldlM', + testProperty "tl_foldlM'" tl_foldlM', +#ifdef MIN_VERSION_tasty_inspection_testing + let _unused = ['_S_foldl'_from_foldlM', '_S_foldl'] in + $(inspectTest ('_S_foldl'_from_foldlM' ==~ '_S_foldl')), +#endif + testCase "fold_apart" fold_apart, + + testGroup "special" [ + testProperty "s_concat_s" s_concat_s, + testProperty "sf_concat" sf_concat, + testProperty "t_concat" t_concat, + testProperty "tl_concat" tl_concat, + testProperty "sf_concatMap" sf_concatMap, + testProperty "t_concatMap" t_concatMap, + testProperty "tl_concatMap" tl_concatMap, + testProperty "sf_any" sf_any, + testProperty "t_any" t_any, + testProperty "tl_any" tl_any, + testProperty "sf_all" sf_all, + testProperty "t_all" t_all, + testProperty "tl_all" tl_all, + testProperty "sf_maximum" sf_maximum, + testProperty "t_maximum" t_maximum, + testProperty "tl_maximum" tl_maximum, + testProperty "sf_minimum" sf_minimum, + testProperty "t_minimum" t_minimum, + testProperty "tl_minimum" tl_minimum, + testProperty "t_isAscii " t_isAscii, + testProperty "tl_isAscii " tl_isAscii, + testCase "isAscii_border" isAscii_border + ] + ], + + testGroup "scans" [ + testProperty "sf_scanl" sf_scanl, + testProperty "t_scanl" t_scanl, + testProperty "tl_scanl" tl_scanl, + testProperty "t_scanl1" t_scanl1, + testProperty "tl_scanl1" tl_scanl1, + testProperty "t_scanr" t_scanr, + testProperty "tl_scanr" tl_scanr, + testProperty "t_scanr1" t_scanr1, + testProperty "tl_scanr1" tl_scanr1, + + testProperty "t_scanl_is_safe" t_scanl_is_safe, + testProperty "tl_scanl_is_safe" tl_scanl_is_safe, + testProperty "t_scanr_is_safe" t_scanr_is_safe, + testProperty "tl_scanr_is_safe" tl_scanr_is_safe + ], + + testGroup "mapAccum" [ + testProperty "t_mapAccumL_char" t_mapAccumL_char, + testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL_char" tl_mapAccumL_char, + testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR_char" t_mapAccumR_char, + testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR_char" tl_mapAccumR_char, + testProperty "tl_mapAccumR" tl_mapAccumR + ], + + testGroup "unfolds" [ + testProperty "tl_cycle" tl_cycle, + testProperty "tl_iterate" tl_iterate, + testProperty "t_unfoldr" t_unfoldr, + testProperty "tl_unfoldr" tl_unfoldr, + testProperty "t_unfoldrN" t_unfoldrN, + testProperty "tl_unfoldrN" tl_unfoldrN + ], + + testGroup "replicate" [ + testProperty "tl_repeat" tl_repeat, + testProperty "s_replicate" s_replicate, + testProperty "t_replicate_char" t_replicate_char, + testProperty "tl_replicate_char" tl_replicate_char, + testProperty "t_length_replicate_char" t_length_replicate_char, + testProperty "tl_length_replicate_char" tl_length_replicate_char, + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_length_replicate" t_length_replicate, + testProperty "tl_length_replicate" tl_length_replicate + ] + + ]
tests/Tests/Properties/Instances.hs view
@@ -7,6 +7,7 @@ ) where import Data.Binary (encode, decodeOrFail)+import Data.Semigroup import Data.String (IsString(fromString)) import Test.QuickCheck import Test.Tasty (TestTree, testGroup)@@ -37,6 +38,12 @@ tl_Show = show `eq` (show . TL.pack) t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s))+t_stimes = \ number -> eq+ ((stimes :: Int -> String -> String) number . unSqrt)+ (unpackS . (stimes :: Int -> T.Text -> T.Text) number . T.pack . unSqrt)+tl_stimes = \ number -> eq+ ((stimes :: Int -> String -> String) number . unSqrt)+ (unpackS . (stimes :: Int -> TL.Text -> TL.Text) number . TL.pack . unSqrt) t_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt) tl_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt) t_mempty = mempty === (unpackS (mempty :: T.Text))@@ -71,6 +78,8 @@ testProperty "tl_Show" tl_Show, testProperty "t_mappend" t_mappend, testProperty "tl_mappend" tl_mappend,+ testProperty "t_stimes" t_stimes,+ testProperty "tl_stimes" tl_stimes, testProperty "t_mconcat" t_mconcat, testProperty "tl_mconcat" tl_mconcat, testProperty "t_mempty" t_mempty,
tests/Tests/Properties/LowLevel.hs view
@@ -1,164 +1,176 @@--- | Test low-level operations--{-# LANGUAGE CPP #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE ScopedTypeVariables #-}--{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-}--#ifdef MIN_VERSION_tasty_inspection_testing-{-# LANGUAGE TemplateHaskell #-}-{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-}-#endif--module Tests.Properties.LowLevel (testLowLevel) where--import Prelude hiding (head, tail)-import Control.Applicative ((<$>), pure)-import Control.Exception as E (SomeException, catch, evaluate)-import Data.Int (Int32, Int64)-import Data.Text.Foreign-import Data.Text.Internal (Text(..), mul, mul32, mul64, safe)-import Data.Word (Word8, Word16, Word32)-import System.IO.Unsafe (unsafePerformIO)-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.HUnit (testCase, assertEqual)-import Test.Tasty.QuickCheck (testProperty)-import Test.QuickCheck hiding ((.&.))-import Tests.QuickCheckUtils-import Tests.Utils-import qualified Data.Text as T-import qualified Data.Text.Foreign as T-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as TL-import qualified Data.Text.Lazy.IO as TL-import qualified Data.Text.IO.Utf8 as TU-import qualified System.IO as IO--#ifdef MIN_VERSION_tasty_inspection_testing-import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf)-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified GHC.CString as GHC-#endif--mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a-mulRef a b- | ab < bot || ab > top = Nothing- | otherwise = Just (fromIntegral ab)- where ab = fromIntegral a * fromIntegral b- top = fromIntegral (maxBound `asTypeOf` a) :: Integer- bot = fromIntegral (minBound `asTypeOf` a) :: Integer--eval :: (a -> b -> c) -> a -> b -> Maybe c-eval f a b = unsafePerformIO $- (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing)--t_mul32 :: Int32 -> Int32 -> Property-t_mul32 a b = mulRef a b === eval mul32 a b--t_mul64 :: Int64 -> Int64 -> Property-t_mul64 a b = mulRef a b === eval mul64 a b--t_mul :: Int -> Int -> Property-t_mul a b = mulRef a b === eval mul a b---- Misc.--t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t-t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t-t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t-t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr-t_use_from0 t = ioProperty $ do- let t' = t `T.snoc` '\0'- (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0)--t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do- roundTrip <- T.withCString t T.peekCString- assertEqual "cstring" t roundTrip--t_peek_cstring_len t = ioProperty $ do- roundTrip <- T.withCStringLen t T.peekCStringLen- assertEqual "cstring_len" t roundTrip--t_copy t = T.copy t === t--t_literal_length1 = assertEqual xs (length xs) byteLen- where- xs = "\0\1\0\1\0"- Text _ _ byteLen = T.pack xs-t_literal_length2 = assertEqual xs (length xs) byteLen- where- xs = "\1\2\3\4\5"- Text _ _ byteLen = T.pack xs-t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys)- where- ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000"- xs = map safe ys--#ifdef MIN_VERSION_tasty_inspection_testing-t_literal_foo :: Text-t_literal_foo = T.pack "foo"-#endif---- Input and output.---- t_put_get = write_read T.unlines T.filter put get--- where put h = withRedirect h IO.stdout . T.putStr--- get h = withRedirect h IO.stdin T.getContents--- tl_put_get = write_read TL.unlines TL.filter put get--- where put h = withRedirect h IO.stdout . TL.putStr--- get h = withRedirect h IO.stdin TL.getContents-t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents id-tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents id--t_write_read_line = write_read (T.concat . take 1) T.filter T.hPutStrLn T.hGetLine (: [])-tl_write_read_line = write_read (TL.concat . take 1) TL.filter TL.hPutStrLn TL.hGetLine (: [])--utf8_write_read = write_read T.unlines T.filter TU.hPutStr TU.hGetContents id-utf8_write_read_line = write_read (T.concat . take 1) T.filter TU.hPutStrLn TU.hGetLine (: [])--testLowLevel :: TestTree-testLowLevel =- testGroup "lowlevel" [- testGroup "mul" [- testProperty "t_mul" t_mul,- testProperty "t_mul32" t_mul32,- testProperty "t_mul64" t_mul64- ],-- testGroup "misc" [- testProperty "t_dropWord8" t_dropWord8,- testProperty "t_takeWord8" t_takeWord8,- testProperty "t_take_drop_8" t_take_drop_8,- testProperty "t_use_from" t_use_from,- testProperty "t_use_from0" t_use_from0,- testProperty "t_copy" t_copy,- testProperty "t_peek_cstring" t_peek_cstring,- testProperty "t_peek_cstring_len" t_peek_cstring_len,- testCase "t_literal_length1" t_literal_length1,- testCase "t_literal_length2" t_literal_length2,- testCase "t_literal_surrogates" t_literal_surrogates-#ifdef MIN_VERSION_tasty_inspection_testing- , $(inspectObligations- [ (`hasNoTypes` [''Char, ''[]])- , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList])- , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#])- , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#])- ]- 't_literal_foo)-#endif- ],-- testGroup "input-output" [- testGroup "t_write_read" t_write_read,- testGroup "tl_write_read" tl_write_read,- testGroup "t_write_read_line" t_write_read_line,- testGroup "tl_write_read_line" tl_write_read_line,- testGroup "utf8_write_read" utf8_write_read,- testGroup "utf8_write_read_line" utf8_write_read_line- -- These tests are subject to I/O race conditions- -- testProperty "t_put_get" t_put_get,- -- testProperty "tl_put_get" tl_put_get- ]- ]+-- | Test low-level operations + +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + +module Tests.Properties.LowLevel (testLowLevel) where + +import Prelude hiding (head, tail) +import Control.Applicative ((<$>), pure) +import Control.Exception as E (SomeException, catch, evaluate) +import Data.Functor.Identity (Identity(..)) +import Data.Int (Int32, Int64) +import Data.Text.Foreign +import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) +import Data.Word (Word8, Word16, Word32) +import System.IO.Unsafe (unsafePerformIO) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty.QuickCheck (testProperty) +import Test.QuickCheck hiding ((.&.)) +import Tests.QuickCheckUtils +import Tests.Utils +import qualified Data.Text as T +import qualified Data.Text.Foreign as T +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.IO.Utf8 as TU +import qualified System.IO as IO + +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified GHC.CString as GHC +#endif + +mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a +mulRef a b + | ab < bot || ab > top = Nothing + | otherwise = Just (fromIntegral ab) + where ab = fromIntegral a * fromIntegral b + top = fromIntegral (maxBound `asTypeOf` a) :: Integer + bot = fromIntegral (minBound `asTypeOf` a) :: Integer + +eval :: (a -> b -> c) -> a -> b -> Maybe c +eval f a b = unsafePerformIO $ + (Just <$> evaluate (f a b)) `E.catch` (\(_::SomeException) -> pure Nothing) + +t_mul32 :: Int32 -> Int32 -> Property +t_mul32 a b = mulRef a b === eval mul32 a b + +t_mul64 :: Int64 -> Int64 -> Property +t_mul64 a b = mulRef a b === eval mul64 a b + +t_mul :: Int -> Int -> Property +t_mul a b = mulRef a b === eval mul a b + +-- Misc. + +t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t +t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t +t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t +t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr +t_use_from0 t = ioProperty $ do + let t' = t `T.snoc` '\0' + (== T.takeWhile (/= '\0') t') <$> useAsPtr t' (const . fromPtr0) + +t_peek_cstring t = T.all (/= '\0') t ==> ioProperty $ do + roundTrip <- T.withCString t T.peekCString + assertEqual "cstring" t roundTrip + +t_peek_cstring_len t = ioProperty $ do + roundTrip <- T.withCStringLen t T.peekCStringLen + assertEqual "cstring_len" t roundTrip + +t_copy t = T.copy t === t + +t_literal_length1 = assertEqual xs (length xs) byteLen + where + xs = "\0\1\0\1\0" + Text _ _ byteLen = T.pack xs +t_literal_length2 = assertEqual xs (length xs) byteLen + where + xs = "\1\2\3\4\5" + Text _ _ byteLen = T.pack xs +t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) + where + ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" + xs = map safe ys + +#ifdef MIN_VERSION_tasty_inspection_testing +t_literal_foo :: Text +t_literal_foo = T.pack "foo" +#endif + +-- Input and output. + +-- t_put_get = write_read T.unlines T.filter put get +-- where put h = withRedirect h IO.stdout . T.putStr +-- get h = withRedirect h IO.stdin T.getContents +-- tl_put_get = write_read TL.unlines TL.filter put get +-- where put h = withRedirect h IO.stdout . TL.putStr +-- get h = withRedirect h IO.stdin TL.getContents + +inputOutput :: TestTree +inputOutput = testGroup "input-output" [ + testProperty "t_write_read" $ write_read arbitrary shrink (T.replace "\n" "\r\n") T.hPutStr T.hGetContents, + testProperty "tl_write_read" $ write_read arbitrary shrink (TL.replace "\n" "\r\n") TL.hPutStr TL.hGetContents, + testProperty "t_write_read_line" $ write_read genTLine shrinkTLine (`T.append` "\r") T.hPutStrLn T.hGetLine, + testProperty "tl_write_read_line" $ write_read genTLLine shrinkTLLine (`TL.append` "\r") TL.hPutStrLn TL.hGetLine, + -- Note: Data.Text.IO.Utf8 does NO newline translation + testProperty "utf8_write_read" $ write_read arbitrary shrink id TU.hPutStr TU.hGetContents, + testProperty "utf8_write_read_line" $ write_read genTLine shrinkTLine id TU.hPutStrLn TU.hGetLine + -- These tests are subject to I/O race conditions + -- testProperty "t_put_get" t_put_get, + -- testProperty "tl_put_get" tl_put_get + ] + +genTLine :: Gen T.Text +genTLine = T.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +genTLLine :: Gen TL.Text +genTLLine = TL.filter (`notElem` ("\r\n" :: String)) <$> arbitrary + +shrinkTLine :: T.Text -> [T.Text] +shrinkTLine = filter (T.all (/= '\n')) . shrink + +shrinkTLLine :: TL.Text -> [TL.Text] +shrinkTLLine = filter (TL.all (/= '\n')) . shrink + +testLowLevel :: TestTree +testLowLevel = + testGroup "lowlevel" [ + testGroup "mul" [ + testProperty "t_mul" t_mul, + testProperty "t_mul32" t_mul32, + testProperty "t_mul64" t_mul64 + ], + + testGroup "misc" [ + testProperty "t_dropWord8" t_dropWord8, + testProperty "t_takeWord8" t_takeWord8, + testProperty "t_take_drop_8" t_take_drop_8, + testProperty "t_use_from" t_use_from, + testProperty "t_use_from0" t_use_from0, + testProperty "t_copy" t_copy, + testProperty "t_peek_cstring" t_peek_cstring, + testProperty "t_peek_cstring_len" t_peek_cstring_len, + testCase "t_literal_length1" t_literal_length1, + testCase "t_literal_length2" t_literal_length2, + testCase "t_literal_surrogates" t_literal_surrogates +#ifdef MIN_VERSION_tasty_inspection_testing + -- Hack to force GHC to keep the binding around until this is fixed: https://gitlab.haskell.org/ghc/ghc/-/issues/26436 + , let _unused = 't_literal_foo in + $(inspectObligations + [ (`hasNoTypes` [''Char, ''[]]) + , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) + , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) + , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) + ] + 't_literal_foo) +#endif + ], + + inputOutput + ]
tests/Tests/Properties/Substrings.hs view
@@ -229,9 +229,8 @@ (l, []) -> [l] (l, _ : s') -> l : loop s' -t_chunksOf_same_lengths k = conjoin . map ((===k) . T.length) . ini . T.chunksOf k- where ini [] = []- ini xs = init xs+t_chunksOf_same_lengths k =+ conjoin . map ((===k) . T.length) . drop 1 . reverse . T.chunksOf k t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0) where len = L.sum . L.map T.length $ T.chunksOf k t
tests/Tests/Properties/Text.hs view
@@ -1,483 +1,491 @@--- | Tests for operations that don't fit in the other @Test.Properties.*@ modules.--{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE CPP #-}-{-# LANGUAGE ViewPatterns #-}--{-# OPTIONS_GHC -fno-warn-missing-signatures #-}--module Tests.Properties.Text- ( testText- ) where--import Data.Char (isLower, isLetter, isUpper)-import Data.Maybe (mapMaybe)-import Data.Text.Internal.Fusion.Size-import Data.Word (Word8)-import Test.QuickCheck-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.QuickCheck (testProperty)-import Tests.QuickCheckUtils-import qualified Data.Char as C-import qualified Data.List as L-import qualified Data.Text as T-import qualified Data.Text.Internal.Fusion as S-import qualified Data.Text.Internal.Fusion.Common as S-import qualified Data.Text.Internal.Lazy.Fusion as SL-import qualified Data.Text.Internal.Lazy.Search as S (indices)-import qualified Data.Text.Internal.Search as T (indices)-import qualified Data.Text.Lazy as TL-import qualified Tests.SlowFunctions as Slow--t_pack_unpack = (T.unpack . T.pack) `eq` id-tl_pack_unpack = (TL.unpack . TL.pack) `eq` id-t_stream_unstream = (S.unstream . S.stream) `eq` id-tl_stream_unstream = (SL.unstream . SL.stream) `eq` id-t_reverse_stream t = (S.reverse . S.reverseStream) t === t-t_singleton c = [c] === (T.unpack . T.singleton) c-tl_singleton c = [c] === (TL.unpack . TL.singleton) c-tl_unstreamChunks x = f 11 x === f 1000 x- where f n = SL.unstreamChunks n . S.streamList-tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id-tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id--s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f)-s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f)-sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p)--t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f)-tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f)-t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g)-tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g)-t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f)-tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f)--s_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . S.intercalate (packS c) . map packS . unSqrt)-t_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . T.intercalate (packS c) . map packS . unSqrt)-tl_intercalate c = (L.intercalate c . unSqrt) `eq`- (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)-t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq`- (T.length . T.intercalate (packS c) . map packS . unSqrt)-tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq`- (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt)-s_intersperse c = L.intersperse c `eqP`- (unpackS . S.intersperse c)-s_intersperse_s c = L.intersperse c `eqP`- (unpackS . S.unstream . S.intersperse c)-sf_intersperse (applyFun -> p) c- = (L.intersperse c . L.filter p) `eqP`- (unpackS . S.intersperse c . S.filter p)-t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c)-tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c)-t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c)-tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c)-t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt)-tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt)-t_reverse = L.reverse `eqP` (unpackS . T.reverse)-tl_reverse = L.reverse `eqP` (unpackS . TL.reverse)-t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)--t_replace s d = (L.intercalate d . splitOn s) `eqP`- (unpackS . T.replace (T.pack s) (T.pack d))-tl_replace s d = (L.intercalate d . splitOn s) `eqP`- (unpackS . TL.replace (TL.pack s) (TL.pack d))--splitOn :: (Eq a) => [a] -> [a] -> [[a]]-splitOn pat src0- | l == 0 = error "splitOn: empty"- | otherwise = go src0- where- l = length pat- go src = search 0 src- where- search _ [] = [src]- search !n s@(_:s')- | pat `L.isPrefixOf` s = take n src : go (drop l s)- | otherwise = search (n+1) s'--s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs- where s = S.streamList xs-sf_toCaseFold_length (applyFun -> p) xs =- (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs)- where s = S.streamList xs-t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t--tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t-#if MIN_VERSION_base(4,16,0)-t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==>- T.toCaseFold (T.singleton c) === T.singleton (C.toLower c)-#endif---- | Baseline generated with GHC 9.2 + text-1.2.5.0,-t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389"-t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower))-t_toCaseFold_cherokeeUpper = conjoin $- map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper---- | Generated with GHC 9.2 + text-1.2.5.0,--- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $--- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c))--- [minBound .. maxBound]-toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"-cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase-cherokeeLower = ['\xAB70'..'\xABBF']--t_toLower_length t = T.length (T.toLower t) >= T.length t-t_toLower_lower t = p (T.toLower t) >= p t- where p = T.length . T.filter isLower-tl_toLower_lower t = p (TL.toLower t) >= p t- where p = TL.length . TL.filter isLower-#if MIN_VERSION_base(4,13,0)-t_toLower_char c = c /= '\304' ==>- T.toLower (T.singleton c) === T.singleton (C.toLower c)-#endif-t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775"--t_toUpper_length t = T.length (T.toUpper t) >= T.length t-t_toUpper_upper t = p (T.toUpper t) >= p t- where p = T.length . T.filter isUpper-tl_toUpper_upper t = p (TL.toUpper t) >= p t- where p = TL.length . TL.filter isUpper-#if MIN_VERSION_base(4,13,0)-t_toUpper_char c = c `notElem` toUpperExceptions ==>- T.toUpper (T.singleton c) === T.singleton (C.toUpper c)-#endif---- | Baseline generated with GHC 9.2 + text-1.2.5.0,-t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341"---- | Generated with GHC 9.2 + text-1.2.5.0,--- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c))--- [minBound .. maxBound]-toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"--t_toTitle_title t = all (<= 1) (caps w)- where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle- -- TIL: there exist uppercase-only letters- w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t-t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian)- where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words- -- Surprise! The Spanish/Portuguese ordinal indicators changed- -- from category Ll (letter, lowercase) to Lo (letter, other)- -- in Unicode 7.0- -- Oh, and there exist lowercase-only letters (see previous test)- stable c = if isLower c- then C.toUpper c /= c- else c /= '\170' && c /= '\186'- -- Georgian text does not have a concept of title case- -- https://en.wikipedia.org/wiki/Georgian_Extended- isGeorgian c = c >= '\4256' && c < '\4352'-#if MIN_VERSION_base(4,13,0)-t_toTitle_char c = c `notElem` toTitleExceptions ==>- T.toTitle (T.singleton c) === T.singleton (C.toUpper c)-#endif---- | Baseline generated with GHC 9.2 + text-1.2.5.0,-t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389"---- | Generated with GHC 9.2 + text-1.2.5.0,--- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c))--- [minBound .. maxBound]-toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279"--t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t-t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t-t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t--ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs))-ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs))-ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs))--ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs))- where- referenceToTitle _ [] = []- referenceToTitle False (y : ys)- | C.isLetter y = C.toUpper y : referenceToTitle True ys- | otherwise = y : referenceToTitle False ys- referenceToTitle True (y : ys)- | C.isLetter y = C.toLower y : referenceToTitle True ys- | otherwise = y : referenceToTitle (not (C.isSpace y)) ys--justifyLeft k c xs = xs ++ L.replicate (k - length xs) c-justifyRight m n xs = L.replicate (m - length xs) n ++ xs-center k c xs- | len >= k = xs- | otherwise = L.replicate l c ++ xs ++ L.replicate r c- where len = length xs- d = k - len- r = d `div` 2- l = d - r--s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c)- where j = fromIntegral (k :: Word8)-s_justifyLeft_s k c = justifyLeft j c `eqP`- (unpackS . S.unstream . S.justifyLeftI j c)- where j = fromIntegral (k :: Word8)-sf_justifyLeft (applyFun -> p) k c- = (justifyLeft j c . L.filter p) `eqP`- (unpackS . S.justifyLeftI j c . S.filter p)- where j = fromIntegral (k :: Word8)-t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c)- where j = fromIntegral (k :: Word8)-tl_justifyLeft k c = justifyLeft j c `eqP`- (unpackS . TL.justifyLeft (fromIntegral j) c)- where j = fromIntegral (k :: Word8)-t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c)- where j = fromIntegral (k :: Word8)-tl_justifyRight k c = justifyRight j c `eqP`- (unpackS . TL.justifyRight (fromIntegral j) c)- where j = fromIntegral (k :: Word8)-t_center k c = center j c `eqP` (unpackS . T.center j c)- where j = fromIntegral (k :: Word8)-tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c)- where j = fromIntegral (k :: Word8)--t_elem c = L.elem c `eqP` T.elem c-tl_elem c = L.elem c `eqP` TL.elem c-sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p)-sf_filter (applyFun -> q) (applyFun -> p)- = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q)--t_filter (applyFun -> p)- = L.filter p `eqP` (unpackS . T.filter p)-tl_filter (applyFun -> p)- = L.filter p `eqP` (unpackS . TL.filter p)-t_filter_filter (applyFun -> p) (applyFun -> q)- = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q)-tl_filter_filter (applyFun -> p) (applyFun -> q)- = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q)-t_length_filter (applyFun -> p)- = (L.length . L.filter p) `eqP` (T.length . T.filter p)-tl_length_filter (applyFun -> p)- = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p)--sf_findBy (applyFun -> q) (applyFun -> p)- = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)-t_find (applyFun -> p) = L.find p `eqP` T.find p-tl_find (applyFun -> p) = L.find p `eqP` TL.find p-t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p)-tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p)--sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l-t_index s i = ((s L.!!) `eq` T.index (packS s)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l--tl_index s i = ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) j- where l = L.length s- j = if l == 0 then 0 else i `mod` (3 * l) - l--t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p-t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t-tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`- TL.count t-t_zip s = L.zip s `eqP` T.zip (packS s)-tl_zip s = L.zip s `eqP` TL.zip (packS s)-sf_zipWith (applyFun -> p) (applyFun2 -> c) s- = (L.zipWith c (L.filter p s) . L.filter p) `eqP`- (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p)-t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))-tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))-t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s))-tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s))--t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s-tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s- where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t)- conc = T.concat . TL.toChunks-t_indices_occurs = \(Sqrt (NotEmpty t)) ts ->- let s = T.intercalate t ts- in Slow.indices t s === T.indices t s--t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8]-tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8]--t_indices_drop n s pref suff = T.indices s t === Slow.indices s t- where- t = T.drop n $ pref `T.append` s `T.append` suff-tl_indices_drop n s pref suff =- map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- t = TL.drop n $ pref `TL.append` s `TL.append` suff--tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0]-tl_indices_drop_chunked n s pref suff =- map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- -- constructing a pathologically chunked haystack- t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff--t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t- where- s = T.singleton c- t = T.drop n $ pref `T.append` s `T.append` suff-tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t)- where- s = TL.singleton c- t = TL.drop n $ pref `TL.append` s `TL.append` suff---- Make a stream appear shorter than it really is, to ensure that--- functions that consume inaccurately sized streams behave--- themselves.-shorten :: Int -> S.Stream a -> S.Stream a-shorten n t@(S.Stream arr off len)- | n > 0 = S.Stream arr off (smaller (exactSize n) len)- | otherwise = t--testText :: TestTree-testText =- testGroup "Text" [- testGroup "creation/elimination" [- testProperty "t_pack_unpack" t_pack_unpack,- testProperty "tl_pack_unpack" tl_pack_unpack,- testProperty "t_stream_unstream" t_stream_unstream,- testProperty "tl_stream_unstream" tl_stream_unstream,- testProperty "t_reverse_stream" t_reverse_stream,- testProperty "t_singleton" t_singleton,- testProperty "tl_singleton" tl_singleton,- testProperty "tl_unstreamChunks" tl_unstreamChunks,- testProperty "tl_chunk_unchunk" tl_chunk_unchunk,- testProperty "tl_from_to_strict" tl_from_to_strict- ],-- testGroup "transformations" [- testProperty "s_map" s_map,- testProperty "s_map_s" s_map_s,- testProperty "sf_map" sf_map,-- testProperty "t_map" t_map,- testProperty "tl_map" tl_map,- testProperty "t_map_map" t_map_map,- testProperty "tl_map_map" tl_map_map,- testProperty "t_length_map" t_length_map,- testProperty "tl_length_map" tl_length_map,-- testProperty "s_intercalate" s_intercalate,- testProperty "t_intercalate" t_intercalate,- testProperty "tl_intercalate" tl_intercalate,- testProperty "t_length_intercalate" t_length_intercalate,- testProperty "tl_length_intercalate" tl_length_intercalate,- testProperty "s_intersperse" s_intersperse,- testProperty "s_intersperse_s" s_intersperse_s,- testProperty "sf_intersperse" sf_intersperse,- testProperty "t_intersperse" t_intersperse,- testProperty "tl_intersperse" tl_intersperse,- testProperty "t_length_intersperse" t_length_intersperse,- testProperty "tl_length_intersperse" tl_length_intersperse,- testProperty "t_transpose" t_transpose,- testProperty "tl_transpose" tl_transpose,- testProperty "t_reverse" t_reverse,- testProperty "tl_reverse" tl_reverse,- testProperty "t_reverse_short" t_reverse_short,- testProperty "t_replace" t_replace,- testProperty "tl_replace" tl_replace,-- testGroup "case conversion" [- testProperty "s_toCaseFold_length" s_toCaseFold_length,- testProperty "sf_toCaseFold_length" sf_toCaseFold_length,- testProperty "t_toCaseFold_length" t_toCaseFold_length,- testProperty "tl_toCaseFold_length" tl_toCaseFold_length,-#if MIN_VERSION_base(4,16,0)- testProperty "t_toCaseFold_char" t_toCaseFold_char,-#endif- testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions,- testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower,- testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper,-- testProperty "t_toLower_length" t_toLower_length,- testProperty "t_toLower_lower" t_toLower_lower,- testProperty "tl_toLower_lower" tl_toLower_lower,- testProperty "t_toLower_dotted_i" t_toLower_dotted_i,-- testProperty "t_toUpper_length" t_toUpper_length,- testProperty "t_toUpper_upper" t_toUpper_upper,- testProperty "tl_toUpper_upper" tl_toUpper_upper,- testProperty "t_toUpper_exceptions" t_toUpper_exceptions,-- testProperty "t_toTitle_title" t_toTitle_title,- testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower,- testProperty "t_toTitle_exceptions" t_toTitle_exceptions,--#if MIN_VERSION_base(4,13,0)- -- Requires base compliant with Unicode 12.0- testProperty "t_toLower_char" t_toLower_char,- testProperty "t_toUpper_char" t_toUpper_char,- testProperty "t_toTitle_char" t_toTitle_char,-#endif-- testProperty "t_toUpper_idempotent" t_toUpper_idempotent,- testProperty "t_toLower_idempotent" t_toLower_idempotent,- testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent,-- testProperty "ascii_toLower" ascii_toLower,- testProperty "ascii_toUpper" ascii_toUpper,- testProperty "ascii_toTitle" ascii_toTitle,- testProperty "ascii_toCaseFold" ascii_toCaseFold- ],-- testGroup "justification" [- testProperty "s_justifyLeft" s_justifyLeft,- testProperty "s_justifyLeft_s" s_justifyLeft_s,- testProperty "sf_justifyLeft" sf_justifyLeft,- testProperty "t_justifyLeft" t_justifyLeft,- testProperty "tl_justifyLeft" tl_justifyLeft,- testProperty "t_justifyRight" t_justifyRight,- testProperty "tl_justifyRight" tl_justifyRight,- testProperty "t_center" t_center,- testProperty "tl_center" tl_center- ]- ],-- testGroup "searching" [- testProperty "t_elem" t_elem,- testProperty "tl_elem" tl_elem,- testProperty "sf_elem" sf_elem,- testProperty "sf_filter" sf_filter,- testProperty "t_filter" t_filter,- testProperty "tl_filter" tl_filter,- testProperty "t_filter_filter" t_filter_filter,- testProperty "tl_filter_filter" tl_filter_filter,- testProperty "t_length_filter" t_length_filter,- testProperty "tl_length_filter" tl_length_filter,- testProperty "sf_findBy" sf_findBy,- testProperty "t_find" t_find,- testProperty "tl_find" tl_find,- testProperty "t_partition" t_partition,- testProperty "tl_partition" tl_partition- ],-- testGroup "indexing" [- testProperty "sf_index" sf_index,- testProperty "t_index" t_index,- testProperty "tl_index" tl_index,- testProperty "t_findIndex" t_findIndex,- testProperty "t_count" t_count,- testProperty "tl_count" tl_count,- testProperty "t_indices" t_indices,- testProperty "tl_indices" tl_indices,- testProperty "t_indices_occurs" t_indices_occurs,-- testProperty "t_indices_drop5" t_indices_drop5,- testProperty "tl_indices_drop5" tl_indices_drop5,- testProperty "t_indices_drop" t_indices_drop,- testProperty "tl_indices_drop" tl_indices_drop,- testProperty "tl_indices_chunked" tl_indices_chunked,- testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked,- testProperty "t_indices_char_drop" t_indices_char_drop,- testProperty "tl_indices_char_drop" tl_indices_char_drop- ],-- testGroup "zips" [- testProperty "t_zip" t_zip,- testProperty "tl_zip" tl_zip,- testProperty "sf_zipWith" sf_zipWith,- testProperty "t_zipWith" t_zipWith,- testProperty "tl_zipWith" tl_zipWith,- testProperty "t_length_zipWith" t_length_zipWith,- testProperty "tl_length_zipWith" tl_length_zipWith- ]- ]+-- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. + +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Avoid restricted function" #-} + +module Tests.Properties.Text + ( testText + ) where + +import Control.Exception (SomeException, evaluate, try) +import Data.Char (isLower, isLetter, isUpper) +import Data.Maybe (mapMaybe) +import Data.Text.Internal.Fusion.Size +import Data.Word (Word8) +import Test.QuickCheck +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.QuickCheck (testProperty) +import Tests.QuickCheckUtils +import qualified Data.Char as C +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy.Fusion as SL +import qualified Data.Text.Internal.Lazy.Search as S (indices) +import qualified Data.Text.Internal.Search as T (indices) +import qualified Data.Text.Lazy as TL +import qualified Tests.SlowFunctions as Slow +#if MIN_VERSION_base(4, 15, 0) +import qualified GHC.Unicode as G (unicodeVersion) +import qualified Data.Text.Internal.Fusion.CaseMapping as T (unicodeVersion) +#endif + +t_pack_unpack = (T.unpack . T.pack) `eq` id +tl_pack_unpack = (TL.unpack . TL.pack) `eq` id +t_stream_unstream = (S.unstream . S.stream) `eq` id +tl_stream_unstream = (SL.unstream . SL.stream) `eq` id +t_reverse_stream t = (S.reverse . S.reverseStream) t === t +t_singleton c = [c] === (T.unpack . T.singleton) c +tl_singleton c = [c] === (TL.unpack . TL.singleton) c +tl_unstreamChunks x = f 11 x === f 1000 x + where f n = SL.unstreamChunks n . S.streamList +tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id +tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id + +s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) +s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) + +t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) +tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) +t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) +tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) +t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) +tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) + +s_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . S.intercalate (packS c) . map packS . unSqrt) +t_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . T.intercalate (packS c) . map packS . unSqrt) +tl_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` + (T.length . T.intercalate (packS c) . map packS . unSqrt) +tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` + (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +s_intersperse c = L.intersperse c `eqP` + (unpackS . S.intersperse c) +s_intersperse_s c = L.intersperse c `eqP` + (unpackS . S.unstream . S.intersperse c) +sf_intersperse (applyFun -> p) c + = (L.intersperse c . L.filter p) `eqP` + (unpackS . S.intersperse c . S.filter p) +t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) +t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) +tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) +t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) +tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) +t_reverse = L.reverse `eqP` (unpackS . T.reverse) +tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) +t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) + +t_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . T.replace (T.pack s) (T.pack d)) +tl_replace s d = (L.intercalate d . splitOn s) `eqP` + (unpackS . TL.replace (TL.pack s) (TL.pack d)) + +splitOn :: (Eq a) => [a] -> [a] -> [[a]] +splitOn pat src0 + | l == 0 = error "splitOn: empty" + | otherwise = go src0 + where + l = length pat + go src = search 0 src + where + search _ [] = [src] + search !n s@(_:s') + | pat `L.isPrefixOf` s = take n src : go (drop l s) + | otherwise = search (n+1) s' + +s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs + where s = S.streamList xs +sf_toCaseFold_length (applyFun -> p) xs = + (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) + where s = S.streamList xs +t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t + +tl_toCaseFold_length t = TL.length (TL.toCaseFold t) >= TL.length t +t_toCaseFold_char c = c `notElem` (toCaseFoldExceptions ++ cherokeeLower ++ cherokeeUpper) ==> + T.toCaseFold (T.singleton c) === T.singleton (C.toLower c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toCaseFold_exceptions = T.unpack (T.toCaseFold (T.pack toCaseFoldExceptions)) === "\956ssi\775\700nsj\780\953\953\776\769\965\776\769\963\946\952\966\960\954\961\949\1381\1410\5104\5105\5106\5107\5108\5109\1074\1076\1086\1089\1090\1090\1098\1123\42571h\817t\776w\778y\778a\702\7777ss\965\787\965\787\768\965\787\769\965\787\834\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7936\953\7937\953\7938\953\7939\953\7940\953\7941\953\7942\953\7943\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\7968\953\7969\953\7970\953\7971\953\7972\953\7973\953\7974\953\7975\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8032\953\8033\953\8034\953\8035\953\8036\953\8037\953\8038\953\8039\953\8048\953\945\953\940\953\945\834\945\834\953\945\953\953\8052\953\951\953\942\953\951\834\951\834\953\951\953\953\776\768\953\776\769\953\834\953\776\834\965\776\768\965\776\769\961\787\965\834\965\776\834\8060\953\969\953\974\953\969\834\969\834\953\969\953fffiflffifflstst\1396\1398\1396\1381\1396\1387\1406\1398\1396\1389" +t_toCaseFold_cherokeeLower = T.all (`elem` cherokeeUpper) (T.toCaseFold (T.pack cherokeeLower)) +t_toCaseFold_cherokeeUpper = conjoin $ + map (\c -> T.toCaseFold (T.singleton c) === T.singleton c) cherokeeUpper + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> c `notElem` (cherokeeUpper ++ cherokeeLower)) $ +-- filter (\c -> T.toCaseFold (T.singleton c) /= T.singleton (Data.Char.toLower c)) +-- [minBound .. maxBound] +toCaseFoldExceptions = "\181\223\304\329\383\496\837\912\944\962\976\977\981\982\1008\1009\1013\1415\5112\5113\5114\5115\5116\5117\7296\7297\7298\7299\7300\7301\7302\7303\7304\7830\7831\7832\7833\7834\7835\7838\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8126\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" +cherokeeUpper = ['\x13A0'..'\x13F7'] -- x13F8..13FF are lowercase +cherokeeLower = ['\xAB70'..'\xABBF'] + +t_toLower_length t = T.length (T.toLower t) >= T.length t +t_toLower_lower t = p (T.toLower t) >= p t + where p = T.length . T.filter isLower +tl_toLower_lower t = p (TL.toLower t) >= p t + where p = TL.length . TL.filter isLower +t_toLower_char c = c /= '\304' ==> + T.toLower (T.singleton c) === T.singleton (C.toLower c) +t_toLower_dotted_i = T.unpack (T.toLower (T.singleton '\304')) === "i\775" + +t_toUpper_length t = T.length (T.toUpper t) >= T.length t +t_toUpper_upper t = p (T.toUpper t) >= p t + where p = T.length . T.filter isUpper +tl_toUpper_upper t = p (TL.toUpper t) >= p t + where p = TL.length . TL.filter isUpper +t_toUpper_char c = c `notElem` toUpperExceptions ==> + T.toUpper (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toUpper_exceptions = T.unpack (T.toUpper (T.pack toUpperExceptions)) === "SS\700NJ\780\921\776\769\933\776\769\1333\1362H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7944\921\7945\921\7946\921\7947\921\7948\921\7949\921\7950\921\7951\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\7976\921\7977\921\7978\921\7979\921\7980\921\7981\921\7982\921\7983\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8040\921\8041\921\8042\921\8043\921\8044\921\8045\921\8046\921\8047\921\8122\921\913\921\902\921\913\834\913\834\921\913\921\8138\921\919\921\905\921\919\834\919\834\921\919\921\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\921\937\921\911\921\937\834\937\834\921\937\921FFFIFLFFIFFLSTST\1348\1350\1348\1333\1348\1339\1358\1350\1348\1341" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toUpper (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toUpperExceptions = "\223\329\496\912\944\1415\7830\7831\7832\7833\7834\8016\8018\8020\8022\8064\8065\8066\8067\8068\8069\8070\8071\8072\8073\8074\8075\8076\8077\8078\8079\8080\8081\8082\8083\8084\8085\8086\8087\8088\8089\8090\8091\8092\8093\8094\8095\8096\8097\8098\8099\8100\8101\8102\8103\8104\8105\8106\8107\8108\8109\8110\8111\8114\8115\8116\8118\8119\8124\8130\8131\8132\8134\8135\8140\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8179\8180\8182\8183\8188\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toTitle_title t = all (<= 1) (caps w) + where caps = fmap (T.length . T.filter isUpper) . T.words . T.toTitle + -- TIL: there exist uppercase-only letters + w = T.filter (\c -> if C.isUpper c then C.toLower c /= c else True) t +t_toTitle_1stNotLower = and . notLow . T.toTitle . T.filter stable . T.filter (not . isGeorgian) + where notLow = mapMaybe (fmap (not . isLower) . (T.find isLetter)) . T.words + -- Surprise! The Spanish/Portuguese ordinal indicators changed + -- from category Ll (letter, lowercase) to Lo (letter, other) + -- in Unicode 7.0 + -- Oh, and there exist lowercase-only letters (see previous test) + stable c = if isLower c + then C.toUpper c /= c + else c /= '\170' && c /= '\186' + -- Georgian text does not have a concept of title case + -- https://en.wikipedia.org/wiki/Georgian_Extended + isGeorgian c = c >= '\4256' && c < '\4352' +t_toTitle_char c = c `notElem` toTitleExceptions ==> + T.toTitle (T.singleton c) === T.singleton (C.toUpper c) + +-- | Baseline generated with GHC 9.2 + text-1.2.5.0, +t_toTitle_exceptions = T.unpack (T.concatMap (T.toTitle . T.singleton) (T.pack toTitleExceptions)) === "Ss\700N\453\453\453\456\456\456\459\459\459J\780\498\498\498\921\776\769\933\776\769\1333\1410\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351H\817T\776W\778Y\778A\702\933\787\933\787\768\933\787\769\933\787\834\8122\837\902\837\913\834\913\834\837\8138\837\905\837\919\834\919\834\837\921\776\768\921\776\769\921\834\921\776\834\933\776\768\933\776\769\929\787\933\834\933\776\834\8186\837\911\837\937\834\937\834\837FfFiFlFfiFflStSt\1348\1398\1348\1381\1348\1387\1358\1398\1348\1389" + +-- | Generated with GHC 9.2 + text-1.2.5.0, +-- filter (\c -> T.toTitle (T.singleton c) /= T.singleton (Data.Char.toUpper c)) +-- [minBound .. maxBound] +toTitleExceptions = "\223\329\452\453\454\455\456\457\458\459\460\496\497\498\499\912\944\1415\4304\4305\4306\4307\4308\4309\4310\4311\4312\4313\4314\4315\4316\4317\4318\4319\4320\4321\4322\4323\4324\4325\4326\4327\4328\4329\4330\4331\4332\4333\4334\4335\4336\4337\4338\4339\4340\4341\4342\4343\4344\4345\4346\4349\4350\4351\7830\7831\7832\7833\7834\8016\8018\8020\8022\8114\8116\8118\8119\8130\8132\8134\8135\8146\8147\8150\8151\8162\8163\8164\8166\8167\8178\8180\8182\8183\64256\64257\64258\64259\64260\64261\64262\64275\64276\64277\64278\64279" + +t_toUpper_idempotent t = T.toUpper (T.toUpper t) === T.toUpper t +t_toLower_idempotent t = T.toLower (T.toLower t) === T.toLower t +t_toCaseFold_idempotent t = T.toCaseFold (T.toCaseFold t) === T.toCaseFold t + +ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) +ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) +ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) + +ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) + where + referenceToTitle _ [] = [] + referenceToTitle False (y : ys) + | C.isLetter y = C.toUpper y : referenceToTitle True ys + | otherwise = y : referenceToTitle False ys + referenceToTitle True (y : ys) + | C.isLetter y = C.toLower y : referenceToTitle True ys + | otherwise = y : referenceToTitle (not (C.isSpace y)) ys + +justifyLeft k c xs = xs ++ L.replicate (k - length xs) c +justifyRight m n xs = L.replicate (m - length xs) n ++ xs +center k c xs + | len >= k = xs + | otherwise = L.replicate l c ++ xs ++ L.replicate r c + where len = length xs + d = k - len + r = d `div` 2 + l = d - r + +s_justifyLeft k c = justifyLeft j c `eqP` (unpackS . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +s_justifyLeft_s k c = justifyLeft j c `eqP` + (unpackS . S.unstream . S.justifyLeftI j c) + where j = fromIntegral (k :: Word8) +sf_justifyLeft (applyFun -> p) k c + = (justifyLeft j c . L.filter p) `eqP` + (unpackS . S.justifyLeftI j c . S.filter p) + where j = fromIntegral (k :: Word8) +t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) + where j = fromIntegral (k :: Word8) +tl_justifyLeft k c = justifyLeft j c `eqP` + (unpackS . TL.justifyLeft (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_justifyRight k c = justifyRight j c `eqP` (unpackS . T.justifyRight j c) + where j = fromIntegral (k :: Word8) +tl_justifyRight k c = justifyRight j c `eqP` + (unpackS . TL.justifyRight (fromIntegral j) c) + where j = fromIntegral (k :: Word8) +t_center k c = center j c `eqP` (unpackS . T.center j c) + where j = fromIntegral (k :: Word8) +tl_center k c = center j c `eqP` (unpackS . TL.center (fromIntegral j) c) + where j = fromIntegral (k :: Word8) + +t_elem c = L.elem c `eqP` T.elem c +tl_elem c = L.elem c `eqP` TL.elem c +sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter (applyFun -> q) (applyFun -> p) + = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) + +t_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . T.filter p) +tl_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . TL.filter p) +t_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) +tl_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) +t_length_filter (applyFun -> p) + = (L.length . L.filter p) `eqP` (T.length . T.filter p) +tl_length_filter (applyFun -> p) + = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) + +sf_findBy (applyFun -> q) (applyFun -> p) + = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_find (applyFun -> p) = L.find p `eqP` T.find p +tl_find (applyFun -> p) = L.find p `eqP` TL.find p +t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) + +sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j + where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l + +t_index :: T.Text -> Int -> Property +t_index xs i = ioProperty $ do + ch <- try (evaluate (T.index xs i)) + pure $ case ch of + Left (_ :: SomeException) -> i < 0 .||. i >= T.length xs + Right c -> i >= 0 .&&. i < T.length xs .&&. c === T.unpack xs L.!! i + +tl_index :: TL.Text -> Int -> Property +tl_index xs i = ioProperty $ do + let i' = fromIntegral i + ch <- try (evaluate (TL.index xs i')) + pure $ case ch of + Left (_ :: SomeException) -> i' < 0 .||. i' >= TL.length xs + Right c -> i >= 0 .&&. i' < TL.length xs .&&. c === TL.unpack xs L.!! i + +t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p +t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t +tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` + TL.count t +t_zip s = L.zip s `eqP` T.zip (packS s) +tl_zip s = L.zip s `eqP` TL.zip (packS s) +sf_zipWith (applyFun -> p) (applyFun2 -> c) s + = (L.zipWith c (L.filter p s) . L.filter p) `eqP` + (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) +t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) +tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) + +t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s +tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s + where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) + conc = T.concat . TL.toChunks +t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> + let s = T.intercalate t ts + in Slow.indices t s === T.indices t s + +t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] +tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] + +t_indices_drop n s pref suff = T.indices s t === Slow.indices s t + where + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_drop n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] +tl_indices_drop_chunked n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + -- constructing a pathologically chunked haystack + t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff + +t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t + where + s = T.singleton c + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + s = TL.singleton c + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +-- Make a stream appear shorter than it really is, to ensure that +-- functions that consume inaccurately sized streams behave +-- themselves. +shorten :: Int -> S.Stream a -> S.Stream a +shorten n t@(S.Stream arr off len) + | n > 0 = S.Stream arr off (smaller (exactSize n) len) + | otherwise = t + +testText :: TestTree +testText = + testGroup "Text" [ + testGroup "creation/elimination" [ + testProperty "t_pack_unpack" t_pack_unpack, + testProperty "tl_pack_unpack" tl_pack_unpack, + testProperty "t_stream_unstream" t_stream_unstream, + testProperty "tl_stream_unstream" tl_stream_unstream, + testProperty "t_reverse_stream" t_reverse_stream, + testProperty "t_singleton" t_singleton, + testProperty "tl_singleton" tl_singleton, + testProperty "tl_unstreamChunks" tl_unstreamChunks, + testProperty "tl_chunk_unchunk" tl_chunk_unchunk, + testProperty "tl_from_to_strict" tl_from_to_strict + ], + + testGroup "transformations" [ + testProperty "s_map" s_map, + testProperty "s_map_s" s_map_s, + testProperty "sf_map" sf_map, + + testProperty "t_map" t_map, + testProperty "tl_map" tl_map, + testProperty "t_map_map" t_map_map, + testProperty "tl_map_map" tl_map_map, + testProperty "t_length_map" t_length_map, + testProperty "tl_length_map" tl_length_map, + + testProperty "s_intercalate" s_intercalate, + testProperty "t_intercalate" t_intercalate, + testProperty "tl_intercalate" tl_intercalate, + testProperty "t_length_intercalate" t_length_intercalate, + testProperty "tl_length_intercalate" tl_length_intercalate, + testProperty "s_intersperse" s_intersperse, + testProperty "s_intersperse_s" s_intersperse_s, + testProperty "sf_intersperse" sf_intersperse, + testProperty "t_intersperse" t_intersperse, + testProperty "tl_intersperse" tl_intersperse, + testProperty "t_length_intersperse" t_length_intersperse, + testProperty "tl_length_intersperse" tl_length_intersperse, + testProperty "t_transpose" t_transpose, + testProperty "tl_transpose" tl_transpose, + testProperty "t_reverse" t_reverse, + testProperty "tl_reverse" tl_reverse, + testProperty "t_reverse_short" t_reverse_short, + testProperty "t_replace" t_replace, + testProperty "tl_replace" tl_replace, + + testGroup "case conversion" [ + testProperty "s_toCaseFold_length" s_toCaseFold_length, + testProperty "sf_toCaseFold_length" sf_toCaseFold_length, + testProperty "t_toCaseFold_length" t_toCaseFold_length, + testProperty "tl_toCaseFold_length" tl_toCaseFold_length, + testProperty "t_toCaseFold_exceptions" t_toCaseFold_exceptions, + testProperty "t_toCaseFold_cherokeeLower" t_toCaseFold_cherokeeLower, + testProperty "t_toCaseFold_cherokeeUpper" t_toCaseFold_cherokeeUpper, + + testProperty "t_toLower_length" t_toLower_length, + testProperty "t_toLower_lower" t_toLower_lower, + testProperty "tl_toLower_lower" tl_toLower_lower, + testProperty "t_toLower_dotted_i" t_toLower_dotted_i, + + testProperty "t_toUpper_length" t_toUpper_length, + testProperty "t_toUpper_upper" t_toUpper_upper, + testProperty "tl_toUpper_upper" tl_toUpper_upper, + testProperty "t_toUpper_exceptions" t_toUpper_exceptions, + + testProperty "t_toTitle_title" t_toTitle_title, + testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, + testProperty "t_toTitle_exceptions" t_toTitle_exceptions, + + testProperty "t_toUpper_idempotent" t_toUpper_idempotent, + testProperty "t_toLower_idempotent" t_toLower_idempotent, + testProperty "t_toCaseFold_idempotent" t_toCaseFold_idempotent, + + testProperty "ascii_toLower" ascii_toLower, + testProperty "ascii_toUpper" ascii_toUpper, + testProperty "ascii_toTitle" ascii_toTitle, + testProperty "ascii_toCaseFold" ascii_toCaseFold + ], + +#if MIN_VERSION_base(4, 15, 0) + -- Requires matching version of Unicode in base and text + testGroup "char case conversion" $ if T.unicodeVersion == G.unicodeVersion then [ + testProperty "t_toCaseFold_char" t_toCaseFold_char, + testProperty "t_toLower_char" t_toLower_char, + testProperty "t_toUpper_char" t_toUpper_char, + testProperty "t_toTitle_char" t_toTitle_char + ] else [], +#endif + + testGroup "justification" [ + testProperty "s_justifyLeft" s_justifyLeft, + testProperty "s_justifyLeft_s" s_justifyLeft_s, + testProperty "sf_justifyLeft" sf_justifyLeft, + testProperty "t_justifyLeft" t_justifyLeft, + testProperty "tl_justifyLeft" tl_justifyLeft, + testProperty "t_justifyRight" t_justifyRight, + testProperty "tl_justifyRight" tl_justifyRight, + testProperty "t_center" t_center, + testProperty "tl_center" tl_center + ] + ], + + testGroup "searching" [ + testProperty "t_elem" t_elem, + testProperty "tl_elem" tl_elem, + testProperty "sf_elem" sf_elem, + testProperty "sf_filter" sf_filter, + testProperty "t_filter" t_filter, + testProperty "tl_filter" tl_filter, + testProperty "t_filter_filter" t_filter_filter, + testProperty "tl_filter_filter" tl_filter_filter, + testProperty "t_length_filter" t_length_filter, + testProperty "tl_length_filter" tl_length_filter, + testProperty "sf_findBy" sf_findBy, + testProperty "t_find" t_find, + testProperty "tl_find" tl_find, + testProperty "t_partition" t_partition, + testProperty "tl_partition" tl_partition + ], + + testGroup "indexing" [ + testProperty "sf_index" sf_index, + testProperty "t_index" t_index, + testProperty "tl_index" tl_index, + testProperty "t_findIndex" t_findIndex, + testProperty "t_count" t_count, + testProperty "tl_count" tl_count, + testProperty "t_indices" t_indices, + testProperty "tl_indices" tl_indices, + testProperty "t_indices_occurs" t_indices_occurs, + + testProperty "t_indices_drop5" t_indices_drop5, + testProperty "tl_indices_drop5" tl_indices_drop5, + testProperty "t_indices_drop" t_indices_drop, + testProperty "tl_indices_drop" tl_indices_drop, + testProperty "tl_indices_chunked" tl_indices_chunked, + testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, + testProperty "t_indices_char_drop" t_indices_char_drop, + testProperty "tl_indices_char_drop" tl_indices_char_drop + ], + + testGroup "zips" [ + testProperty "t_zip" t_zip, + testProperty "tl_zip" tl_zip, + testProperty "sf_zipWith" sf_zipWith, + testProperty "t_zipWith" t_zipWith, + testProperty "tl_zipWith" tl_zipWith, + testProperty "t_length_zipWith" t_length_zipWith, + testProperty "tl_length_zipWith" tl_length_zipWith + ] + ]
tests/Tests/Properties/Transcoding.hs view
@@ -1,7 +1,10 @@ -- | Tests for encoding and decoding {-# LANGUAGE CPP, OverloadedStrings, ScopedTypeVariables #-}-{-# OPTIONS_GHC -fno-warn-missing-signatures #-}+{-# OPTIONS_GHC -Wno-missing-signatures #-}+{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}+{-# OPTIONS_GHC -Wno-x-partial #-}+ module Tests.Properties.Transcoding ( testTranscoding ) where
tests/Tests/QuickCheckUtils.hs view
@@ -1,307 +1,326 @@--- | This module provides quickcheck utilities, e.g. arbitrary and show--- instances, and comparison functions, so we can focus on the actual properties--- in the 'Tests.Properties' module.----{-# LANGUAGE CPP #-}-{-# LANGUAGE DeriveFunctor #-}-{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE NamedFieldPuns #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE ScopedTypeVariables #-}--{-# OPTIONS_GHC -fno-warn-orphans #-}--module Tests.QuickCheckUtils- ( BigInt(..)- , NotEmpty(..)- , Sqrt(..)- , SpacyString(..)- , SkewedBool(..)-- , Precision(..)- , precision-- , DecodeErr(..)- , genDecodeErr-- , Stringy(..)- , unpack2- , eq- , eqP- , eqPSqrt-- , write_read- ) where--import Control.Arrow ((***))-import Data.Bool (bool)-import Data.Char (isSpace)-import Data.Text.Foreign (I8)-import Data.Text.Lazy.Builder.RealFloat (FPFormat(..))-import Data.Word (Word8, Word16)-import GHC.IO.Encoding.Types (TextEncoding(TextEncoding,textEncodingName))-import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, discard, counterexample, scale, (.&&.), NonEmptyList(..), forAll, getPositive)-import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized)-import Test.Tasty (TestTree, testGroup)-import Test.Tasty.QuickCheck (testProperty)-import Tests.Utils-import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as BL-import qualified Data.Text as T-import qualified Data.Text.Encoding.Error as T-import qualified Data.Text.Internal.Fusion as TF-import qualified Data.Text.Internal.Fusion.Common as TF-import qualified Data.Text.Internal.Lazy as TL-import qualified Data.Text.Internal.Lazy.Fusion as TLF-import qualified Data.Text.Lazy as TL-import qualified System.IO as IO--genWord8 :: Gen Word8-genWord8 = chooseAny--genWord16 :: Gen Word16-genWord16 = chooseAny--instance Arbitrary I8 where- arbitrary = arbitrarySizedIntegral- shrink = shrinkIntegral--instance Arbitrary B.ByteString where- arbitrary = B.pack `fmap` listOf genWord8- shrink = map B.pack . shrink . B.unpack--instance Arbitrary BL.ByteString where- arbitrary = oneof- [ BL.fromChunks <$> arbitrary- -- so that a single utf8 code point could appear split over up to 4 chunks- , BL.fromChunks . map B.singleton <$> listOf genWord8- -- so that a code point with 4 byte long utf8 representation- -- could appear split over 3 non-singleton chunks- , (\a b c -> BL.fromChunks [a, b, c])- <$> arbitrary- <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8)- <*> arbitrary- ]- shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs)---- | For tests that have O(n^2) running times or input sizes, resize--- their inputs to the square root of the originals.-newtype Sqrt a = Sqrt { unSqrt :: a }- deriving (Eq, Show)--instance Arbitrary a => Arbitrary (Sqrt a) where- arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary- where- smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs- shrink = map Sqrt . shrink . unSqrt--instance Arbitrary T.Text where- arbitrary = do- t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary- -- Generate chunks that start in the middle of their buffers.- (\i -> T.drop i t) <$> choose (0, T.length t)- shrink = map T.pack . shrink . T.unpack--instance Arbitrary TL.Text where- arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary- shrink = map TL.pack . shrink . TL.unpack--newtype BigInt = Big Integer- deriving (Eq, Show)--instance Arbitrary BigInt where- arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e)- shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l]- where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer--newtype NotEmpty a = NotEmpty { notEmpty :: a }- deriving (Eq, Ord, Show)--instance Arbitrary (NotEmpty T.Text) where- arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary- shrink = fmap (NotEmpty . T.pack . getNonEmpty)- . shrink . NonEmpty . T.unpack . notEmpty--instance Arbitrary (NotEmpty TL.Text) where- arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary- shrink = fmap (NotEmpty . TL.pack . getNonEmpty)- . shrink . NonEmpty . TL.unpack . notEmpty--data DecodeErr = Lenient | Ignore | Strict | Replace- deriving (Show, Eq, Bounded, Enum)--genDecodeErr :: DecodeErr -> Gen T.OnDecodeError-genDecodeErr Lenient = return T.lenientDecode-genDecodeErr Ignore = return T.ignore-genDecodeErr Strict = return T.strictDecode-genDecodeErr Replace = (\c _ _ -> c) <$> frequency- [ (1, return Nothing)- , (50, Just <$> arbitraryUnicodeChar)- ]--instance Arbitrary DecodeErr where- arbitrary = arbitraryBoundedEnum--class Stringy s where- packS :: String -> s- unpackS :: s -> String- splitAtS :: Int -> s -> (s,s)- packSChunkSize :: Int -> String -> s- packSChunkSize _ = packS--instance Stringy String where- packS = id- unpackS = id- splitAtS = splitAt--instance Stringy (TF.Stream Char) where- packS = TF.streamList- unpackS = TF.unstreamList- splitAtS n s = (TF.take n s, TF.drop n s)--instance Stringy T.Text where- packS = T.pack- unpackS = T.unpack- splitAtS = T.splitAt--instance Stringy TL.Text where- packSChunkSize k = TLF.unstreamChunks k . TF.streamList- packS = TL.pack- unpackS = TL.unpack- splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) .- TL.splitAt . fromIntegral--unpack2 :: (Stringy s) => (s,s) -> (String,String)-unpack2 = unpackS *** unpackS---- Do two functions give the same answer?-eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property-eq a b s = a s =^= b s---- What about with the RHS packed?-eqP :: (Eq a, Show a, Stringy s) =>- (String -> a) -> (s -> a) -> String -> Word8 -> Property-eqP f g s w = counterexample "orig" (f s =^= g t) .&&.- counterexample "mini" (f s =^= g mini) .&&.- counterexample "head" (f sa =^= g ta) .&&.- counterexample "tail" (f sb =^= g tb)- where t = packS s- mini = packSChunkSize 10 s- (sa,sb) = splitAt m s- (ta,tb) = splitAtS m t- l = length s- m | l == 0 = n- | otherwise = n `mod` l- n = fromIntegral w--eqPSqrt :: (Eq a, Show a, Stringy s) =>- (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property-eqPSqrt f g s = eqP f g (unSqrt s)--instance Arbitrary FPFormat where- arbitrary = arbitraryBoundedEnum--newtype Precision a = Precision (Maybe Int)- deriving (Eq, Show)--precision :: a -> Precision a -> Maybe Int-precision _ (Precision prec) = prec--arbitraryPrecision :: Int -> Gen (Precision a)-arbitraryPrecision maxDigits = Precision <$> do- n <- choose (-1,maxDigits)- return $ if n == -1- then Nothing- else Just n--instance Arbitrary (Precision Float) where- arbitrary = arbitraryPrecision 11- shrink = map Precision . shrink . precision undefined--instance Arbitrary (Precision Double) where- arbitrary = arbitraryPrecision 22- shrink = map Precision . shrink . precision undefined--#if !MIN_VERSION_QuickCheck(2,14,3)-instance Arbitrary IO.Newline where- arbitrary = oneof [return IO.LF, return IO.CRLF]--instance Arbitrary IO.NewlineMode where- arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary-#endif--instance Arbitrary IO.BufferMode where- arbitrary = oneof [ return IO.NoBuffering,- return IO.LineBuffering,- return (IO.BlockBuffering Nothing),- (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap`- genWord16 ]---- This test harness is complex! What property are we checking?------ Reading after writing a multi-line file should give the same--- results as were written.------ What do we vary while checking this property?--- * The lines themselves, scrubbed to contain neither CR nor LF. (By--- working with a list of lines, we ensure that the data will--- sometimes contain line endings.)--- * Newline translation mode.--- * Buffering.-write_read :: forall a b c.- (Eq a, Show a, Show c, Arbitrary c)- => ([b] -> a)- -> ((Char -> Bool) -> b -> b)- -> (IO.Handle -> a -> IO ())- -> (IO.Handle -> IO a)- -> (c -> [b])- -> [TestTree]-write_read unline filt writer reader modData- = encodings <&> \enc@TextEncoding {textEncodingName} -> testGroup textEncodingName- [ testProperty "NoBuffering" $ propTest enc (pure IO.NoBuffering)- , testProperty "LineBuffering" $ propTest enc (pure IO.LineBuffering)- , testProperty "BlockBuffering" $ propTest enc blockBuffering- ]- where- propTest :: TextEncoding -> Gen IO.BufferMode -> IO.NewlineMode -> c -> Property- propTest _ _ (IO.NewlineMode IO.LF IO.CRLF) _ = discard- propTest enc genBufferMode nl d = forAll genBufferMode $ \mode -> ioProperty $ withTempFile $ \_ h -> do- let ts = modData d- t = unline . map (filt (not . (`elem` "\r\n"))) $ ts- IO.hSetEncoding h enc- IO.hSetNewlineMode h nl- IO.hSetBuffering h mode- () <- writer h t- IO.hSeek h IO.AbsoluteSeek 0- r <- reader h- let isEq = r == t- seq isEq $ pure $ counterexample (show r ++ bool " /= " " == " isEq ++ show t) isEq-- encodings = [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be]-- blockBuffering :: Gen IO.BufferMode- blockBuffering = IO.BlockBuffering <$> fmap (fmap $ min 4 . getPositive) arbitrary---- Generate various Unicode space characters with high probability-arbitrarySpacyChar :: Gen Char-arbitrarySpacyChar = oneof- [ arbitraryUnicodeChar- , elements $ filter isSpace [minBound..maxBound]- ]--newtype SpacyString = SpacyString { getSpacyString :: String }- deriving (Eq, Ord, Show, Read)--instance Arbitrary SpacyString where- arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar- shrink (SpacyString xs) = SpacyString `fmap` shrink xs--newtype SkewedBool = Skewed { getSkewed :: Bool }- deriving Show--instance Arbitrary SkewedBool where- arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)]--(<&>) :: [a] -> (a -> b) -> [b]-(<&>) = flip fmap-+-- | This module provides quickcheck utilities, e.g. arbitrary and show +-- instances, and comparison functions, so we can focus on the actual properties +-- in the 'Tests.Properties' module. +-- +{-# LANGUAGE CPP #-} +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} + +{-# OPTIONS_GHC -fno-warn-orphans #-} + +module Tests.QuickCheckUtils + ( BigInt(..) + , NotEmpty(..) + , Sqrt(..) + , SpacyString(..) + , SkewedBool(..) + + , Precision(..) + , precision + + , DecodeErr(..) + , genDecodeErr + + , Stringy(..) + , unpack2 + , eq + , eqP + , eqPSqrt + + , write_read + ) where + +import Control.Arrow ((***)) +import Control.Monad (when) +import Data.Char (isSpace) +import Data.IORef (writeIORef) +import Data.Text.Foreign (I8) +import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) +import Data.Word (Word8) +#if !MIN_VERSION_QuickCheck(2,17,0) +import Data.Word (Word16) +#endif +import qualified GHC.IO.Buffer as GIO +import qualified GHC.IO.Handle.Internals as GIO +import qualified GHC.IO.Handle.Types as GIO +import GHC.IO.Encoding.Types (TextEncoding(textEncodingName)) +import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, counterexample, scale, (.&&.), NonEmptyList(..), forAllShrink) +import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) +import Tests.Utils +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import qualified Data.Text as T +import qualified Data.Text.Encoding.Error as T +import qualified Data.Text.Internal.Fusion as TF +import qualified Data.Text.Internal.Fusion.Common as TF +import qualified Data.Text.Internal.Lazy as TL +import qualified Data.Text.Internal.Lazy.Fusion as TLF +import qualified Data.Text.Lazy as TL +import qualified System.IO as IO + +genWord8 :: Gen Word8 +genWord8 = chooseAny + +instance Arbitrary I8 where + arbitrary = arbitrarySizedIntegral + shrink = shrinkIntegral + +instance Arbitrary B.ByteString where + arbitrary = B.pack `fmap` listOf genWord8 + shrink = map B.pack . shrink . B.unpack + +instance Arbitrary BL.ByteString where + arbitrary = oneof + [ BL.fromChunks <$> arbitrary + -- so that a single utf8 code point could appear split over up to 4 chunks + , BL.fromChunks . map B.singleton <$> listOf genWord8 + -- so that a code point with 4 byte long utf8 representation + -- could appear split over 3 non-singleton chunks + , (\a b c -> BL.fromChunks [a, b, c]) + <$> arbitrary + <*> ((\a b -> B.pack [a, b]) <$> genWord8 <*> genWord8) + <*> arbitrary + ] + shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) + +-- | For tests that have O(n^2) running times or input sizes, resize +-- their inputs to the square root of the originals. +newtype Sqrt a = Sqrt { unSqrt :: a } + deriving (Eq, Show) + +instance Arbitrary a => Arbitrary (Sqrt a) where + arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary + where + smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs + shrink = map Sqrt . shrink . unSqrt + +instance Arbitrary T.Text where + arbitrary = do + t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary + -- Generate chunks that start in the middle of their buffers. + (\i -> T.drop i t) <$> choose (0, T.length t) + shrink = map T.pack . shrink . T.unpack + +instance Arbitrary TL.Text where + arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary + shrink = map TL.pack . shrink . TL.unpack + +newtype BigInt = Big Integer + deriving (Eq, Show) + +instance Arbitrary BigInt where + arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e) + shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] + where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer + +newtype NotEmpty a = NotEmpty { notEmpty :: a } + deriving (Eq, Ord, Show) + +instance Arbitrary (NotEmpty T.Text) where + arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . T.pack . getNonEmpty) + . shrink . NonEmpty . T.unpack . notEmpty + +instance Arbitrary (NotEmpty TL.Text) where + arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . TL.pack . getNonEmpty) + . shrink . NonEmpty . TL.unpack . notEmpty + +data DecodeErr = Lenient | Ignore | Strict | Replace + deriving (Show, Eq, Bounded, Enum) + +genDecodeErr :: DecodeErr -> Gen T.OnDecodeError +genDecodeErr Lenient = return T.lenientDecode +genDecodeErr Ignore = return T.ignore +genDecodeErr Strict = return T.strictDecode +genDecodeErr Replace = (\c _ _ -> c) <$> frequency + [ (1, return Nothing) + , (50, Just <$> arbitraryUnicodeChar) + ] + +instance Arbitrary DecodeErr where + arbitrary = arbitraryBoundedEnum + +class Stringy s where + packS :: String -> s + unpackS :: s -> String + splitAtS :: Int -> s -> (s,s) + packSChunkSize :: Int -> String -> s + packSChunkSize _ = packS + +instance Stringy String where + packS = id + unpackS = id + splitAtS = splitAt + +instance Stringy (TF.Stream Char) where + packS = TF.streamList + unpackS = TF.unstreamList + splitAtS n s = (TF.take n s, TF.drop n s) + +instance Stringy T.Text where + packS = T.pack + unpackS = T.unpack + splitAtS = T.splitAt + +instance Stringy TL.Text where + packSChunkSize k = TLF.unstreamChunks k . TF.streamList + packS = TL.pack + unpackS = TL.unpack + splitAtS = ((TL.lazyInvariant *** TL.lazyInvariant) .) . + TL.splitAt . fromIntegral + +unpack2 :: (Stringy s) => (s,s) -> (String,String) +unpack2 = unpackS *** unpackS + +-- Do two functions give the same answer? +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property +eq a b s = a s =^= b s + +-- What about with the RHS packed? +eqP :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> String -> Word8 -> Property +eqP f g s w = counterexample "orig" (f s =^= g t) .&&. + counterexample "mini" (f s =^= g mini) .&&. + counterexample "head" (f sa =^= g ta) .&&. + counterexample "tail" (f sb =^= g tb) + where t = packS s + mini = packSChunkSize 10 s + (sa,sb) = splitAt m s + (ta,tb) = splitAtS m t + l = length s + m | l == 0 = n + | otherwise = n `mod` l + n = fromIntegral w + +eqPSqrt :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property +eqPSqrt f g s = eqP f g (unSqrt s) + +instance Arbitrary FPFormat where + arbitrary = arbitraryBoundedEnum + +newtype Precision a = Precision (Maybe Int) + deriving (Eq, Show) + +precision :: a -> Precision a -> Maybe Int +precision _ (Precision prec) = prec + +arbitraryPrecision :: Int -> Gen (Precision a) +arbitraryPrecision maxDigits = Precision <$> do + n <- choose (-1,maxDigits) + return $ if n == -1 + then Nothing + else Just n + +instance Arbitrary (Precision Float) where + arbitrary = arbitraryPrecision 11 + shrink = map Precision . shrink . precision undefined + +instance Arbitrary (Precision Double) where + arbitrary = arbitraryPrecision 22 + shrink = map Precision . shrink . precision undefined + +#if !MIN_VERSION_QuickCheck(2,14,3) +instance Arbitrary IO.Newline where + arbitrary = oneof [return IO.LF, return IO.CRLF] + +instance Arbitrary IO.NewlineMode where + arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary +#endif + +#if !MIN_VERSION_QuickCheck(2,17,0) +instance Arbitrary IO.BufferMode where + arbitrary = oneof [ return IO.NoBuffering, + return IO.LineBuffering, + return (IO.BlockBuffering Nothing), + (IO.BlockBuffering . Just . (+1) . fromIntegral) `fmap` + genWord16 ] + +genWord16 :: Gen Word16 +genWord16 = chooseAny +#endif + +-- This test harness is complex! What property are we checking? +-- +-- Reading after writing a multi-line file should give the same +-- results as were written. +-- +-- What do we vary while checking this property? +-- * The lines themselves, scrubbed to contain neither CR nor LF. (By +-- working with a list of lines, we ensure that the data will +-- sometimes contain line endings.) +-- * Newline translation mode. +-- * Buffering. +write_read :: forall a. + (Eq a, Show a) + => Gen a + -> (a -> [a]) + -> (a -> a) -- ^ replace '\n' with '\r\n' (for multiline tests) or append '\r' (for single-line tests) + -> (IO.Handle -> a -> IO ()) + -> (IO.Handle -> IO a) + -> Property +write_read genTxt shrinkTxt expandNl writer reader + = forAllShrink genEncoding shrinkEncoding propTest + where + propTest :: TextEncoding -> IO.BufferMode -> Property + propTest enc mode = forAllShrink genTxt shrinkTxt $ \txt -> ioProperty $ do + file <- emptyTempFile + let with nl k = IO.withFile file IO.ReadWriteMode $ \h -> do + IO.hSetEncoding h enc + IO.hSetBuffering h mode + IO.hSetNewlineMode h nl + setSmallBuffer h + k h + -- Put a very small buffer in Handle to easily test boundary conditions in `writeBlocks` + setSmallBuffer h = GIO.withHandle_ "setSmallBuffer" h $ \h_ -> do + buf <- GIO.newCharBuffer 9 GIO.WriteBuffer + writeIORef (GIO.haCharBuffer h_) buf + readExpecting h txt' msg = do + out <- reader h + when (txt' /= out) $ error (show txt' ++ " /= " ++ show out ++ msg) + -- 'reader' may be 'hGetContents', which closes the handle + -- So we reopen a new file every time. + + -- Test with CRLF encoding + with (IO.NewlineMode IO.CRLF IO.CRLF) $ \h -> do + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 1)" + + -- Re-read without CRLF decoding to check that we did encode CRLF correctly + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + readExpecting h (expandNl txt) " (at location 2)" + + -- Test without CRLF encoding + with (IO.NewlineMode IO.LF IO.LF) $ \h -> do + IO.hSetFileSize h 0 + writer h txt + IO.hSeek h IO.AbsoluteSeek 0 + readExpecting h txt " (at location 3)" + + genEncoding = elements [IO.utf8, IO.utf8_bom, IO.utf16, IO.utf16le, IO.utf16be, IO.utf32, IO.utf32le, IO.utf32be] + shrinkEncoding enc = if textEncodingName enc == textEncodingName IO.utf8 then [] else [IO.utf8] + +-- Generate various Unicode space characters with high probability +arbitrarySpacyChar :: Gen Char +arbitrarySpacyChar = oneof + [ arbitraryUnicodeChar + , elements $ filter isSpace [minBound..maxBound] + ] + +newtype SpacyString = SpacyString { getSpacyString :: String } + deriving (Eq, Ord, Show, Read) + +instance Arbitrary SpacyString where + arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar + shrink (SpacyString xs) = SpacyString `fmap` shrink xs + +newtype SkewedBool = Skewed { getSkewed :: Bool } + deriving Show + +instance Arbitrary SkewedBool where + arbitrary = Skewed <$> frequency [(1, pure False), (5, pure True)]
tests/Tests/RebindableSyntaxTest.hs view
@@ -1,14 +1,18 @@-{-# LANGUAGE RebindableSyntax, TemplateHaskell #-}--module Tests.RebindableSyntaxTest where--import qualified Data.Text as Text-import Language.Haskell.TH.Syntax (lift)-import Test.Tasty.HUnit (testCase, assertEqual)-import Test.Tasty (TestTree, testGroup)-import Prelude (($))--tests :: TestTree-tests = testGroup "RebindableSyntax"- [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a")- ]+{-# LANGUAGE CPP, RebindableSyntax, TemplateHaskell #-} + +module Tests.RebindableSyntaxTest where + +import qualified Data.Text as Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertEqual) +import Test.Tasty (TestTree, testGroup) +import Prelude (($)) + +tests :: TestTree +tests = testGroup "RebindableSyntax" + [ testCase "test" $ assertEqual "a" $(lift (Text.pack "a")) (Text.pack "a") + ]
tests/Tests/Regressions.hs view
@@ -1,204 +1,236 @@--- | Regression tests for specific bugs.----{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE ScopedTypeVariables #-}--module Tests.Regressions- (- tests- ) where--import Control.Exception (SomeException, handle)-import Data.Char (isLetter, chr)-import GHC.Exts (Int(..), sizeofByteArray#)-import System.IO-import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure)-import qualified Data.ByteString as B-import Data.ByteString.Char8 ()-import qualified Data.ByteString.Lazy as LB-import qualified Data.Text as T-import qualified Data.Text.Array as TA-import qualified Data.Text.Encoding as TE-import qualified Data.Text.Encoding.Error as E-import qualified Data.Text.Internal as T-import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E-import qualified Data.Text.Internal.Lazy.Fusion as LF-import qualified Data.Text.IO as T-import qualified Data.Text.Lazy as LT-import qualified Data.Text.Lazy.Builder as TB-import qualified Data.Text.Lazy.Encoding as LE-import qualified Data.Text.Unsafe as T-import qualified Test.Tasty as F-import qualified Test.Tasty.HUnit as F-import Test.Tasty.HUnit ((@?=))-import System.Directory (removeFile)--import Tests.Utils (withTempFile)---- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring--- caused either a segfault or attempt to allocate a negative number--- of bytes.-lazy_encode_crash :: IO ()-lazy_encode_crash = withTempFile $ \ _ h ->- LB.hPut h . LE.encodeUtf8 . LT.pack . replicate 100000 $ 'a'---- Reported by Pieter Laeremans: attempting to read an incorrectly--- encoded file can result in a crash in the RTS (i.e. not merely an--- exception).-hGetContents_crash :: IO ()-hGetContents_crash = do- (path, h) <- openTempFile "." "crashy.txt"- B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) >> hClose h- h' <- openFile path ReadMode- hSetEncoding h' utf8- handle (\(_::SomeException) -> return ()) $- T.hGetContents h' >> assertFailure "T.hGetContents should crash"- hClose h'- removeFile path---- Reported by Ian Lynagh: attempting to allocate a sufficiently large--- string (via either Array.new or Text.replicate) could result in an--- integer overflow.-replicate_crash :: IO ()-replicate_crash = handle (\(_::SomeException) -> return ()) $- T.replicate (2^power) "0123456789abcdef" `seq`- assertFailure "T.replicate should crash"- where- power | maxBound == (2147483647::Int) = 28- | otherwise = 60 :: Int---- Reported by John Millikin: a UTF-8 decode error handler could--- return a bogus substitution character, which we would write without--- checking.-utf8_decode_unsafe :: IO ()-utf8_decode_unsafe = do- let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80"- assertBool "broken error recovery shouldn't break us" (t == "\xfffd")---- Reported by Eric Seidel: we mishandled mapping Chars that fit in a--- single Word16 to Chars that require two.-mapAccumL_resize :: IO ()-mapAccumL_resize = do- let f a _ = (a, '\65536')- count = 5- val = T.mapAccumL f (0::Int) (T.replicate count "a")- assertEqual "mapAccumL should correctly fill buffers for four-byte results"- (0, T.replicate count "\65536") val- assertEqual "mapAccumL should correctly size buffers for four-byte results"- (count * 4) (T.lengthWord8 (snd val))---- See GitHub #197-t197 :: IO ()-t197 =- assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00")- where- currencyParser x = cond == 1- where- cond = length fltr- fltr = filter (== ',') x--t221 :: IO ()-t221 =- assertEqual "toLower of large input shouldn't crash"- (T.toLower (T.replicate 200000 "0") `seq` ())- ()--t227 :: IO ()-t227 =- assertEqual "take (-3) shouldn't crash with overflow"- (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?")- 0--t280_fromString :: IO ()-t280_fromString =- assertEqual "TB.fromString performs replacement on invalid scalar values"- (TB.toLazyText (TB.fromString "\xD800"))- (LT.pack "\xFFFD")--t280_singleton :: IO ()-t280_singleton =- assertEqual "TB.singleton performs replacement on invalid scalar values"- (TB.toLazyText (TB.singleton '\xD800'))- (LT.pack "\xFFFD")---- See GitHub issue #301--- This tests whether the "TEXT take . drop -> unfused" rule is applied to the--- slice function. When the slice function is fused, a new array will be--- constructed that is shorter than the original array. Without fusion the--- array remains unmodified.-t301 :: IO ()-t301 = do- assertEqual "The length of the array remains the same despite slicing"- (I# (sizeofByteArray# originalArr))- (I# (sizeofByteArray# newArr))-- assertEqual "The new array still contains the original value"- (T.Text (TA.ByteArray newArr) originalOff originalLen)- original- where- !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890"- !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original--t330 :: IO ()-t330 = do- let decodeL = LE.decodeUtf8With E.lenientDecode- assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked"- (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]]))- (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]]))---- Stream decoders should not loop on incomplete code points-t525 :: IO ()-t525 = do- let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs)- decodeUtf8With E.lenientDecode "\xC0" @?= "\65533"- LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533"- LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533"- LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533"- LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533"---- Stream decoders skip one invalid byte at a time-t528 :: IO ()-t528 = do- let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs)- decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536"- LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536"- LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536"- LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0"- LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280"--t529 :: IO ()-t529 = do- let decode = TE.decodeUtf8With E.lenientDecode- -- https://github.com/haskell/bytestring/issues/575- assertEqual "Data.ByteString.isValidUtf8 should work correctly"- (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0]))- (decode (B.pack (33 : replicate 31 0 ++ [128, 0])))---- See Github #559--- filter/filter fusion rules should apply predicates in the right order.-t559 :: IO ()-t559 = do- T.filter undefined (T.filter (const False) "a") @?= ""- LT.filter undefined (LT.filter (const False) "a") @?= ""--tests :: F.TestTree-tests = F.testGroup "Regressions"- [ F.testCase "hGetContents_crash" hGetContents_crash- , F.testCase "lazy_encode_crash" lazy_encode_crash- , F.testCase "mapAccumL_resize" mapAccumL_resize- , F.testCase "replicate_crash" replicate_crash- , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe- , F.testCase "t197" t197- , F.testCase "t221" t221- , F.testCase "t227" t227- , F.testCase "t280/fromString" t280_fromString- , F.testCase "t280/singleton" t280_singleton- , F.testCase "t301" t301- , F.testCase "t330" t330- , F.testCase "t525" t525- , F.testCase "t528" t528- , F.testCase "t529" t529- , F.testCase "t559" t559- ]+-- | Regression tests for specific bugs. +-- +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Tests.Regressions + ( + tests + ) where + +import Control.Exception (ErrorCall, SomeException, handle, evaluate, displayException, try) +import Data.Char (isLetter, chr) +import GHC.Exts (Int(..), sizeofByteArray#) +import System.IO +import System.IO.Temp (withSystemTempFile) +import Test.Tasty.HUnit (assertBool, assertEqual, assertFailure, (@?=)) +import qualified Data.ByteString as B +import Data.ByteString.Char8 () +import qualified Data.ByteString.Lazy as LB +import Data.Semigroup (stimes) +import qualified Data.Text as T +import qualified Data.Text.Array as TA +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as E +import qualified Data.Text.Internal as T +import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E +import qualified Data.Text.Internal.Lazy.Fusion as LF +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as LT +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Text.Lazy.Encoding as LE +import qualified Data.Text.Unsafe as T +import qualified Test.Tasty as F +import qualified Test.Tasty.HUnit as F +import Tests.Utils (withTempFile) +import System.IO.Error (isFullError) + +-- Reported by Michael Snoyman: UTF-8 encoding a large lazy bytestring +-- caused either a segfault or attempt to allocate a negative number +-- of bytes. +lazy_encode_crash :: IO () +lazy_encode_crash = withTempFile $ \ _ h -> do + putRes <- try $ LB.hPut h $ LE.encodeUtf8 $ LT.pack $ replicate 100000 'a' + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> pure () + +-- Reported by Pieter Laeremans: attempting to read an incorrectly +-- encoded file can result in a crash in the RTS (i.e. not merely an +-- exception). +hGetContents_crash :: IO () +hGetContents_crash = withSystemTempFile "crashy.txt" $ \path h -> do + putRes <- try $ B.hPut h (B.pack [0x78, 0xc4 ,0x0a]) + case putRes of + Left e + -- If disk is full (as it happens on some of our CI runners), it's not our issue, skip it + | isFullError e -> pure () + | otherwise -> assertFailure $ "hPut crashed because of " ++ displayException e + Right () -> do + hClose h + h' <- openFile path ReadMode + hSetEncoding h' utf8 + handle (\(_::SomeException) -> pure ()) $ + T.hGetContents h' >> assertFailure "T.hGetContents should crash" + hClose h' + +-- Reported by Ian Lynagh: attempting to allocate a sufficiently large +-- string (via either Array.new or Text.replicate) could result in an +-- integer overflow. +replicate_crash :: IO () +replicate_crash = handle (\(_::SomeException) -> return ()) $ + T.replicate (2^power) "0123456789abcdef" `seq` + assertFailure "T.replicate should crash" + where + power | maxBound == (2147483647::Int) = 28 + | otherwise = 60 :: Int + +-- Reported by John Millikin: a UTF-8 decode error handler could +-- return a bogus substitution character, which we would write without +-- checking. +utf8_decode_unsafe :: IO () +utf8_decode_unsafe = do + let t = TE.decodeUtf8With (\_ _ -> Just '\xdc00') "\x80" + assertBool "broken error recovery shouldn't break us" (t == "\xfffd") + +-- Reported by Eric Seidel: we mishandled mapping Chars that fit in a +-- single Word16 to Chars that require two. +mapAccumL_resize :: IO () +mapAccumL_resize = do + let f a _ = (a, '\65536') + count = 5 + val = T.mapAccumL f (0::Int) (T.replicate count "a") + assertEqual "mapAccumL should correctly fill buffers for four-byte results" + (0, T.replicate count "\65536") val + assertEqual "mapAccumL should correctly size buffers for four-byte results" + (count * 4) (T.lengthWord8 (snd val)) + +-- See GitHub #197 +t197 :: IO () +t197 = + assertBool "length (filter (==',') \"0,00\") should be 1" (currencyParser "0,00") + where + currencyParser x = cond == 1 + where + cond = length fltr + fltr = filter (== ',') x + +t221 :: IO () +t221 = + assertEqual "toLower of large input shouldn't crash" + (T.toLower (T.replicate 200000 "0") `seq` ()) + () + +t227 :: IO () +t227 = + assertEqual "take (-3) shouldn't crash with overflow" + (T.length $ T.filter isLetter $ T.take (-3) "Hello! How are you doing today?") + 0 + +t280_fromString :: IO () +t280_fromString = + assertEqual "TB.fromString performs replacement on invalid scalar values" + (TB.toLazyText (TB.fromString "\xD800")) + (LT.pack "\xFFFD") + +t280_singleton :: IO () +t280_singleton = + assertEqual "TB.singleton performs replacement on invalid scalar values" + (TB.toLazyText (TB.singleton '\xD800')) + (LT.pack "\xFFFD") + +-- See GitHub issue #301 +-- This tests whether the "TEXT take . drop -> unfused" rule is applied to the +-- slice function. When the slice function is fused, a new array will be +-- constructed that is shorter than the original array. Without fusion the +-- array remains unmodified. +t301 :: IO () +t301 = do + assertEqual "The length of the array remains the same despite slicing" + (I# (sizeofByteArray# originalArr)) + (I# (sizeofByteArray# newArr)) + + assertEqual "The new array still contains the original value" + (T.Text (TA.ByteArray newArr) originalOff originalLen) + original + where + !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" + !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original + +t330 :: IO () +t330 = do + let decodeL = LE.decodeUtf8With E.lenientDecode + assertEqual "The lenient decoding of lazy bytestrings should not depend on how they are chunked" + (decodeL (LB.fromChunks [B.pack [194], B.pack [97, 98, 99]])) + (decodeL (LB.fromChunks [B.pack [194, 97, 98, 99]])) + +-- Stream decoders should not loop on incomplete code points +t525 :: IO () +t525 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0" @?= "\65533" + LE.decodeUtf16BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf16LEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32BEWith E.lenientDecode "\0" @?= "\65533" + LE.decodeUtf32LEWith E.lenientDecode "\0" @?= "\65533" + +-- Stream decoders skip one invalid byte at a time +t528 :: IO () +t528 = do + let decodeUtf8With onErr bs = LF.unstream (E.streamUtf8 onErr bs) + decodeUtf8With E.lenientDecode "\xC0\xF0\x90\x80\x80" @?= "\65533\65536" + LE.decodeUtf16BEWith E.lenientDecode "\xD8\xD8\x00\xDC\x00" @?= "\65533\65536" + LE.decodeUtf16LEWith E.lenientDecode "\xD8\xD8\x00\xD8\x00\xDC" @?= "\65533\65533\65536" + LE.decodeUtf32BEWith E.lenientDecode "\xFF\x00\x00\x00\x00" @?= "\65533\0" + LE.decodeUtf32LEWith E.lenientDecode "\x00\x00\xFF\x00\x00" @?= "\65533\65280" + +t529 :: IO () +t529 = do + let decode = TE.decodeUtf8With E.lenientDecode + -- https://github.com/haskell/bytestring/issues/575 + assertEqual "Data.ByteString.isValidUtf8 should work correctly" + (T.pack (chr 33 : replicate 31 (chr 0) ++ [chr 65533, chr 0])) + (decode (B.pack (33 : replicate 31 0 ++ [128, 0]))) + +-- See Github #559 +-- filter/filter fusion rules should apply predicates in the right order. +t559 :: IO () +t559 = do + T.filter undefined (T.filter (const False) "a") @?= "" + LT.filter undefined (LT.filter (const False) "a") @?= "" + +-- Github #633 +-- stimes checked for an `a` to `Int` to `a` roundtrip, but the `a` and `Int` values could represent different integers. +t633 :: IO () +t633 = + handle (\(_ :: ErrorCall) -> return ()) $ do + _ <- evaluate (stimes (maxBound :: Word) "a" :: T.Text) + assertFailure "should fail" + +t648 :: IO () +t648 = withTempFile $ \_ h -> do + hSetEncoding h utf8 + hSetNewlineMode h (NewlineMode LF CRLF) + hSetBuffering h (BlockBuffering $ Just 4) + let line = T.replicate 2047 "_" + T.hPutStrLn h line + hSeek h AbsoluteSeek 0 + line' <- T.hGetLine h + T.append line "\r" @?= line' + +tests :: F.TestTree +tests = F.testGroup "Regressions" + [ F.testCase "hGetContents_crash" hGetContents_crash + , F.testCase "lazy_encode_crash" lazy_encode_crash + , F.testCase "mapAccumL_resize" mapAccumL_resize + , F.testCase "replicate_crash" replicate_crash + , F.testCase "utf8_decode_unsafe" utf8_decode_unsafe + , F.testCase "t197" t197 + , F.testCase "t221" t221 + , F.testCase "t227" t227 + , F.testCase "t280/fromString" t280_fromString + , F.testCase "t280/singleton" t280_singleton + , F.testCase "t301" t301 + , F.testCase "t330" t330 + , F.testCase "t525" t525 + , F.testCase "t528" t528 + , F.testCase "t529" t529 + , F.testCase "t559" t559 + , F.testCase "t633" t633 + , F.testCase "t648" t648 + ]
@@ -1,133 +1,138 @@-{-# LANGUAGE MagicHash #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TemplateHaskell #-}-{-# LANGUAGE BangPatterns #-}--{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-}-{-# OPTIONS_GHC -Wno-x-partial #-}--module Tests.ShareEmpty- ( tests- ) where--import Control.Exception (evaluate)-import Data.Text-import Language.Haskell.TH.Syntax (lift)-import Test.Tasty.HUnit (testCase, assertFailure, assertEqual)-import Test.Tasty (TestTree, testGroup)-import GHC.Exts-import GHC.Stack-import qualified Data.List as L-import qualified Data.List.NonEmpty as NonEmptyList-import qualified Data.Text as T----- | assert that a text value is represented by the same pointer--- as the 'empty' value.-assertPtrEqEmpty :: HasCallStack => Text -> IO ()-assertPtrEqEmpty t = do - t' <- evaluate t- empty' <- evaluate empty- assertEqual "" empty' t'- case reallyUnsafePtrEquality# empty' t' of- 1# -> pure ()- _ -> assertFailure "Pointers are not equal"-{-# NOINLINE assertPtrEqEmpty #-}--tests :: TestTree-tests = testGroup "empty Text values are shared"- [ testCase "empty = empty" $ assertPtrEqEmpty T.empty- , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack ""- , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString ""- , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack ""))- , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a"- , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b"- , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty- , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " []- , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' ""- , testCase "reverse empty = empty" $ assertPtrEqEmpty $- T.reverse empty- , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $- T.replace "needle" "replacement" empty- , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold ""- , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower ""- , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper ""- , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle ""- , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $- justifyLeft 0 ' ' empty- , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $- justifyRight 0 ' ' empty- , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $- T.center 0 ' ' empty- , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $- T.transpose [empty]- , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat []- , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty]- , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x"- , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty- , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $- T.unfoldr (const Nothing) ()- , testCase "take 0 _ = empty" $ assertPtrEqEmpty $- T.take 0 "xyz"- , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $- T.takeEnd 0 "xyz"- , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $- T.takeWhile (const False) "xyz"- , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $- T.takeWhileEnd (const False) "xyz"- , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $- T.drop 5 "xyz"- , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $- T.dropEnd 5 "xyz"- , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $- T.dropWhile (const True) "xyz"- , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $- dropWhileEnd (const True) "xyz"- , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $- dropAround (const True) empty- , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty- , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty- , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty- , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123"- , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $- snd $ T.splitAt 5 "123"- , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $- fst $ T.span (const False) "123"- , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $- snd $ T.span (const True) "123"- , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $- fst $ T.span (const False) "123"- , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $- snd $ T.span (const True) "123"- , testCase "fst (spanM (const $ pure False) _) = empty" $- assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123"- , testCase "snd (spanM (const $ pure True) _) = empty" $- assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123"- , testCase "fst (spanEndM (const $ pure True) _) = empty" $- assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123"- , testCase "snd (spanEndM (const $ pure False) _) = empty" $- assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123"- , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty- , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty- , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty- , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123"- , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123"- , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty- , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty- , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123"- , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123"- , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') ""- , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234"- , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" ""- , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines []- , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords []- , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $- T.stripPrefix empty empty- , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $- T.stripSuffix empty empty- , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $- mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123"- , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $- mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz"- , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy ""- ]+{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE BangPatterns #-} + +{-# OPTIONS_GHC -Wno-unrecognised-warning-flags #-} +{-# OPTIONS_GHC -Wno-x-partial #-} + +module Tests.ShareEmpty + ( tests + ) where + +import Control.Exception (evaluate) +import Data.Text +#if __GLASGOW_HASKELL__ >= 914 +import Language.Haskell.TH.Lift (lift) +#else +import Language.Haskell.TH.Syntax (lift) +#endif +import Test.Tasty.HUnit (testCase, assertFailure, assertEqual) +import Test.Tasty (TestTree, testGroup) +import GHC.Exts +import GHC.Stack +import qualified Data.List as L +import qualified Data.List.NonEmpty as NonEmptyList +import qualified Data.Text as T + + +-- | assert that a text value is represented by the same pointer +-- as the 'empty' value. +assertPtrEqEmpty :: HasCallStack => Text -> IO () +assertPtrEqEmpty t = do + t' <- evaluate t + empty' <- evaluate empty + assertEqual "" empty' t' + case reallyUnsafePtrEquality# empty' t' of + 1# -> pure () + _ -> assertFailure "Pointers are not equal" +{-# NOINLINE assertPtrEqEmpty #-} + +tests :: TestTree +tests = testGroup "empty Text values are shared" + [ testCase "empty = empty" $ assertPtrEqEmpty T.empty + , testCase "pack \"\" = empty" $ assertPtrEqEmpty $ T.pack "" + , testCase "fromString \"\" = empty" $ assertPtrEqEmpty $ fromString "" + , testCase "$(lift \"\") = empty" $ assertPtrEqEmpty $ $(lift (pack "")) + , testCase "tail of a singleton = empty" $ assertPtrEqEmpty $ T.tail "a" + , testCase "init of a singleton = empty" $ assertPtrEqEmpty $ T.init "b" + , testCase "map _ empty = empty" $ assertPtrEqEmpty $ T.map id empty + , testCase "intercalate _ [] = empty" $ assertPtrEqEmpty $ T.intercalate ", " [] + , testCase "intersperse _ empty = empty" $ assertPtrEqEmpty $ T.intersperse ',' "" + , testCase "reverse empty = empty" $ assertPtrEqEmpty $ + T.reverse empty + , testCase "replace _ _ empty = empty" $ assertPtrEqEmpty $ + T.replace "needle" "replacement" empty + , testCase "toCaseFold empty = empty" $ assertPtrEqEmpty $ T.toCaseFold "" + , testCase "toLower empty = empty" $ assertPtrEqEmpty $ T.toLower "" + , testCase "toUpper empty = empty" $ assertPtrEqEmpty $ T.toUpper "" + , testCase "toTitle empty = empty" $ assertPtrEqEmpty $ T.toTitle "" + , testCase "justifyLeft 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyLeft 0 ' ' empty + , testCase "justifyRight 0 _ empty = empty" $ assertPtrEqEmpty $ + justifyRight 0 ' ' empty + , testCase "center 0 _ empty = empty" $ assertPtrEqEmpty $ + T.center 0 ' ' empty + , testCase "transpose [empty] = [empty]" $ mapM_ assertPtrEqEmpty $ + T.transpose [empty] + , testCase "concat [] = empty" $ assertPtrEqEmpty $ T.concat [] + , testCase "concat [empty] = empty" $ assertPtrEqEmpty $ T.concat [empty] + , testCase "replicate 0 _ = empty" $ assertPtrEqEmpty $ T.replicate 0 "x" + , testCase "replicate _ empty = empty" $ assertPtrEqEmpty $ T.replicate 10 empty + , testCase "unfoldr (const Nothing) _ = empty" $ assertPtrEqEmpty $ + T.unfoldr (const Nothing) () + , testCase "take 0 _ = empty" $ assertPtrEqEmpty $ + T.take 0 "xyz" + , testCase "takeEnd 0 _ = empty" $ assertPtrEqEmpty $ + T.takeEnd 0 "xyz" + , testCase "takeWhile (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhile (const False) "xyz" + , testCase "takeWhileEnd (const False) _ = empty" $ assertPtrEqEmpty $ + T.takeWhileEnd (const False) "xyz" + , testCase "drop n x = empty where n > len x" $ assertPtrEqEmpty $ + T.drop 5 "xyz" + , testCase "dropEnd n x = empty where n > len x" $ assertPtrEqEmpty $ + T.dropEnd 5 "xyz" + , testCase "dropWhile (const True) x = empty" $ assertPtrEqEmpty $ + T.dropWhile (const True) "xyz" + , testCase "dropWhileEnd (const True) x = empty" $ assertPtrEqEmpty $ + dropWhileEnd (const True) "xyz" + , testCase "dropAround _ empty = empty" $ assertPtrEqEmpty $ + dropAround (const True) empty + , testCase "stripStart empty = empty" $ assertPtrEqEmpty $ T.stripStart empty + , testCase "stripEnd empty = empty" $ assertPtrEqEmpty $ T.stripEnd empty + , testCase "strip empty = empty" $ assertPtrEqEmpty $ T.strip empty + , testCase "fst (splitAt 0 _) = empty" $ assertPtrEqEmpty $ fst $ T.splitAt 0 "123" + , testCase "snd (splitAt n x) = empty where n > len x" $ assertPtrEqEmpty $ + snd $ T.splitAt 5 "123" + , testCase "fst (span (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (span (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (break (const False) _) = empty" $ assertPtrEqEmpty $ + fst $ T.span (const False) "123" + , testCase "snd (break (const True) _) = empty" $ assertPtrEqEmpty $ + snd $ T.span (const True) "123" + , testCase "fst (spanM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanM (const $ pure False) "123" + , testCase "snd (spanM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanM (const $ pure True) "123" + , testCase "fst (spanEndM (const $ pure True) _) = empty" $ + assertPtrEqEmpty . fst =<< T.spanEndM (const $ pure True) "123" + , testCase "snd (spanEndM (const $ pure False) _) = empty" $ + assertPtrEqEmpty . snd =<< T.spanEndM (const $ pure False) "123" + , testCase "groupBy _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.groupBy (==) empty + , testCase "inits empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.inits empty + , testCase "initsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.initsNE empty + , testCase "inits _ = [empty, ...]" $ assertPtrEqEmpty $ L.head $ T.inits "123" + , testCase "initsNE _ = empty :| ..." $ assertPtrEqEmpty $ NonEmptyList.head $ T.initsNE "123" + , testCase "tails empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.tails empty + , testCase "tailsNE empty = singleton empty" $ mapM_ assertPtrEqEmpty $ T.tailsNE empty + , testCase "tails _ = [..., empty]" $ assertPtrEqEmpty $ L.last $ T.tails "123" + , testCase "tailsNE _ = reverse (empty :| ...)" $ assertPtrEqEmpty $ NonEmptyList.last $ T.tailsNE "123" + , testCase "split _ empty = [empty]" $ mapM_ assertPtrEqEmpty $ T.split (== 'a') "" + , testCase "filter (const False) _ = empty" $ assertPtrEqEmpty $ T.filter (const False) "1234" + , testCase "zipWith const empty empty = empty" $ assertPtrEqEmpty $ T.zipWith const "" "" + , testCase "unlines [] = empty" $ assertPtrEqEmpty $ T.unlines [] + , testCase "unwords [] = empty" $ assertPtrEqEmpty $ T.unwords [] + , testCase "stripPrefix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripPrefix empty empty + , testCase "stripSuffix empty empty = Just empty" $ mapM_ assertPtrEqEmpty $ + T.stripSuffix empty empty + , testCase "commonPrefixes \"xyz\" \"123\" = Just (_, empty, _)" $ + mapM_ (assertPtrEqEmpty . (\(_, x, _) -> x)) $ T.commonPrefixes "xyz" "123" + , testCase "commonPrefixes \"xyz\" \"xyz\" = Just (_, _, empty)" $ + mapM_ (assertPtrEqEmpty . (\(_, _, x) -> x)) $ T.commonPrefixes "xyz" "xyz" + , testCase "copy empty = empty" $ assertPtrEqEmpty $ T.copy "" + ]
tests/Tests/Utils.hs view
@@ -1,51 +1,50 @@--- | Miscellaneous testing utilities----{-# LANGUAGE ScopedTypeVariables #-}-module Tests.Utils- (- (=^=)- , withRedirect- , withTempFile- ) where--import Control.Exception (SomeException, bracket, bracket_, evaluate, try)-import Control.Monad (when, unless)-import GHC.IO.Handle.Internals (withHandle)-import System.Directory (removeFile)-import System.IO (Handle, hClose, hFlush, hIsOpen, hIsClosed, hIsWritable, openTempFile)-import Test.QuickCheck (Property, ioProperty, property, (===), counterexample)---- Ensure that two potentially bottom values (in the sense of crashing--- for some inputs, not looping infinitely) either both crash, or both--- give comparable results for some input.-(=^=) :: (Eq a, Show a) => a -> a -> Property-i =^= j = ioProperty $ do- x <- try (evaluate i)- y <- try (evaluate j)- return $ case (x, y) of- (Left (_ :: SomeException), Left (_ :: SomeException))- -> property True- (Right a, Right b) -> a === b- e -> counterexample ("Divergence: " ++ show e) $ property False-infix 4 =^=-{-# NOINLINE (=^=) #-}--withTempFile :: (FilePath -> Handle -> IO a) -> IO a-withTempFile = bracket (openTempFile "." "crashy.txt") cleanupTemp . uncurry- where- cleanupTemp (path,h) = do- closed <- hIsClosed h- unless closed $ hClose h- removeFile path--withRedirect :: Handle -> Handle -> IO a -> IO a-withRedirect tmp h = bracket_ swap swap- where- whenM p a = p >>= (`when` a)- swap = do- whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp- whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h- withHandle "spam" tmp $ \tmph -> do- hh <- withHandle "spam" h $ \hh ->- return (tmph,hh)- return (hh,())+-- | Miscellaneous testing utilities +-- +{-# LANGUAGE ScopedTypeVariables #-} +module Tests.Utils + ( + (=^=) + , withRedirect + , withTempFile + , emptyTempFile + ) where + +import Control.Exception (SomeException, bracket_, evaluate, try) +import Control.Monad (when) +import System.IO.Temp (withSystemTempFile, emptySystemTempFile) +import GHC.IO.Handle.Internals (withHandle) +import System.IO (Handle, hFlush, hIsOpen, hIsWritable) +import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) + +-- Ensure that two potentially bottom values (in the sense of crashing +-- for some inputs, not looping infinitely) either both crash, or both +-- give comparable results for some input. +(=^=) :: (Eq a, Show a) => a -> a -> Property +i =^= j = ioProperty $ do + x <- try (evaluate i) + y <- try (evaluate j) + return $ case (x, y) of + (Left (_ :: SomeException), Left (_ :: SomeException)) + -> property True + (Right a, Right b) -> a === b + e -> counterexample ("Divergence: " ++ show e) $ property False +infix 4 =^= +{-# NOINLINE (=^=) #-} + +withTempFile :: (FilePath -> Handle -> IO a) -> IO a +withTempFile = withSystemTempFile "crashy.txt" + +emptyTempFile :: IO FilePath +emptyTempFile = emptySystemTempFile "crashy.txt" + +withRedirect :: Handle -> Handle -> IO a -> IO a +withRedirect tmp h = bracket_ swap swap + where + whenM p a = p >>= (`when` a) + swap = do + whenM (hIsOpen tmp) $ whenM (hIsWritable tmp) $ hFlush tmp + whenM (hIsOpen h) $ whenM (hIsWritable h) $ hFlush h + withHandle "spam" tmp $ \tmph -> do + hh <- withHandle "spam" h $ \hh -> + return (tmph,hh) + return (hh,())
text.cabal view
@@ -1,365 +1,385 @@-cabal-version: 2.2-name: text-version: 2.1.2--homepage: https://github.com/haskell/text-bug-reports: https://github.com/haskell/text/issues-synopsis: An efficient packed Unicode text type.-description:- .- An efficient packed, immutable Unicode text type (both strict and- lazy).- .- The 'Text' type represents Unicode character strings, in a time and- space-efficient manner. This package provides text processing- capabilities that are optimized for performance critical use, both- in terms of large data quantities and high speed.- .- The 'Text' type provides character-encoding, type-safe case- conversion via whole-string case conversion functions (see "Data.Text").- It also provides a range of functions for converting 'Text' values to- and from 'ByteStrings', using several standard encodings- (see "Data.Text.Encoding").- .- Efficient locale-sensitive support for text IO is also supported- (see "Data.Text.IO").- .- These modules are intended to be imported qualified, to avoid name- clashes with Prelude functions, e.g.- .- > import qualified Data.Text as T- .- == ICU Support- .- To use an extended and very rich family of functions for working- with Unicode text (including normalization, regular expressions,- non-standard encodings, text breaking, and locales), see- the [text-icu package](https://hackage.haskell.org/package/text-icu)- based on the well-respected and liberally- licensed [ICU library](http://site.icu-project.org/).--license: BSD-2-Clause-license-file: LICENSE-author: Bryan O'Sullivan <bos@serpentine.com>-maintainer: Haskell Text Team <andrew.lelechenko@gmail.com>, Core Libraries Committee-copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko-category: Data, Text-build-type: Simple-tested-with:- GHC == 8.4.4- GHC == 8.6.5- GHC == 8.8.4- GHC == 8.10.7- GHC == 9.0.2- GHC == 9.2.8- GHC == 9.4.8- GHC == 9.6.4- GHC == 9.8.2- GHC == 9.10.1--extra-source-files:- -- scripts/CaseFolding.txt- -- scripts/SpecialCasing.txt- README.md- changelog.md- scripts/*.hs- simdutf/LICENSE-APACHE- simdutf/LICENSE-MIT- simdutf/simdutf.h- tests/literal-rule-test.sh- tests/LiteralRuleTest.hs--flag developer- description: operate in developer mode- default: False- manual: True--flag simdutf- description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed- default: True- manual: True--flag pure-haskell- description: Don't use text's standard C routines- NB: This feature is not fully implemented. Several C routines are still in- use.-- When this flag is true, text will use pure Haskell variants of the- routines. This is not recommended except for use with GHC's JavaScript- backend.-- This flag also disables simdutf.-- default: False- manual: True--flag ExtendedBenchmarks- description: Runs extra benchmarks which can be very slow.- default: False- manual: True--library- if arch(javascript) || flag(pure-haskell)- cpp-options: -DPURE_HASKELL- else- c-sources: cbits/is_ascii.c- cbits/reverse.c- cbits/utils.c- if (arch(aarch64))- c-sources: cbits/aarch64/measure_off.c- else- c-sources: cbits/measure_off.c-- hs-source-dirs: src-- if flag(simdutf) && !(arch(javascript) || flag(pure-haskell))- exposed-modules: Data.Text.Internal.Validate.Simd- include-dirs: simdutf- cxx-sources: simdutf/simdutf.cpp- cbits/validate_utf8.cpp- cxx-options: -std=c++17- cpp-options: -DSIMDUTF- if impl(ghc >= 9.4)- build-depends: system-cxx-std-lib == 1.0- elif os(darwin) || os(freebsd)- extra-libraries: c++- elif os(openbsd)- extra-libraries: c++ c++abi pthread- elif os(windows)- -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later- if impl(ghc < 9.3)- extra-libraries: stdc++- else- extra-libraries: c++ c++abi- elif arch(wasm32)- cpp-options: -DSIMDUTF_NO_THREADS- cxx-options: -fno-exceptions- extra-libraries: c++ c++abi- else- extra-libraries: stdc++-- -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++.- -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417- if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1)- build-depends: base < 0-- -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker.- if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1)- build-depends: base < 0-- -- GHC 8.10 has linking issues (probably TH-related) on ARM.- if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*)- build-depends: base < 0-- -- Subword primitives in GHC 9.2.1 are broken on ARM platforms.- if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1)- build-depends: base < 0-- -- NetBSD + GHC 9.2.1 + TH + C++ does not work together.- -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577- if flag(simdutf) && os(netbsd) && impl(ghc < 9.4)- build-depends: base < 0-- exposed-modules:- Data.Text- Data.Text.Array- Data.Text.Encoding- Data.Text.Encoding.Error- Data.Text.Foreign- Data.Text.IO- Data.Text.IO.Utf8- Data.Text.Internal- Data.Text.Internal.ArrayUtils- Data.Text.Internal.Builder- Data.Text.Internal.Builder.Functions- Data.Text.Internal.Builder.Int.Digits- Data.Text.Internal.Builder.RealFloat.Functions- Data.Text.Internal.ByteStringCompat- Data.Text.Internal.PrimCompat- Data.Text.Internal.Encoding- Data.Text.Internal.Encoding.Fusion- Data.Text.Internal.Encoding.Fusion.Common- Data.Text.Internal.Encoding.Utf16- Data.Text.Internal.Encoding.Utf32- Data.Text.Internal.Encoding.Utf8- Data.Text.Internal.Fusion- Data.Text.Internal.Fusion.CaseMapping- Data.Text.Internal.Fusion.Common- Data.Text.Internal.Fusion.Size- Data.Text.Internal.Fusion.Types- Data.Text.Internal.IO- Data.Text.Internal.Lazy- Data.Text.Internal.Lazy.Encoding.Fusion- Data.Text.Internal.Lazy.Fusion- Data.Text.Internal.Lazy.Search- Data.Text.Internal.Private- Data.Text.Internal.Read- Data.Text.Internal.Search- Data.Text.Internal.StrictBuilder- Data.Text.Internal.Unsafe- Data.Text.Internal.Unsafe.Char- Data.Text.Internal.Validate- Data.Text.Internal.Validate.Native- Data.Text.Lazy- Data.Text.Lazy.Builder- Data.Text.Lazy.Builder.Int- Data.Text.Lazy.Builder.RealFloat- Data.Text.Lazy.Encoding- Data.Text.Lazy.IO- Data.Text.Lazy.Internal- Data.Text.Lazy.Read- Data.Text.Read- Data.Text.Unsafe-- other-modules:- Data.Text.Show- Data.Text.Internal.Measure- Data.Text.Internal.Reverse- Data.Text.Internal.Transformation- Data.Text.Internal.IsAscii-- build-depends:- array >= 0.3 && < 0.6,- base >= 4.11 && < 5,- binary >= 0.5 && < 0.9,- bytestring >= 0.10.4 && < 0.13,- deepseq >= 1.1 && < 1.6,- ghc-prim >= 0.2 && < 0.12,- template-haskell >= 2.5 && < 2.23-- if impl(ghc < 9.4)- build-depends: data-array-byte >= 0.1 && < 0.2-- ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2- if flag(developer)- ghc-options: -fno-ignore-asserts- cpp-options: -DASSERTS- if impl(ghc >= 9.2.2)- ghc-options: -fcheck-prim-bounds-- default-language: Haskell2010- default-extensions:- NondecreasingIndentation- other-extensions:- BangPatterns- CPP- DeriveDataTypeable- ExistentialQuantification- ForeignFunctionInterface- GeneralizedNewtypeDeriving- MagicHash- OverloadedStrings- Rank2Types- RankNTypes- RecordWildCards- Safe- ScopedTypeVariables- TemplateHaskellQuotes- Trustworthy- TypeFamilies- UnboxedTuples- UnliftedFFITypes--source-repository head- type: git- location: https://github.com/haskell/text--test-suite tests- type: exitcode-stdio-1.0- ghc-options:- -Wall -threaded -rtsopts -with-rtsopts=-N-- hs-source-dirs: tests- main-is: Tests.hs- other-modules:- Tests.Lift- Tests.Properties- Tests.Properties.Basics- Tests.Properties.Builder- Tests.Properties.Folds- Tests.Properties.Instances- Tests.Properties.LowLevel- Tests.Properties.Read- Tests.Properties.Substrings- Tests.Properties.Text- Tests.Properties.Transcoding- Tests.Properties.Validate- Tests.QuickCheckUtils- Tests.RebindableSyntaxTest- Tests.Regressions- Tests.SlowFunctions- Tests.ShareEmpty- Tests.Utils-- build-depends:- QuickCheck >= 2.12.6 && < 2.16,- base <5,- binary,- bytestring,- deepseq,- directory,- ghc-prim,- tasty,- tasty-hunit,- tasty-quickcheck,- template-haskell,- transformers,- text- if impl(ghc < 9.4)- build-depends: data-array-byte >= 0.1 && < 0.2- -- Plugin infrastructure does not work properly in 8.6.1, and- -- ghc-9.2.1 library depends on parsec, which causes a circular dependency.- if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2)- build-depends: tasty-inspection-testing-- default-language: Haskell2010- default-extensions: NondecreasingIndentation--benchmark text-benchmarks- type: exitcode-stdio-1.0-- ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m"- if impl(ghc >= 8.6)- ghc-options: -fproc-alignment=64- if flag(ExtendedBenchmarks)- cpp-options: -DExtendedBenchmarks-- build-depends: base,- bytestring >= 0.10.4,- containers,- deepseq,- directory,- filepath,- tasty-bench >= 0.2,- text,- transformers-- hs-source-dirs: benchmarks/haskell- main-is: Benchmarks.hs- other-modules:- Benchmarks.Builder- Benchmarks.Concat- Benchmarks.DecodeUtf8- Benchmarks.EncodeUtf8- Benchmarks.Equality- Benchmarks.FileRead- Benchmarks.FileWrite- Benchmarks.FoldLines- Benchmarks.Micro- Benchmarks.Multilang- Benchmarks.Programs.BigTable- Benchmarks.Programs.Cut- Benchmarks.Programs.Fold- Benchmarks.Programs.Sort- Benchmarks.Programs.StripTags- Benchmarks.Programs.Throughput- Benchmarks.Pure- Benchmarks.ReadNumbers- Benchmarks.Replace- Benchmarks.Search- Benchmarks.Stream- Benchmarks.WordFrequencies-- default-language: Haskell2010- default-extensions: NondecreasingIndentation- other-extensions: DeriveGeneric+cabal-version: 2.2 +name: text +version: 2.1.4 + +homepage: https://github.com/haskell/text +bug-reports: https://github.com/haskell/text/issues +synopsis: An efficient packed Unicode text type. +description: + . + An efficient packed, immutable Unicode text type (both strict and + lazy). + . + The 'Text' type represents Unicode character strings, in a time and + space-efficient manner. This package provides text processing + capabilities that are optimized for performance critical use, both + in terms of large data quantities and high speed. + . + The 'Text' type provides character-encoding, type-safe case + conversion via whole-string case conversion functions (see "Data.Text"). + It also provides a range of functions for converting 'Text' values to + and from 'ByteStrings', using several standard encodings + (see "Data.Text.Encoding"). + . + Efficient locale-sensitive support for text IO is also supported + (see "Data.Text.IO"). + . + These modules are intended to be imported qualified, to avoid name + clashes with Prelude functions, e.g. + . + > import qualified Data.Text as T + . + == ICU Support + . + To use an extended and very rich family of functions for working + with Unicode text (including normalization, regular expressions, + non-standard encodings, text breaking, and locales), see + the [text-icu package](https://hackage.haskell.org/package/text-icu) + based on the well-respected and liberally + licensed [ICU library](http://site.icu-project.org/). + +license: BSD-2-Clause +license-file: LICENSE +author: Bryan O'Sullivan <bos@serpentine.com> +maintainer: Haskell Text Team <andrew.lelechenko@gmail.com>, Core Libraries Committee +copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko +category: Data, Text +build-type: Simple +tested-with: + GHC == 8.4.4 + GHC == 8.6.5 + GHC == 8.8.4 + GHC == 8.10.7 + GHC == 9.0.2 + GHC == 9.2.8 + GHC == 9.4.8 + GHC == 9.6.7 + GHC == 9.8.4 + GHC == 9.10.1 + GHC == 9.12.2 + +extra-source-files: + -- scripts/CaseFolding.txt + -- scripts/SpecialCasing.txt + scripts/*.hs + simdutf/LICENSE-APACHE + simdutf/LICENSE-MIT + simdutf/simdutf_c.h + simdutf/simdutf.h + tests/literal-rule-test.sh + tests/LiteralRuleTest.hs +extra-doc-files: + README.md + changelog.md + +flag developer + description: operate in developer mode + default: False + manual: True + +flag simdutf + description: use simdutf library, causes Data.Text.Internal.Validate.Simd to be exposed + default: True + manual: True + +flag pure-haskell + description: Don't use text's standard C routines + NB: This feature is not fully implemented. Several C routines are still in + use. + + When this flag is true, text will use pure Haskell variants of the + routines. This is not recommended except for use with GHC's JavaScript + backend. + + This flag also disables simdutf. + + default: False + manual: True + +flag ExtendedBenchmarks + description: Runs extra benchmarks which can be very slow. + default: False + manual: True + +library + if arch(javascript) || flag(pure-haskell) + cpp-options: -DPURE_HASKELL + else + c-sources: cbits/is_ascii.c + cbits/reverse.c + cbits/utils.c + if (arch(aarch64)) + c-sources: cbits/aarch64/measure_off.c + else + c-sources: cbits/measure_off.c + + hs-source-dirs: src + + if flag(simdutf) && !(arch(javascript) || flag(pure-haskell)) + exposed-modules: Data.Text.Internal.Validate.Simd + include-dirs: simdutf + c-sources: simdutf/hs_simdutf.c + cxx-sources: simdutf/simdutf.cpp + cxx-options: -std=c++17 + cpp-options: -DSIMDUTF + if impl(ghc >= 9.4) + build-depends: system-cxx-std-lib == 1.0 + elif os(darwin) || os(freebsd) + extra-libraries: c++ + elif os(openbsd) + extra-libraries: c++ c++abi pthread + elif os(windows) + -- GHC's Windows toolchain is based on clang/libc++ in GHC 9.4 and later + if impl(ghc < 9.3) + extra-libraries: stdc++ + else + extra-libraries: c++ c++abi + elif arch(wasm32) + cpp-options: -DSIMDUTF_NO_THREADS + cxx-options: -fno-exceptions + extra-libraries: c++ c++abi + else + extra-libraries: stdc++ + + -- Certain version of GHC crash on Windows, when TemplateHaskell encounters C++. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19417 + if flag(simdutf) && os(windows) && impl(ghc >= 8.8 && < 8.10.5 || == 9.0.1) + build-depends: base < 0 + + -- For GHC 8.2, 8.6.3 and 8.10.1 even TH + C crash Windows linker. + if os(windows) && impl(ghc >= 8.2 && < 8.4 || == 8.6.3 || == 8.10.1) + build-depends: base < 0 + + -- GHC 8.10 has linking issues (probably TH-related) on ARM. + if (arch(aarch64) || arch(arm)) && impl(ghc == 8.10.*) + build-depends: base < 0 + + -- Subword primitives in GHC 9.2.1 are broken on ARM platforms. + if (arch(aarch64) || arch(arm)) && impl(ghc == 9.2.1) + build-depends: base < 0 + + -- NetBSD + GHC 9.2.1 + TH + C++ does not work together. + -- https://gitlab.haskell.org/ghc/ghc/-/issues/22577 + if flag(simdutf) && os(netbsd) && impl(ghc < 9.4) + build-depends: base < 0 + + exposed-modules: + Data.Text + Data.Text.Array + Data.Text.Encoding + Data.Text.Encoding.Error + Data.Text.Foreign + Data.Text.IO + Data.Text.IO.Utf8 + Data.Text.Internal + Data.Text.Internal.ArrayUtils + Data.Text.Internal.Builder + Data.Text.Internal.Builder.Functions + Data.Text.Internal.Builder.Int.Digits + Data.Text.Internal.Builder.RealFloat.Functions + Data.Text.Internal.ByteStringCompat + Data.Text.Internal.PrimCompat + Data.Text.Internal.Encoding + Data.Text.Internal.Encoding.Fusion + Data.Text.Internal.Encoding.Fusion.Common + Data.Text.Internal.Encoding.Utf16 + Data.Text.Internal.Encoding.Utf32 + Data.Text.Internal.Encoding.Utf8 + Data.Text.Internal.Fusion + Data.Text.Internal.Fusion.CaseMapping + Data.Text.Internal.Fusion.Common + Data.Text.Internal.Fusion.Size + Data.Text.Internal.Fusion.Types + Data.Text.Internal.IO + Data.Text.Internal.Lazy + Data.Text.Internal.Lazy.Encoding.Fusion + Data.Text.Internal.Lazy.Fusion + Data.Text.Internal.Lazy.Search + Data.Text.Internal.Private + Data.Text.Internal.Read + Data.Text.Internal.Search + Data.Text.Internal.StrictBuilder + Data.Text.Internal.Unsafe + Data.Text.Internal.Unsafe.Char + Data.Text.Internal.Validate + Data.Text.Internal.Validate.Native + Data.Text.Lazy + Data.Text.Lazy.Builder + Data.Text.Lazy.Builder.Int + Data.Text.Lazy.Builder.RealFloat + Data.Text.Lazy.Encoding + Data.Text.Lazy.IO + Data.Text.Lazy.Internal + Data.Text.Lazy.Read + Data.Text.Read + Data.Text.Unsafe + + other-modules: + Data.Text.Show + Data.Text.Internal.Measure + Data.Text.Internal.Reverse + Data.Text.Internal.Transformation + Data.Text.Internal.IsAscii + + build-depends: + array >= 0.3 && < 0.6, + base >= 4.11 && < 5, + binary >= 0.8.3 && < 0.9, + bytestring >= 0.10.4 && < 0.13, + deepseq >= 1.1 && < 1.6, + ghc-prim >= 0.2 && < 0.15, + + -- template-haskell-lift was added as a boot library in GHC-9.14 + -- once we no longer wish to backport releases to older major releases of GHC, + -- this conditional can be dropped + if impl(ghc < 9.14) + build-depends: template-haskell >= 2.5 && < 3 + else + build-depends: template-haskell-lift >= 0.1 && <0.2 + + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + + ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2 + if flag(developer) + ghc-options: -fno-ignore-asserts + cpp-options: -DASSERTS + if impl(ghc >= 9.2.2) + ghc-options: -fcheck-prim-bounds + + default-language: Haskell2010 + default-extensions: + NondecreasingIndentation + other-extensions: + BangPatterns + CPP + DeriveDataTypeable + ExistentialQuantification + ForeignFunctionInterface + GeneralizedNewtypeDeriving + MagicHash + OverloadedStrings + Rank2Types + RankNTypes + RecordWildCards + Safe + ScopedTypeVariables + TemplateHaskellQuotes + Trustworthy + TypeFamilies + UnboxedTuples + UnliftedFFITypes + +source-repository head + type: git + location: https://github.com/haskell/text + +test-suite tests + type: exitcode-stdio-1.0 + ghc-options: + -Wall -threaded -rtsopts -with-rtsopts=-N + + hs-source-dirs: tests + main-is: Tests.hs + other-modules: + Tests.Lift + Tests.Properties + Tests.Properties.Basics + Tests.Properties.Builder + Tests.Properties.Folds + Tests.Properties.Instances + Tests.Properties.LowLevel + Tests.Properties.Read + Tests.Properties.Substrings + Tests.Properties.Text + Tests.Properties.Transcoding + Tests.Properties.CornerCases + Tests.Properties.Validate + Tests.QuickCheckUtils + Tests.RebindableSyntaxTest + Tests.Regressions + Tests.SlowFunctions + Tests.ShareEmpty + Tests.Utils + + build-depends: + QuickCheck >= 2.12.6 && < 2.18, + base <5, + binary, + bytestring, + deepseq, + ghc-prim, + tasty, + tasty-hunit, + tasty-quickcheck, + temporary, + transformers, + text + if impl(ghc < 9.4) + build-depends: data-array-byte >= 0.1 && < 0.2 + if impl(ghc < 9.14) + build-depends: template-haskell + else + build-depends: template-haskell-lift + + -- Plugin infrastructure does not work properly in 8.6.1, and + -- ghc-9.2.1 library depends on parsec, which causes a circular dependency. + if impl(ghc >= 8.2.1 && < 8.6 || >= 8.6.2 && < 9.2 || >= 9.2.2) + build-depends: tasty-inspection-testing + + -- https://github.com/haskellari/splitmix/issues/101 + if os(openbsd) + build-depends: splitmix < 0.1.3 || > 0.1.3.1 + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + +benchmark text-benchmarks + type: exitcode-stdio-1.0 + + ghc-options: -Wall -O2 -rtsopts "-with-rtsopts=-A32m" + if impl(ghc >= 8.6) + ghc-options: -fproc-alignment=64 + if flag(ExtendedBenchmarks) + cpp-options: -DExtendedBenchmarks + + build-depends: base, + bytestring >= 0.10.4, + containers, + deepseq, + directory, + filepath, + tasty-bench >= 0.2, + temporary, + text, + transformers + + hs-source-dirs: benchmarks/haskell + main-is: Benchmarks.hs + other-modules: + Benchmarks.Builder + Benchmarks.Concat + Benchmarks.DecodeUtf8 + Benchmarks.EncodeUtf8 + Benchmarks.Equality + Benchmarks.FileRead + Benchmarks.FileWrite + Benchmarks.FoldLines + Benchmarks.Micro + Benchmarks.Multilang + Benchmarks.Programs.BigTable + Benchmarks.Programs.Cut + Benchmarks.Programs.Fold + Benchmarks.Programs.Sort + Benchmarks.Programs.StripTags + Benchmarks.Programs.Throughput + Benchmarks.Pure + Benchmarks.ReadNumbers + Benchmarks.Replace + Benchmarks.Search + Benchmarks.Stream + Benchmarks.WordFrequencies + + default-language: Haskell2010 + default-extensions: NondecreasingIndentation + other-extensions: DeriveGeneric