diff --git a/README.markdown b/README.markdown --- a/README.markdown +++ b/README.markdown @@ -1,7 +1,6 @@ -# `text`: Fast, packed Unicode strings, using stream fusion [![Hackage](http://img.shields.io/hackage/v/text.svg)](https://hackage.haskell.org/package/text) +# text [![Hackage](http://img.shields.io/hackage/v/text.svg)](https://hackage.haskell.org/package/text) [![Stackage LTS](http://stackage.org/package/text/badge/lts)](http://stackage.org/lts/package/text) [![Stackage Nightly](http://stackage.org/package/text/badge/nightly)](http://stackage.org/nightly/package/text) -This package provides the Data.Text library, a library for the space- -and time-efficient manipulation of Unicode text in Haskell. +Haskell library for space- and time-efficient operations over Unicode text. # Get involved! @@ -11,15 +10,14 @@ The main repo: ```bash -git clone git://github.com/haskell/text.git +git clone https://github.com/haskell/text ``` To run benchmarks please clone and unpack test files: ```bash -git clone https://github.com/bos/text-test-data benchmarks/text-test-data -cd benchmarks/text-test-data -make +git clone https://github.com/haskell/text-test-data benchmarks/text-test-data +make -Cbenchmarks/text-test-data ``` # Authors @@ -29,4 +27,4 @@ Duncan Coutts, and Don Stewart. The core library was fleshed out, debugged, and tested by Bryan -O'Sullivan , and he is the current maintainer. +O'Sullivan. Transition from UTF-16 to UTF-8 is by Andrew Lelechenko. diff --git a/benchmarks/cbits-bench/time_iconv.c b/benchmarks/cbits-bench/time_iconv.c deleted file mode 100644 --- a/benchmarks/cbits-bench/time_iconv.c +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include -#include -#include - -int time_iconv(char *srcbuf, size_t srcbufsize) -{ - uint16_t *destbuf = NULL; - size_t destbufsize; - static uint16_t *origdestbuf; - static size_t origdestbufsize; - iconv_t ic = (iconv_t) -1; - int ret = 0; - - if (ic == (iconv_t) -1) { - ic = iconv_open("UTF-16LE", "UTF-8"); - if (ic == (iconv_t) -1) { - ret = -1; - goto done; - } - } - - destbufsize = srcbufsize * sizeof(uint16_t); - if (destbufsize > origdestbufsize) { - free(origdestbuf); - origdestbuf = destbuf = malloc(origdestbufsize = destbufsize); - } else { - destbuf = origdestbuf; - } - - iconv(ic, &srcbuf, &srcbufsize, (char**) &destbuf, &destbufsize); - - done: - return ret; -} diff --git a/benchmarks/haskell/Benchmarks.hs b/benchmarks/haskell/Benchmarks.hs --- a/benchmarks/haskell/Benchmarks.hs +++ b/benchmarks/haskell/Benchmarks.hs @@ -1,14 +1,18 @@ --- | Main module to run the micro benchmarks --- +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} + module Main ( main ) where import Test.Tasty.Bench (defaultMain, bgroup, env) import System.FilePath (()) -import System.IO (IOMode (WriteMode), openFile, hSetEncoding, utf8) +import System.IO +#ifdef mingw32_HOST_OS +import System.Directory (removeFile) +#endif + import qualified Benchmarks.Builder as Builder import qualified Benchmarks.Concat as Concat import qualified Benchmarks.DecodeUtf8 as DecodeUtf8 @@ -16,7 +20,6 @@ import qualified Benchmarks.Equality as Equality import qualified Benchmarks.FileRead as FileRead import qualified Benchmarks.FoldLines as FoldLines -import qualified Benchmarks.Mul as Mul import qualified Benchmarks.Multilang as Multilang import qualified Benchmarks.Pure as Pure import qualified Benchmarks.ReadNumbers as ReadNumbers @@ -32,32 +35,56 @@ import qualified Benchmarks.Programs.StripTags as Programs.StripTags import qualified Benchmarks.Programs.Throughput as Programs.Throughput +mkSink :: IO (FilePath, Handle) +mkSink = do +#ifdef mingw32_HOST_OS + (sinkFn, sink) <- openTempFile "." "dev.null" +#else + let sinkFn = "/dev/null" + sink <- openFile sinkFn WriteMode +#endif + hSetEncoding sink utf8 + pure (sinkFn, sink) + +rmSink :: FilePath -> IO () +#ifdef mingw32_HOST_OS +rmSink = removeFile +#else +rmSink _ = pure () +#endif + main :: IO () main = do - sink <- openFile "/dev/null" WriteMode - hSetEncoding sink utf8 + let tf = ("benchmarks/text-test-data" ) + -- Cannot use envWithCleanup, because there is no instance NFData Handle + (sinkFn, sink) <- mkSink defaultMain [ Builder.benchmark , Concat.benchmark - , env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html") - , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml") - , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmark "ascii") - , env (DecodeUtf8.initEnv (tf "russian.txt")) (DecodeUtf8.benchmark "russian") - , env (DecodeUtf8.initEnv (tf "japanese.txt")) (DecodeUtf8.benchmark "japanese") - , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmarkASCII) - , EncodeUtf8.benchmark "non-ASCII" "επανάληψη 竺法蘭共譯" - , EncodeUtf8.benchmark "ASCII" "lorem ipsum" + , bgroup "DecodeUtf8" + [ env (DecodeUtf8.initEnv (tf "libya-chinese.html")) (DecodeUtf8.benchmark "html") + , env (DecodeUtf8.initEnv (tf "yiwiki.xml")) (DecodeUtf8.benchmark "xml") + , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmark "ascii") + , env (DecodeUtf8.initEnv (tf "russian.txt")) (DecodeUtf8.benchmark "russian") + , env (DecodeUtf8.initEnv (tf "japanese.txt")) (DecodeUtf8.benchmark "japanese") + , env (DecodeUtf8.initEnv (tf "ascii.txt")) (DecodeUtf8.benchmarkASCII) + ] + , bgroup "EncodeUtf8" + [ EncodeUtf8.benchmark "non-ASCII" "επανάληψη 竺法蘭共譯" + , EncodeUtf8.benchmark "ASCII" "lorem ipsum" + ] , env (Equality.initEnv (tf "japanese.txt")) Equality.benchmark , FileRead.benchmark (tf "russian.txt") , FoldLines.benchmark (tf "russian.txt") - , env Mul.initEnv Mul.benchmark , Multilang.benchmark - , env (Pure.initEnv (tf "tiny.txt")) (Pure.benchmark "tiny") - , env (Pure.initEnv (tf "ascii-small.txt")) (Pure.benchmark "ascii-small") - , env (Pure.initEnv (tf "ascii.txt")) (Pure.benchmark "ascii") - , env (Pure.initEnv (tf "english.txt")) (Pure.benchmark "english") - , env (Pure.initEnv (tf "russian-small.txt")) (Pure.benchmark "russian") - , env (Pure.initEnv (tf "japanese.txt")) (Pure.benchmark "japanese") + , bgroup "Pure" + [ env (Pure.initEnv (tf "tiny.txt")) (Pure.benchmark "tiny") + , env (Pure.initEnv (tf "ascii-small.txt")) (Pure.benchmark "ascii-small") + , env (Pure.initEnv (tf "ascii.txt")) (Pure.benchmark "ascii") + , env (Pure.initEnv (tf "english.txt")) (Pure.benchmark "english") + , env (Pure.initEnv (tf "russian-small.txt")) (Pure.benchmark "russian") + , env (Pure.initEnv (tf "japanese.txt")) (Pure.benchmark "japanese") + ] , env (ReadNumbers.initEnv (tf "numbers.txt")) ReadNumbers.benchmark , env (Replace.initEnv (tf "russian.txt")) (Replace.benchmark "принимая" "своем") , env (Search.initEnv (tf "russian.txt")) (Search.benchmark "принимая") @@ -72,6 +99,4 @@ , Programs.Throughput.benchmark (tf "russian.txt") sink ] ] - where - -- Location of a test file - tf = ("benchmarks/text-test-data" ) + rmSink sinkFn diff --git a/benchmarks/haskell/Benchmarks/Builder.hs b/benchmarks/haskell/Benchmarks/Builder.hs --- a/benchmarks/haskell/Benchmarks/Builder.hs +++ b/benchmarks/haskell/Benchmarks/Builder.hs @@ -10,11 +10,7 @@ ) where import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) -import Data.Binary.Builder as B import Data.ByteString.Char8 () -import qualified Data.ByteString.Builder as Blaze -import qualified Data.ByteString as SB -import qualified Data.ByteString.Lazy as LB import qualified Data.Text as T import qualified Data.Text.Lazy as LT import qualified Data.Text.Lazy.Builder as LTB @@ -26,12 +22,6 @@ [ bgroup "Comparison" [ bench "LazyText" $ nf (LT.length . LTB.toLazyText . mconcat . map LTB.fromText) texts - , bench "Binary" $ nf - (LB.length . B.toLazyByteString . mconcat . map B.fromByteString) - byteStrings - , bench "Blaze" $ nf - (LB.length . Blaze.toLazyByteString . mconcat . map Blaze.stringUtf8) - strings ] , bgroup "Int" [ bgroup "Decimal" @@ -61,13 +51,3 @@ texts :: [T.Text] texts = take 200000 $ cycle ["foo", "λx", "由の"] {-# NOINLINE texts #-} - --- Note that the non-ascii characters will be chopped -byteStrings :: [SB.ByteString] -byteStrings = take 200000 $ cycle ["foo", "λx", "由の"] -{-# NOINLINE byteStrings #-} - --- Note that the non-ascii characters will be chopped -strings :: [String] -strings = take 200000 $ cycle ["foo", "λx", "由の"] -{-# NOINLINE strings #-} diff --git a/benchmarks/haskell/Benchmarks/DecodeUtf8.hs b/benchmarks/haskell/Benchmarks/DecodeUtf8.hs --- a/benchmarks/haskell/Benchmarks/DecodeUtf8.hs +++ b/benchmarks/haskell/Benchmarks/DecodeUtf8.hs @@ -1,5 +1,4 @@ {-# LANGUAGE ForeignFunctionInterface #-} -{-# OPTIONS_GHC -fno-warn-deprecations #-} -- | Test decoding of UTF-8 -- @@ -21,14 +20,8 @@ , benchmarkASCII ) where -import Foreign.C.Types -import Data.ByteString.Internal (ByteString(..)) import Data.ByteString.Lazy.Internal (ByteString(..)) -import Foreign.Ptr (Ptr, plusPtr) -import Foreign.ForeignPtr (withForeignPtr) -import Data.Word (Word8) -import qualified Test.Tasty.Bench as C -import Test.Tasty.Bench (Benchmark, bgroup, nf, whnfIO) +import Test.Tasty.Bench import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T @@ -46,17 +39,15 @@ benchmark :: String -> Env -> Benchmark benchmark kind ~(bs, lbs) = - let bench name = C.bench (name ++ "+" ++ kind) - decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of + let decodeStream (Chunk b0 bs0) = case T.streamDecodeUtf8 b0 of T.Some t0 _ f0 -> t0 : go f0 bs0 where go f (Chunk b bs1) = case f b of T.Some t1 _ f1 -> t1 : go f1 bs1 go _ _ = [] decodeStream _ = [] - in bgroup "DecodeUtf8" + in bgroup kind [ bench "Strict" $ nf T.decodeUtf8 bs , bench "Stream" $ nf decodeStream lbs - , bench "IConv" $ whnfIO $ iconv bs , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs , bench "Lazy" $ nf TL.decodeUtf8 lbs @@ -66,17 +57,11 @@ benchmarkASCII :: Env -> Benchmark benchmarkASCII ~(bs, lbs) = - bgroup "DecodeASCII" - [ C.bench "strict decodeUtf8" $ nf T.decodeUtf8 bs - , C.bench "strict decodeLatin1" $ nf T.decodeLatin1 bs - , C.bench "strict decodeASCII" $ nf T.decodeASCII bs - , C.bench "lazy decodeUtf8" $ nf TL.decodeUtf8 lbs - , C.bench "lazy decodeLatin1" $ nf TL.decodeLatin1 lbs - , C.bench "lazy decodeASCII" $ nf TL.decodeASCII lbs + bgroup "ascii" + [ bench "strict decodeUtf8" $ nf T.decodeUtf8 bs + , bench "strict decodeLatin1" $ nf T.decodeLatin1 bs + , bench "strict decodeASCII" $ nf T.decodeASCII bs + , bench "lazy decodeUtf8" $ nf TL.decodeUtf8 lbs + , bench "lazy decodeLatin1" $ nf TL.decodeLatin1 lbs + , bench "lazy decodeASCII" $ nf TL.decodeASCII lbs ] - -iconv :: B.ByteString -> IO CInt -iconv (PS fp off len) = withForeignPtr fp $ \ptr -> - time_iconv (ptr `plusPtr` off) (fromIntegral len) - -foreign import ccall unsafe time_iconv :: Ptr Word8 -> CSize -> IO CInt diff --git a/benchmarks/haskell/Benchmarks/EncodeUtf8.hs b/benchmarks/haskell/Benchmarks/EncodeUtf8.hs --- a/benchmarks/haskell/Benchmarks/EncodeUtf8.hs +++ b/benchmarks/haskell/Benchmarks/EncodeUtf8.hs @@ -10,8 +10,10 @@ ( benchmark ) where -import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) +import Test.Tasty.Bench (Benchmark, bgroup, bench, nf, whnf) import qualified Data.ByteString as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Prim as BP import qualified Data.ByteString.Lazy as BL import qualified Data.Text as T import qualified Data.Text.Encoding as T @@ -20,9 +22,11 @@ benchmark :: String -> String -> Benchmark benchmark name string = - bgroup "EncodeUtf8" - [ bench ("Text (" ++ name ++ ")") $ whnf (B.length . T.encodeUtf8) text - , bench ("LazyText (" ++ name ++ ")") $ whnf (BL.length . TL.encodeUtf8) lazyText + bgroup name + [ bench "Text" $ whnf (B.length . T.encodeUtf8) text + , bench "LazyText" $ whnf (BL.length . TL.encodeUtf8) lazyText + , bench "Text/encodeUtf8Builder" $ nf (B.toLazyByteString . T.encodeUtf8Builder) text + , bench "Text/encodeUtf8BuilderEscaped" $ nf (B.toLazyByteString . T.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) text ] where -- The string in different formats diff --git a/benchmarks/haskell/Benchmarks/Mul.hs b/benchmarks/haskell/Benchmarks/Mul.hs deleted file mode 100644 --- a/benchmarks/haskell/Benchmarks/Mul.hs +++ /dev/null @@ -1,146 +0,0 @@ -module Benchmarks.Mul - ( initEnv - , benchmark - ) where - -import Control.Exception (evaluate) -import Test.Tasty.Bench -import Data.Int (Int32, Int64) -import Data.Text.Internal (mul32, mul64) -import qualified Data.Vector.Unboxed as U - -oldMul :: Int64 -> Int64 -> Int64 -oldMul m n - | n == 0 = 0 - | m <= maxBound `quot` n = m * n - | otherwise = error "overflow" - -type Env = (U.Vector (Int32,Int32), U.Vector (Int64,Int64)) - -initEnv :: IO Env -initEnv = do - x <- evaluate testVector32 - y <- evaluate testVector64 - return (x, y) - -benchmark :: Env -> Benchmark -benchmark ~(tv32, tv64) = bgroup "Mul" - [ bench "oldMul" $ whnf (U.map (uncurry oldMul)) tv64 - , bench "mul64" $ whnf (U.map (uncurry mul64)) tv64 - , bench "*64" $ whnf (U.map (uncurry (*))) tv64 - , bench "mul32" $ whnf (U.map (uncurry mul32)) tv32 - , bench "*32" $ whnf (U.map (uncurry (*))) tv32 - ] - -testVector64 :: U.Vector (Int64,Int64) -testVector64 = U.fromList [ - (0,1248868987182846646),(169004623633872,24458),(482549039517835,7614), - (372,8157063115504364),(27,107095594861148252),(3,63249878517962420), - (4363,255694473572912),(86678474,1732634806),(1572453024,1800489338), - (9384523143,77053781),(49024709555,75095046),(7,43457620410239131), - (8,8201563008844571),(387719037,1520696708),(189869238220197,1423), - (46788016849611,23063),(503077742109974359,0),(104,1502010908706487), - (30478140346,207525518),(80961140129236192,14),(4283,368012829143675), - (1028719181728108146,6),(318904,5874863049591),(56724427166898,110794), - (234539368,31369110449),(2,251729663598178612),(103291548194451219,5), - (76013,5345328755566),(1769631,2980846129318),(40898,60598477385754), - (0,98931348893227155),(573555872156917492,3),(318821187115,4476566), - (11152874213584,243582),(40274276,16636653248),(127,4249988676030597), - (103543712111871836,5),(71,16954462148248238),(3963027173504,216570), - (13000,503523808916753),(17038308,20018685905),(0,510350226577891549), - (175898,3875698895405),(425299191292676,5651),(17223451323664536,50), - (61755131,14247665326),(0,1018195131697569303),(36433751497238985,20), - (3473607861601050,1837),(1392342328,1733971838),(225770297367,3249655), - (14,127545244155254102),(1751488975299136,2634),(3949208,504190668767), - (153329,831454434345),(1066212122928663658,2),(351224,2663633539556), - (344565,53388869217),(35825609350446863,54),(276011553660081475,10), - (1969754174790470349,3),(35,68088438338633),(506710,3247689556438), - (11099382291,327739909),(105787303549,32824363),(210366111,14759049409), - (688893241579,3102676),(8490,70047474429581),(152085,29923000251880), - (5046974599257095,400),(4183167795,263434071),(10089728,502781960687), - (44831977765,4725378),(91,8978094664238578),(30990165721,44053350), - (1772377,149651820860),(243420621763408572,4),(32,5790357453815138), - (27980806337993771,5),(47696295759774,20848),(1745874142313778,1098), - (46869334770121,1203),(886995283,1564424789),(40679396544,76002479), - (1,672849481568486995),(337656187205,3157069),(816980552858963,6003), - (2271434085804831543,1),(0,1934521023868747186),(6266220038281,15825), - (4160,107115946987394),(524,246808621791561),(0,1952519482439636339), - (128,2865935904539691),(1044,3211982069426297),(16000511542473,88922), - (1253596745404082,2226),(27041,56836278958002),(23201,49247489754471), - (175906590497,21252392),(185163584757182295,24),(34742225226802197,150), - (2363228,250824838408),(216327527109550,45),(24,81574076994520675), - (28559899906542,15356),(10890139774837133,511),(2293,707179303654492), - (2749366833,40703233),(0,4498229704622845986),(439,4962056468281937), - (662,1453820621089921),(16336770612459631,220),(24282989393,74239137), - (2724564648490195,3),(743672760,124992589),(4528103,704330948891), - (6050483122491561,250),(13322953,13594265152),(181794,22268101450214), - (25957941712,75384092),(43352,7322262295009),(32838,52609059549923), - (33003585202001564,2),(103019,68430142267402),(129918230800,8742978), - (0,2114347379589080688),(2548,905723041545274),(222745067962838382,0), - (1671683850790425181,1),(455,4836932776795684),(794227702827214,6620), - (212534135175874,1365),(96432431858,29784975),(466626763743380,3484), - (29793949,53041519613),(8359,309952753409844),(3908960585331901,26), - (45185288970365760,114),(10131829775,68110174),(58039242399640479,83), - (628092278238719399,6),(1,196469106875361889),(302336625,16347502444), - (148,3748088684181047),(1,1649096568849015456),(1019866864,2349753026), - (8211344830,569363306),(65647579546873,34753),(2340190,1692053129069), - (64263301,30758930355),(48681618072372209,110),(7074794736,47640197), - (249634721521,7991792),(1162917363807215,232),(7446433349,420634045), - (63398619383,60709817),(51359004508011,14200),(131788797028647,7072), - (52079887791430043,7),(7,136277667582599838),(28582879735696,50327), - (1404582800566278,833),(469164435,15017166943),(99567079957578263,49), - (1015285971,3625801566),(321504843,4104079293),(5196954,464515406632), - (114246832260876,7468),(8149664437,487119673),(12265299,378168974869), - (37711995764,30766513),(3971137243,710996152),(483120070302,603162), - (103009942,61645547145),(8476344625340,6987),(547948761229739,1446), - (42234,18624767306301),(13486714173011,58948),(4,198309153268019840), - (9913176974,325539248),(28246225540203,116822),(2882463945582154,18), - (959,25504987505398),(3,1504372236378217710),(13505229956793,374987), - (751661959,457611342),(27375926,36219151769),(482168869,5301952074), - (1,1577425863241520640),(714116235611821,1164),(904492524250310488,0), - (5983514941763398,68),(10759472423,23540686),(72539568471529,34919), - (4,176090672310337473),(938702842110356453,1),(673652445,3335287382), - (3111998893666122,917),(1568013,3168419765469)] - -testVector32 :: U.Vector (Int32,Int32) -testVector32 = U.fromList [ - (39242,410),(0,100077553),(2206,9538),(509400240,1),(38048,6368), - (1789,651480),(2399,157032),(701,170017),(5241456,14),(11212,70449), - (1,227804876),(749687254,1),(74559,2954),(1158,147957),(410604456,1), - (170851,1561),(92643422,1),(6192,180509),(7,24202210),(3440,241481), - (5753677,5),(294327,1622),(252,4454673),(127684121,11),(28315800,30), - (340370905,0),(1,667887987),(592782090,1),(49023,27641),(750,290387), - (72886,3847),(0,301047933),(3050276,473),(1,788366142),(59457,15813), - (637726933,1),(1135,344317),(853616,264),(696816,493),(7038,12046), - (125219574,4),(803694088,1),(107081726,1),(39294,21699),(16361,38191), - (132561123,12),(1760,23499),(847543,484),(175687349,1),(2963,252678), - (6248,224553),(27596,4606),(5422922,121),(1542,485890),(131,583035), - (59096,4925),(3637115,132),(0,947225435),(86854,6794),(2984745,339), - (760129569,1),(1,68260595),(380835652,2),(430575,2579),(54514,7211), - (15550606,3),(9,27367402),(3007053,207),(7060988,60),(28560,27130), - (1355,21087),(10880,53059),(14563646,4),(461886361,1),(2,169260724), - (241454126,2),(406797,1),(61631630,16),(44473,5943),(63869104,12), - (950300,1528),(2113,62333),(120817,9358),(100261456,1),(426764723,1), - (119,12723684),(3,53358711),(4448071,18),(1,230278091),(238,232102), - (8,57316440),(42437979,10),(6769,19555),(48590,22006),(11500585,79), - (2808,97638),(42,26952545),(11,32104194),(23954638,1),(785427272,0), - (513,81379),(31333960,37),(897772,1009),(4,25679692),(103027993,12), - (104972702,11),(546,443401),(7,65137092),(88574269,3),(872139069,0), - (2,97417121),(378802603,0),(141071401,4),(22613,10575),(2191743,118), - (470,116119),(7062,38166),(231056,1847),(43901963,9),(2400,70640), - (63553,1555),(34,11249573),(815174,1820),(997894011,0),(98881794,2), - (5448,43132),(27956,9),(904926,1357),(112608626,3),(124,613021), - (282086,1966),(99,10656881),(113799,1501),(433318,2085),(442,948171), - (165380,1043),(28,14372905),(14880,50462),(2386,219918),(229,1797565), - (1174961,298),(3925,41833),(3903515,299),(15690452,111),(360860521,3), - (7440846,81),(2541026,507),(0,492448477),(6869,82469),(245,8322939), - (3503496,253),(123495298,0),(150963,2299),(33,4408482),(1,200911107), - (305,252121),(13,123369189),(215846,8181),(2440,65387),(776764401,1), - (1241172,434),(8,15493155),(81953961,6),(17884993,5),(26,6893822), - (0,502035190),(1,582451018),(2,514870139),(227,3625619),(49,12720258), - (1456769,207),(94797661,10),(234407,893),(26843,5783),(15688,24547), - (4091,86268),(4339448,151),(21360,6294),(397046497,2),(1227,205936), - (9966,21959),(160046791,1),(0,159992224),(27,24974797),(19177,29334), - (4136148,42),(21179785,53),(61256583,31),(385,344176),(7,11934915), - (1,18992566),(3488065,5),(768021,224),(36288474,7),(8624,117561), - (8,20341439),(5903,261475),(561,1007618),(1738,392327),(633049,1708)] diff --git a/benchmarks/haskell/Benchmarks/Pure.hs b/benchmarks/haskell/Benchmarks/Pure.hs --- a/benchmarks/haskell/Benchmarks/Pure.hs +++ b/benchmarks/haskell/Benchmarks/Pure.hs @@ -14,8 +14,8 @@ import Control.DeepSeq (NFData (..)) import Control.Exception (evaluate) +import Data.Char (chr, ord) import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) -import GHC.Base (Char (..), Int (..), chr#, ord#, (+#)) import GHC.Generics (Generic) import GHC.Int (Int64) import qualified Data.ByteString.Char8 as BS @@ -74,7 +74,7 @@ benchmark :: String -> Env -> Benchmark benchmark kind ~Env{..} = - bgroup "Pure" + bgroup kind [ bgroup "append" [ benchT $ nf (T.append tb) ta , benchTL $ nf (TL.append tlb) tla @@ -279,17 +279,17 @@ ] ] , bgroup "Builder" - [ bench ("mappend char+" ++ kind) $ + [ bench "mappend char" $ nf (TL.length . TB.toLazyText . mappendNChar 'a') 10000 - , bench ("mappend 8 char+" ++ kind) $ + , bench "mappend 8 char" $ nf (TL.length . TB.toLazyText . mappend8Char) 'a' - , bench ("mappend text+" ++ kind) $ + , bench "mappend text" $ nf (TL.length . TB.toLazyText . mappendNText short) 10000 ] ] where - benchT = bench ("Text+" ++ kind) - benchTL = bench ("LazyText+" ++ kind) + benchT = bench "Text" + benchTL = bench "LazyText" c = 'й' p0 = (== c) @@ -297,10 +297,14 @@ lw = "право" tsw = T.pack lw tlw = TL.fromChunks [tsw] - f (C# c#) = C# (chr# (ord# c# +# 1#)) - g (I# i#) (C# c#) = (I# (i# +# 1#), C# (chr# (ord# c# +# i#))) len l _ = l + (1::Int) short = T.pack "short" + + -- Valid 'Char' are in range [0..0x10FFFF], otherwise 'chr' throws an 'error'. + -- 'Data.Text.Internal.safe' does not validate this, it assumes that inputs + -- has been already sanitized to belong to the range. + f !ch = chr (min 0x10FFFF (ord ch + 1)) + g !i !ch = (i + 1, chr (min 0x10FFFF (ord ch + i))) data B where B :: NFData a => a -> B diff --git a/benchmarks/haskell/Benchmarks/ReadNumbers.hs b/benchmarks/haskell/Benchmarks/ReadNumbers.hs --- a/benchmarks/haskell/Benchmarks/ReadNumbers.hs +++ b/benchmarks/haskell/Benchmarks/ReadNumbers.hs @@ -23,10 +23,6 @@ import Test.Tasty.Bench (Benchmark, bgroup, bench, whnf) import Data.List (foldl') -import Numeric (readDec, readFloat, readHex) -import qualified Data.ByteString.Char8 as B -import qualified Data.ByteString.Lazy.Char8 as BL -import qualified Data.ByteString.Lex.Fractional as B import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL @@ -34,27 +30,18 @@ import qualified Data.Text.Lazy.Read as TL import qualified Data.Text.Read as T -type Env = ([String], [T.Text], [TL.Text], [B.ByteString], [BL.ByteString]) +type Env = ([T.Text], [TL.Text]) initEnv :: FilePath -> IO Env initEnv fp = do - -- Read all files into lines: string, text, lazy text, bytestring, lazy - -- bytestring - s <- lines `fmap` readFile fp t <- T.lines `fmap` T.readFile fp tl <- TL.lines `fmap` TL.readFile fp - b <- B.lines `fmap` B.readFile fp - bl <- BL.lines `fmap` BL.readFile fp - return (s, t, tl, b, bl) + return (t, tl) benchmark :: Env -> Benchmark -benchmark ~(s, t, tl, b, bl) = +benchmark ~(t, tl) = bgroup "ReadNumbers" - [ bench "DecimalString" $ whnf (int . string readDec) s - , bench "HexadecimalString" $ whnf (int . string readHex) s - , bench "DoubleString" $ whnf (double . string readFloat) s - - , bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t + [ bench "DecimalText" $ whnf (int . text (T.signed T.decimal)) t , bench "HexadecimalText" $ whnf (int . text (T.signed T.hexadecimal)) t , bench "DoubleText" $ whnf (double . text T.double) t , bench "RationalText" $ whnf (double . text T.rational) t @@ -67,12 +54,6 @@ whnf (double . text TL.double) tl , bench "RationalLazyText" $ whnf (double . text TL.rational) tl - - , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b - , bench "DoubleByteString" $ whnf (double . byteString B.readDecimal) b - - , bench "DecimalLazyByteString" $ - whnf (int . byteString BL.readInt) bl ] where -- Used for fixing types @@ -81,20 +62,8 @@ double :: Double -> Double double = id -string :: (Ord a, Num a) => (t -> [(a, t)]) -> [t] -> a -string reader = foldl' go 1000000 - where - go z t = case reader t of [(n, _)] -> min n z - _ -> z - text :: (Ord a, Num a) => (t -> Either String (a,t)) -> [t] -> a text reader = foldl' go 1000000 where go z t = case reader t of Left _ -> z Right (n, _) -> min n z - -byteString :: (Ord a, Num a) => (t -> Maybe (a,t)) -> [t] -> a -byteString reader = foldl' go 1000000 - where - go z t = case reader t of Nothing -> z - Just (n, _) -> min n z diff --git a/benchmarks/haskell/Benchmarks/Replace.hs b/benchmarks/haskell/Benchmarks/Replace.hs --- a/benchmarks/haskell/Benchmarks/Replace.hs +++ b/benchmarks/haskell/Benchmarks/Replace.hs @@ -11,40 +11,26 @@ ) where import Test.Tasty.Bench (Benchmark, bgroup, bench, nf) -import qualified Data.ByteString.Char8 as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Search as BL -import qualified Data.ByteString.Search as B import qualified Data.Text as T -import qualified Data.Text.Encoding as T import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.Encoding as TL import qualified Data.Text.Lazy.IO as TL -type Env = (T.Text, B.ByteString, TL.Text, BL.ByteString) +type Env = (T.Text, TL.Text) initEnv :: FilePath -> IO Env initEnv fp = do tl <- TL.readFile fp - bl <- BL.readFile fp let !t = TL.toStrict tl - !b = T.encodeUtf8 t - return (t, b, tl, bl) + return (t, tl) benchmark :: String -> String -> Env -> Benchmark -benchmark pat sub ~(t, b, tl, bl) = +benchmark pat sub ~(t, tl) = bgroup "Replace" [ bench "Text" $ nf (T.length . T.replace tpat tsub) t - , bench "ByteString" $ nf (BL.length . B.replace bpat bsub) b , bench "LazyText" $ nf (TL.length . TL.replace tlpat tlsub) tl - , bench "LazyByteString" $ nf (BL.length . BL.replace blpat blsub) bl ] where tpat = T.pack pat tsub = T.pack sub tlpat = TL.pack pat tlsub = TL.pack sub - bpat = T.encodeUtf8 tpat - bsub = T.encodeUtf8 tsub - blpat = B.concat $ BL.toChunks $ TL.encodeUtf8 tlpat - blsub = B.concat $ BL.toChunks $ TL.encodeUtf8 tlsub diff --git a/benchmarks/haskell/Benchmarks/Search.hs b/benchmarks/haskell/Benchmarks/Search.hs --- a/benchmarks/haskell/Benchmarks/Search.hs +++ b/benchmarks/haskell/Benchmarks/Search.hs @@ -10,43 +10,27 @@ ) where import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import qualified Data.ByteString.Lazy.Search as BL -import qualified Data.ByteString.Search as B import qualified Data.Text as T -import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL -type Env = (B.ByteString, BL.ByteString, T.Text, TL.Text) +type Env = (T.Text, TL.Text) initEnv :: FilePath -> IO Env initEnv fp = do - b <- B.readFile fp - bl <- BL.readFile fp t <- T.readFile fp tl <- TL.readFile fp - return (b, bl, t, tl) + return (t, tl) benchmark :: T.Text -> Env -> Benchmark -benchmark needleT ~(b, bl, t, tl) = +benchmark needleT ~(t, tl) = bgroup "FileIndices" - [ bench "ByteString" $ whnf (byteString needleB) b - , bench "LazyByteString" $ whnf (lazyByteString needleB) bl - , bench "Text" $ whnf (text needleT) t + [ bench "Text" $ whnf (text needleT) t , bench "LazyText" $ whnf (lazyText needleTL) tl ] where - needleB = T.encodeUtf8 needleT needleTL = TL.fromChunks [needleT] - -byteString :: B.ByteString -> B.ByteString -> Int -byteString needle = length . B.indices needle - -lazyByteString :: B.ByteString -> BL.ByteString -> Int -lazyByteString needle = length . BL.indices needle text :: T.Text -> T.Text -> Int text = T.count diff --git a/benchmarks/haskell/Benchmarks/WordFrequencies.hs b/benchmarks/haskell/Benchmarks/WordFrequencies.hs --- a/benchmarks/haskell/Benchmarks/WordFrequencies.hs +++ b/benchmarks/haskell/Benchmarks/WordFrequencies.hs @@ -14,29 +14,23 @@ ) where import Test.Tasty.Bench (Benchmark, bench, bgroup, whnf) -import Data.Char (toLower) import Data.List (foldl') import Data.Map (Map) -import qualified Data.ByteString.Char8 as B import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.IO as T -type Env = (String, B.ByteString, T.Text) +type Env = T.Text initEnv :: FilePath -> IO Env initEnv fp = do - s <- readFile fp - b <- B.readFile fp t <- T.readFile fp - return (s, b, t) + return t benchmark :: Env -> Benchmark -benchmark ~(s, b, t) = +benchmark ~t = bgroup "WordFrequencies" - [ bench "String" $ whnf (frequencies . words . map toLower) s - , bench "ByteString" $ whnf (frequencies . B.words . B.map toLower) b - , bench "Text" $ whnf (frequencies . T.words . T.toLower) t + [ bench "Text" $ whnf (frequencies . T.words . T.toLower) t ] frequencies :: Ord a => [a] -> Map a Int diff --git a/cbits/cbits.c b/cbits/cbits.c deleted file mode 100644 --- a/cbits/cbits.c +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright (c) 2011 Bryan O'Sullivan . - * - * Portions copyright (c) 2008-2010 Björn Höhrmann . - * - * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. - */ - -#include -#include -#include -#if defined(__x86_64__) -#include -#include -#endif - -#include "text_cbits.h" - -void _hs_text_memcpy(void *dest, size_t doff, const void *src, size_t soff, - size_t n) -{ - memcpy(dest + (doff<<1), src + (soff<<1), n<<1); -} - -int _hs_text_memcmp(const void *a, size_t aoff, const void *b, size_t boff, - size_t n) -{ - return memcmp(a + (aoff<<1), b + (boff<<1), n<<1); -} - -#define UTF8_ACCEPT 0 -#define UTF8_REJECT 12 - -static const uint8_t utf8d[] = { - /* - * The first part of the table maps bytes to character classes that - * to reduce the size of the transition table and create bitmasks. - */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - - /* - * The second part is a transition table that maps a combination of - * a state of the automaton and a character class to a state. - */ - 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, - 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, - 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, - 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, - 12,36,12,12,12,12,12,12,12,12,12,12, -}; - -static inline uint32_t -decode(uint32_t *state, uint32_t* codep, uint32_t byte) { - uint32_t type = utf8d[byte]; - - *codep = (*state != UTF8_ACCEPT) ? - (byte & 0x3fu) | (*codep << 6) : - (0xff >> type) & (byte); - - return *state = utf8d[256 + *state + type]; -} - -/* - * The ISO 8859-1 (aka latin-1) code points correspond exactly to the first 256 unicode - * code-points, therefore we can trivially convert from a latin-1 encoded bytestring to - * an UTF16 array - */ -void -_hs_text_decode_latin1(uint16_t *dest, const uint8_t *src, - const uint8_t *srcend) -{ - const uint8_t *p = src; - -#if defined(__i386__) || defined(__x86_64__) - /* This optimization works on a little-endian systems by using - (aligned) 32-bit loads instead of 8-bit loads - */ - - /* consume unaligned prefix */ - while (p != srcend && (uintptr_t)p & 0x3) - *dest++ = *p++; - -#if defined(__x86_64__) - /* All the intrinsics used here are from SSE2, - * so every x86_64 CPU supports them. - */ - const __m128i zeros = _mm_set1_epi32(0); - while (p < srcend - 7) { - /* Load 8 bytes of ASCII data */ - const __m128i ascii = _mm_cvtsi64_si128(*((const uint64_t *)p)); - /* Interleave with zeros */ - const __m128i utf16 = _mm_unpacklo_epi8(ascii, zeros); - /* Store the resulting 16 bytes into destination */ - _mm_storeu_si128((__m128i *)dest, utf16); - - dest += 8; - p += 8; - } -#else - /* iterate over 32-bit aligned loads */ - while (p < srcend - 3) { - const uint32_t w = *((const uint32_t *)p); - - *dest++ = w & 0xff; - *dest++ = (w >> 8) & 0xff; - *dest++ = (w >> 16) & 0xff; - *dest++ = (w >> 24) & 0xff; - - p += 4; - } -#endif -#endif - - /* handle unaligned suffix */ - while (p != srcend) - *dest++ = *p++; -} - -/* - * A best-effort decoder. Runs until it hits either end of input or - * the start of an invalid byte sequence. - * - * At exit, we update *destoff with the next offset to write to, *src - * with the next source location past the last one successfully - * decoded, and return the next source location to read from. - * - * Moreover, we expose the internal decoder state (state0 and - * codepoint0), allowing one to restart the decoder after it - * terminates (say, due to a partial codepoint). - * - * In particular, there are a few possible outcomes, - * - * 1) We decoded the buffer entirely: - * In this case we return srcend - * state0 == UTF8_ACCEPT - * - * 2) We met an invalid encoding - * In this case we return the address of the first invalid byte - * state0 == UTF8_REJECT - * - * 3) We reached the end of the buffer while decoding a codepoint - * In this case we return a pointer to the first byte of the partial codepoint - * state0 != UTF8_ACCEPT, UTF8_REJECT - * - */ -#if defined(__GNUC__) || defined(__clang__) -static inline uint8_t const * -_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff, - const uint8_t **src, const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) - __attribute((always_inline)); -#endif - -static inline uint8_t const * -_hs_text_decode_utf8_int(uint16_t *const dest, size_t *destoff, - const uint8_t **src, const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) -{ - uint16_t *d = dest + *destoff; - const uint8_t *s = *src, *last = *src; - uint32_t state = *state0; - uint32_t codepoint = *codepoint0; - - while (s < srcend) { -#if defined(__i386__) || defined(__x86_64__) - /* - * This code will only work on a little-endian system that - * supports unaligned loads. - * - * It gives a substantial speed win on data that is purely or - * partly ASCII (e.g. HTML), at only a slight cost on purely - * non-ASCII text. - */ - - if (state == UTF8_ACCEPT) { -#if defined(__x86_64__) - const __m128i zeros = _mm_set1_epi32(0); - while (s < srcend - 8) { - const uint64_t hopefully_eight_ascii_chars = *((uint64_t *) s); - if ((hopefully_eight_ascii_chars & 0x8080808080808080LL) != 0LL) - break; - s += 8; - - /* Load 8 bytes of ASCII data */ - const __m128i eight_ascii_chars = _mm_cvtsi64_si128(hopefully_eight_ascii_chars); - /* Interleave with zeros */ - const __m128i eight_utf16_chars = _mm_unpacklo_epi8(eight_ascii_chars, zeros); - /* Store the resulting 16 bytes into destination */ - _mm_storeu_si128((__m128i *)d, eight_utf16_chars); - d += 8; - } -#else - while (s < srcend - 4) { - codepoint = *((uint32_t *) s); - if ((codepoint & 0x80808080) != 0) - break; - s += 4; - /* - * Tried 32-bit stores here, but the extra bit-twiddling - * slowed the code down. - */ - *d++ = (uint16_t) (codepoint & 0xff); - *d++ = (uint16_t) ((codepoint >> 8) & 0xff); - *d++ = (uint16_t) ((codepoint >> 16) & 0xff); - *d++ = (uint16_t) ((codepoint >> 24) & 0xff); - } -#endif - last = s; - } /* end if (state == UTF8_ACCEPT) */ -#endif - - if (decode(&state, &codepoint, *s++) != UTF8_ACCEPT) { - if (state != UTF8_REJECT) - continue; - break; - } - - if (codepoint <= 0xffff) - *d++ = (uint16_t) codepoint; - else { - *d++ = (uint16_t) (0xD7C0 + (codepoint >> 10)); - *d++ = (uint16_t) (0xDC00 + (codepoint & 0x3FF)); - } - last = s; - } - - *destoff = d - dest; - *codepoint0 = codepoint; - *state0 = state; - *src = last; - - return s; -} - -uint8_t const * -_hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff, - const uint8_t **src, - const uint8_t *srcend, - uint32_t *codepoint0, uint32_t *state0) -{ - _hs_text_decode_utf8_int(dest, destoff, src, srcend, codepoint0, state0); - - return *src; -} - -/* - * Helper to decode buffer and discard final decoder state - */ -const uint8_t * -_hs_text_decode_utf8(uint16_t *const dest, size_t *destoff, - const uint8_t *src, const uint8_t *const srcend) -{ - uint32_t codepoint; - uint32_t state = UTF8_ACCEPT; - _hs_text_decode_utf8_int(dest, destoff, &src, srcend, - &codepoint, &state); - return src; -} - -void -_hs_text_encode_utf8(uint8_t **destp, const uint16_t *src, size_t srcoff, - size_t srclen) -{ - const uint16_t *srcend; - uint8_t *dest = *destp; - - src += srcoff; - srcend = src + srclen; - - ascii: -#if defined(__x86_64__) - while (srcend - src >= 8) { - union { uint64_t halves[2]; __m128i whole; } eight_chars; - eight_chars.whole = _mm_loadu_si128((__m128i *) src); - - const uint64_t w = eight_chars.halves[0]; - if (w & 0xFF80FF80FF80FF80ULL) { - if (!(w & 0x000000000000FF80ULL)) { - *dest++ = w & 0xFFFF; - src++; - if (!(w & 0x00000000FF800000ULL)) { - *dest++ = (w >> 16) & 0xFFFF; - src++; - if (!(w & 0x0000FF8000000000ULL)) { - *dest++ = (w >> 32) & 0xFFFF; - src++; - } - } - } - break; - } - - if (eight_chars.halves[1] & 0xFF80FF80FF80FF80ULL) { - break; - } - - const __m128i eight_ascii_chars = _mm_packus_epi16(eight_chars.whole, eight_chars.whole); - _mm_storel_epi64((__m128i *)dest, eight_ascii_chars); - - dest += 8; - src += 8; - } -#endif - -#if defined(__i386__) - while (srcend - src >= 2) { - uint32_t w = *((uint32_t *) src); - - if (w & 0xFF80FF80) - break; - *dest++ = w & 0xFFFF; - *dest++ = w >> 16; - src += 2; - } -#endif - - while (src < srcend) { - uint16_t w = *src++; - - if (w <= 0x7F) { - *dest++ = w; - /* An ASCII byte is likely to begin a run of ASCII bytes. - Falling back into the fast path really helps performance. */ - goto ascii; - } - else if (w <= 0x7FF) { - *dest++ = (w >> 6) | 0xC0; - *dest++ = (w & 0x3f) | 0x80; - } - else if (w < 0xD800 || w > 0xDBFF) { - *dest++ = (w >> 12) | 0xE0; - *dest++ = ((w >> 6) & 0x3F) | 0x80; - *dest++ = (w & 0x3F) | 0x80; - } else { - uint32_t c = ((((uint32_t) w) - 0xD800) << 10) + - (((uint32_t) *src++) - 0xDC00) + 0x10000; - *dest++ = (c >> 18) | 0xF0; - *dest++ = ((c >> 12) & 0x3F) | 0x80; - *dest++ = ((c >> 6) & 0x3F) | 0x80; - *dest++ = (c & 0x3F) | 0x80; - } - } - - *destp = dest; -} diff --git a/cbits/is_ascii.c b/cbits/is_ascii.c new file mode 100644 --- /dev/null +++ b/cbits/is_ascii.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#ifdef __x86_64__ +#include +#include +#endif +#include + +/* + _hs_text_is_ascii takes a UTF-8 encoded buffer, + and returns the length of the ASCII-compatible prefix. +*/ +const size_t _hs_text_is_ascii(const uint8_t *src0, const uint8_t *srcend){ + const uint8_t *src = src0; + +#ifdef __x86_64__ + // I experimented with larger vector registers, + // but did not notice any measurable speed up, so let's keep it simple. + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are < 128? + uint16_t mask = _mm_movemask_epi8(w128); + if (mask) break; + src+= 16; + } +#endif + + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + if (w64 & 0x8080808080808080ULL) break; + src+= 8; + } + + while (src < srcend){ + uint8_t leadByte = *src; + if(leadByte >= 0x80) break; + src++; + } + + return src - src0; +} diff --git a/cbits/measure_off.c b/cbits/measure_off.c new file mode 100644 --- /dev/null +++ b/cbits/measure_off.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#ifdef __x86_64__ +#include +#include +#include +#include +#endif +#include + +#ifndef __STDC_NO_ATOMICS__ +#include +#endif + +bool has_avx512_vl_bw() { +#ifdef __x86_64__ + uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; + __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); + // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features + const bool has_avx512_bw = ebx & (1 << 30); + const bool has_avx512_vl = ebx & (1 << 31); + // printf("cpuid=%d=cpuid\n", has_avx512_bw && has_avx512_vl); + return has_avx512_bw && has_avx512_vl; +#else + return false; +#endif +} + +/* + measure_off_naive / measure_off_avx / measure_off_sse + take a UTF-8 sequence between src and srcend, and a number of characters cnt. + If the sequence is long enough to contain cnt characters, then return how many bytes + remained unconsumed. Otherwise, if the sequence is shorter, return + negated count of lacking characters. Cf. _hs_text_measure_off below. +*/ + +static inline const ssize_t measure_off_naive(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + // Count leading bytes in 8 byte sequence + while (src < srcend - 7){ + uint64_t w64; + memcpy(&w64, src, sizeof(uint64_t)); + size_t leads = __builtin_popcountll(((w64 << 1) | ~w64) & 0x8080808080808080ULL); + if (cnt < leads) break; + cnt-= leads; + src+= 8; + } + + // Skip until next leading byte + while (src < srcend){ + uint8_t w8 = *src; + if ((int8_t)w8 >= -0x40) break; + src++; + } + + // Finish up with tail + while (src < srcend && cnt > 0){ + uint8_t leadByte = *src++; + cnt--; + src+= (leadByte >= 0xc0) + (leadByte >= 0xe0) + (leadByte >= 0xf0); + } + + return cnt == 0 ? (ssize_t)(srcend - src) : (ssize_t)(- cnt); +} + +#ifdef __x86_64__ +__attribute__((target("avx512vl,avx512bw"))) +static const ssize_t measure_off_avx(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ + while (src < srcend - 63){ + __m512i w512 = _mm512_loadu_si512((__m512i *)src); + // Which bytes are either < 128 or >= 192? + uint64_t mask = _mm512_cmpgt_epi8_mask(w512, _mm512_set1_epi8(0xBF)); + size_t leads = __builtin_popcountll(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 64; + } + + // Cannot proceed to measure_off_sse, because of AVX-SSE transition penalties + // https://software.intel.com/content/www/us/en/develop/articles/avoiding-avx-sse-transition-penalties.html + + if (src < srcend - 31){ + __m256i w256 = _mm256_loadu_si256((__m256i *)src); + uint32_t mask = _mm256_cmpgt_epi8_mask(w256, _mm256_set1_epi8(0xBF)); + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 32; + } + } + + if (src < srcend - 15){ + __m128i w128 = _mm_maskz_loadu_epi16(0xFF, (__m128i *)src); // not _mm_loadu_si128; and GCC does not have _mm_loadu_epi16 + uint16_t mask = _mm_cmpgt_epi8_mask(w128, _mm_set1_epi8(0xBF)); // not _mm_movemask_epi8 + size_t leads = __builtin_popcountl(mask); + if (cnt >= leads){ + cnt-= leads; + src+= 16; + } + } + + return measure_off_naive(src, srcend, cnt); +} +#endif + +static const ssize_t measure_off_sse(const uint8_t *src, const uint8_t *srcend, size_t cnt) +{ +#ifdef __x86_64__ + while (src < srcend - 15){ + __m128i w128 = _mm_loadu_si128((__m128i *)src); + // Which bytes are either < 128 or >= 192? + uint16_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(w128, _mm_set1_epi8(0xBF))); + size_t leads = __builtin_popcount(mask); + if (cnt < leads) break; + cnt-= leads; + src+= 16; + } +#endif + + return measure_off_naive(src, srcend, cnt); +} + +typedef const ssize_t (*measure_off_t) (const uint8_t*, const uint8_t*, size_t); + +/* + _hs_text_measure_off takes a UTF-8 encoded buffer, specified by (src, off, len), + and a number of code points (aka characters) cnt. If the buffer is long enough + to contain cnt characters, then _hs_text_measure_off returns a non-negative number, + measuring their size in code units (aka bytes). If the buffer is shorter, + _hs_text_measure_off returns a non-positive number, which is a negated total count + of characters available in the buffer. If len = 0 or cnt = 0, this function returns 0 + as well. + + This scheme allows us to implement both take/drop and length with the same C function. + + The input buffer (src, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +ssize_t _hs_text_measure_off(const uint8_t *src, size_t off, size_t len, size_t cnt) { + static _Atomic measure_off_t s_impl = (measure_off_t)NULL; + measure_off_t impl = atomic_load_explicit(&s_impl, memory_order_relaxed); + if (!impl) { +#ifdef __x86_64__ + impl = has_avx512_vl_bw() ? measure_off_avx : measure_off_sse; +#else + impl = measure_off_sse; +#endif + atomic_store_explicit(&s_impl, impl, memory_order_relaxed); + } + ssize_t ret = (*impl)(src + off, src + off + len, cnt); + return ret >= 0 ? ((ssize_t)len - ret) : (- (cnt + ret)); +} diff --git a/cbits/reverse.c b/cbits/reverse.c new file mode 100644 --- /dev/null +++ b/cbits/reverse.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include + +/* + _hs_text_reverse takes a UTF-8 encoded buffer, specified by (src0, off, len), + and reverses it, writing output starting from dst0. + + The input buffer (src0, off, len) must be a valid UTF-8 sequence, + this condition is not checked. +*/ +void _hs_text_reverse(uint8_t *dst0, const uint8_t *src0, size_t off, size_t len) +{ + const uint8_t *src = src0 + off; + const uint8_t *srcend = src + len; + uint8_t *dst = dst0 + len - 1; + + while (src < srcend){ + uint8_t leadByte = *src++; + if (leadByte < 0x80){ + *dst-- = leadByte; + } else if (leadByte < 0xe0){ + *(dst-1) = leadByte; + *dst = *src++; + dst-=2; + } else if (leadByte < 0xf0){ + *(dst-2) = leadByte; + *(dst-1) = *src++; + *dst = *src++; + dst-=3; + } else { + *(dst-3) = leadByte; + *(dst-2) = *src++; + *(dst-1) = *src++; + *dst = *src++; + dst-=4; + } + } +} diff --git a/cbits/utils.c b/cbits/utils.c new file mode 100644 --- /dev/null +++ b/cbits/utils.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2021 Andrew Lelechenko + */ + +#include +#include +#include +#include + +/* Changed name to disambiguate from _hs_text_memcmp, + which could be present in system-wide headers from installed ghc package */ +int _hs_text_memcmp2(const void *arr1, size_t off1, const void *arr2, size_t off2, size_t len) +{ + return memcmp(arr1 + off1, arr2 + off2, len); +} + +ssize_t _hs_text_memchr(const void *arr, size_t off, size_t len, uint8_t byte) +{ + const void *ptr = memchr(arr + off, byte, len); + return ptr == NULL ? -1 : ptr - (arr + off); +} diff --git a/cbits/validate_utf8.cpp b/cbits/validate_utf8.cpp new file mode 100644 --- /dev/null +++ b/cbits/validate_utf8.cpp @@ -0,0 +1,6 @@ +#include "simdutf.h" + +extern "C" +int _hs_text_is_valid_utf8(const char* str, size_t len){ + return simdutf::validate_utf8(str, len); +} diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,32 @@ +### 2.0 + +* [Switch internal representation of text from UTF-16 to UTF-8](https://github.com/haskell/text/pull/365): + * Functions in `Data.Text.Array` now operate over arrays of `Word8` instead of `Word16`. + * Rename constructors of `Array` and `MArray` to `ByteArray` and `MutableByteArray`. + * Rename functions and types in `Data.Text.Foreign` to reflect switch + from `Word16` to `Word8`. + * Rename slicing functions in `Data.Text.Unsafe` to reflect switch + from `Word16` to `Word8`. + * Rename `Data.Text.Internal.Unsafe.Char.unsafeChr` to `unsafeChr16`. + * Change semantics and order of arguments of `Data.Text.Array.copyI`: + pass length, not end offset. + * Extend `Data.Text.Internal.Encoding.Utf8` to provide more UTF-8 related routines. + * Extend interface of `Data.Text.Array` with more utility functions. + * Add `instance Show Data.Text.Unsafe.Iter`. + * Add `Data.Text.measureOff`. + * Extend `Data.Text.Unsafe` with `iterArray` and `reverseIterArray`. + * Export `Data.Text.Internal.Lazy.equal`. + * Export `Data.Text.Internal.append`. + * Add `Data.Text.Internal.Private.spanAscii_`. + * Replacement characters in `decodeUtf8With` are no longer limited to Basic Multilingual Plane. +* [Disable implicit fusion rules](https://github.com/haskell/text/pull/348) +* [Add `Data.Text.Encoding.decodeUtf8Lenient`](https://github.com/haskell/text/pull/342) +* [Remove `Data.Text.Internal.Unsafe.Shift`](https://github.com/haskell/text/pull/343) +* [Remove `Data.Text.Internal.Functions`](https://github.com/haskell/text/pull/354) +* [Bring type of `Data.Text.Unsafe.reverseIter` in line with `iter`](https://github.com/haskell/text/pull/355) +* [Add `instance Bounded FPFormat`](https://github.com/haskell/text/pull/355) +* [Add HasCallStack to partial functions](https://github.com/haskell/text/pull/388) + ### 1.2.5.0 * [Support sized primitives from GHC 9.2](https://github.com/haskell/text/pull/305) diff --git a/include/text_cbits.h b/include/text_cbits.h deleted file mode 100644 --- a/include/text_cbits.h +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2013 Bryan O'Sullivan . - */ - -#ifndef _text_cbits_h -#define _text_cbits_h - -#define UTF8_ACCEPT 0 -#define UTF8_REJECT 12 - -#endif diff --git a/scripts/CaseFolding.hs b/scripts/CaseFolding.hs --- a/scripts/CaseFolding.hs +++ b/scripts/CaseFolding.hs @@ -11,6 +11,7 @@ ) where import Arsec +import Data.Bits data Fold = Fold { code :: Char @@ -34,13 +35,19 @@ parseCF name = parse entries name <$> readFile name mapCF :: CaseFolding -> [String] -mapCF (CF _ ms) = typ ++ (map nice . filter p $ ms) ++ [last] +mapCF (CF _ ms) = typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] where - typ = ["foldMapping :: forall s. Char -> s -> Step (CC s) Char" - ,"{-# NOINLINE foldMapping #-}"] - last = "foldMapping c s = Yield (toLower c) (CC s '\\0' '\\0')" - nice c = "-- " ++ name c ++ "\n" ++ - "foldMapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")" - where [x,y,z] = (map showC . take 3) (mapping c ++ repeat '\0') + ms' = filter p ms p f = status f `elem` "CF" && mapping f /= [toLower (code f)] + unusual = map code ms' + usual = filter (\c -> toLower c /= c && c `notElem` unusual) [minBound..maxBound] + + typ = ["foldMapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE foldMapping #-}" + ,"foldMapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = mapping c ++ repeat '\0' + printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (toLower c)) diff --git a/scripts/CaseMapping.hs b/scripts/CaseMapping.hs --- a/scripts/CaseMapping.hs +++ b/scripts/CaseMapping.hs @@ -22,14 +22,17 @@ let comments = map ("--" ++) $ take 2 (cfComments cfs) ++ take 2 (scComments scs) mapM_ (hPutStrLn h) $ - ["{-# LANGUAGE Rank2Types #-}" - ,"-- AUTOMATICALLY GENERATED - DO NOT EDIT" + ["-- AUTOMATICALLY GENERATED - DO NOT EDIT" ,"-- Generated by scripts/CaseMapping.hs"] ++ comments ++ ["" + ,"{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-}" + ,"{-# OPTIONS_GHC -Wno-partial-type-signatures #-}" ,"module Data.Text.Internal.Fusion.CaseMapping where" - ,"import Data.Char" - ,"import Data.Text.Internal.Fusion.Types" + ,"import GHC.Int" + ,"import GHC.Exts" + ,"unI64 :: Int64 -> _ {- unboxed Int64 -}" + ,"unI64 (I64# n) = n" ,""] mapM_ (hPutStrLn h) (mapSC "upper" upper toUpper scs) mapM_ (hPutStrLn h) (mapSC "lower" lower toLower scs) diff --git a/scripts/SpecialCasing.hs b/scripts/SpecialCasing.hs --- a/scripts/SpecialCasing.hs +++ b/scripts/SpecialCasing.hs @@ -11,6 +11,7 @@ ) where import Arsec +import Data.Bits data SpecialCasing = SC { scComments :: [Comment], scCasing :: [Case] } deriving (Show) @@ -40,17 +41,23 @@ mapSC :: String -> (Case -> String) -> (Char -> Char) -> SpecialCasing -> [String] mapSC which access twiddle (SC _ ms) = - typ ++ (map nice . filter p $ ms) ++ [last] + typ ++ map printUnusual ms' ++ map printUsual usual ++ [last] where - typ = [which ++ "Mapping :: forall s. Char -> s -> Step (CC s) Char" - ,"{-# NOINLINE " ++ which ++ "Mapping #-}"] - last = which ++ "Mapping c s = Yield (to" ++ ucFirst which ++ " c) (CC s '\\0' '\\0')" - nice c = "-- " ++ name c ++ "\n" ++ - which ++ "Mapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")" - where [x,y,z] = (map showC . take 3) (access c ++ repeat '\0') + ms' = filter p ms p c = [k] /= a && a /= [twiddle k] && null (conditions c) where a = access c k = code c + unusual = map code ms' + usual = filter (\c -> twiddle c /= c && c `notElem` unusual) [minBound..maxBound] + + typ = [which ++ "Mapping :: Char# -> _ {- unboxed Int64 -}" + ,"{-# NOINLINE " ++ which ++ "Mapping #-}" + ,which ++ "Mapping = \\case"] + last = " _ -> unI64 0" + printUnusual c = " -- " ++ name c ++ "\n" ++ + " " ++ showC (code c) ++ "# -> unI64 " ++ show (ord x + (ord y `shiftL` 21) + (ord z `shiftL` 42)) + where x:y:z:_ = access c ++ repeat '\0' + printUsual c = " " ++ showC c ++ "# -> unI64 " ++ show (ord (twiddle c)) ucFirst (c:cs) = toUpper c : cs ucFirst [] = [] diff --git a/simdutf/LICENSE-APACHE b/simdutf/LICENSE-APACHE new file mode 100644 --- /dev/null +++ b/simdutf/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 The simdutf authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/simdutf/LICENSE-MIT b/simdutf/LICENSE-MIT new file mode 100644 --- /dev/null +++ b/simdutf/LICENSE-MIT @@ -0,0 +1,18 @@ +Copyright 2021 The simdutf authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/simdutf/simdutf.cpp b/simdutf/simdutf.cpp new file mode 100644 --- /dev/null +++ b/simdutf/simdutf.cpp @@ -0,0 +1,14204 @@ +/* auto-generated on 2021-07-29 10:43:28 -0400. Do not edit! */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf.cpp +/* begin file src/simdutf.cpp */ +#include "simdutf.h" +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=implementation.cpp +/* begin file src/implementation.cpp */ +#include +#include +#include + +// Useful for debugging purposes +namespace simdutf { +namespace { + +template +std::string toBinaryString(T b) { + std::string binary = ""; + T mask = T(1) << (sizeof(T) * CHAR_BIT - 1); + while (mask > 0) { + binary += ((b & mask) == 0) ? '0' : '1'; + mask >>= 1; + } + return binary; +} +} +} + +// Implementations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64.h +/* begin file src/simdutf/arm64.h */ +#ifndef SIMDUTF_ARM64_H +#define SIMDUTF_ARM64_H + +#ifdef SIMDUTF_FALLBACK_H +#error "arm64.h must be included before fallback.h" +#endif + + +#ifndef SIMDUTF_IMPLEMENTATION_ARM64 +#define SIMDUTF_IMPLEMENTATION_ARM64 (SIMDUTF_IS_ARM64) +#endif +#define SIMDUTF_CAN_ALWAYS_RUN_ARM64 SIMDUTF_IMPLEMENTATION_ARM64 && SIMDUTF_IS_ARM64 + + + +#if SIMDUTF_IMPLEMENTATION_ARM64 + +namespace simdutf { +/** + * Implementation for NEON (ARMv8). + */ +namespace arm64 { +} // namespace arm64 +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/implementation.h +/* begin file src/simdutf/arm64/implementation.h */ +#ifndef SIMDUTF_ARM64_IMPLEMENTATION_H +#define SIMDUTF_ARM64_IMPLEMENTATION_H + + +namespace simdutf { +namespace arm64 { + +namespace { +using namespace simdutf; +} + +class implementation final : public simdutf::implementation { +public: + simdutf_really_inline implementation() : simdutf::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} + simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; + simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t length) const noexcept; + simdutf_warn_unused size_t count_utf8(const char * buf, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept; +}; + +} // namespace arm64 +} // namespace simdutf + +#endif // SIMDUTF_ARM64_IMPLEMENTATION_H +/* end file src/simdutf/arm64/implementation.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/begin.h +/* begin file src/simdutf/arm64/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "arm64" +// #define SIMDUTF_IMPLEMENTATION arm64 +/* end file src/simdutf/arm64/begin.h */ + +// Declarations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/intrinsics.h +/* begin file src/simdutf/arm64/intrinsics.h */ +#ifndef SIMDUTF_ARM64_INTRINSICS_H +#define SIMDUTF_ARM64_INTRINSICS_H + + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +#endif // SIMDUTF_ARM64_INTRINSICS_H +/* end file src/simdutf/arm64/intrinsics.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/bitmanipulation.h +/* begin file src/simdutf/arm64/bitmanipulation.h */ +#ifndef SIMDUTF_ARM64_BITMANIPULATION_H +#define SIMDUTF_ARM64_BITMANIPULATION_H + +namespace simdutf { +namespace arm64 { +namespace { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +NO_SANITIZE_UNDEFINED +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num-1); +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int leading_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDUTF_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int count_ones(uint64_t input_num) { + return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +} + +simdutf_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + *result = value1 + value2; + return *result < value1; +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif +} + +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf + +#endif // SIMDUTF_ARM64_BITMANIPULATION_H +/* end file src/simdutf/arm64/bitmanipulation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/bitmask.h +/* begin file src/simdutf/arm64/bitmask.h */ +#ifndef SIMDUTF_ARM64_BITMASK_H +#define SIMDUTF_ARM64_BITMASK_H + +namespace simdutf { +namespace arm64 { +namespace { + +// +// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. +// +// For example, prefix_xor(00100100) == 00011100 +// +simdutf_really_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; +} + +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf + +#endif +/* end file src/simdutf/arm64/bitmask.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/simd.h +/* begin file src/simdutf/arm64/simd.h */ +#ifndef SIMDUTF_ARM64_SIMD_H +#define SIMDUTF_ARM64_SIMD_H + +#include + + +namespace simdutf { +namespace arm64 { +namespace { +namespace simd { + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +namespace { +// Start of private section with Visual Studio workaround + + +/** + * make_uint8x16_t initializes a SIMD register (uint8x16_t). + * This is needed because, incredibly, the syntax uint8x16_t x = {1,2,3...} + * is not recognized under Visual Studio! This is a workaround. + * Using a std::initializer_list as a parameter resulted in + * inefficient code. With the current approach, if the parameters are + * compile-time constants, + * GNU GCC compiles it to ldr, the same as uint8x16_t x = {1,2,3...}. + * You should not use this function except for compile-time constants: + * it is not efficient. + */ +simdutf_really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, + uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8, + uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12, + uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) { + // Doing a load like so end ups generating worse code. + // uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, + // x9, x10,x11,x12,x13,x14,x15,x16}; + // return vld1q_u8(array); + uint8x16_t x{}; + // incredibly, Visual Studio does not allow x[0] = x1 + x = vsetq_lane_u8(x1, x, 0); + x = vsetq_lane_u8(x2, x, 1); + x = vsetq_lane_u8(x3, x, 2); + x = vsetq_lane_u8(x4, x, 3); + x = vsetq_lane_u8(x5, x, 4); + x = vsetq_lane_u8(x6, x, 5); + x = vsetq_lane_u8(x7, x, 6); + x = vsetq_lane_u8(x8, x, 7); + x = vsetq_lane_u8(x9, x, 8); + x = vsetq_lane_u8(x10, x, 9); + x = vsetq_lane_u8(x11, x, 10); + x = vsetq_lane_u8(x12, x, 11); + x = vsetq_lane_u8(x13, x, 12); + x = vsetq_lane_u8(x14, x, 13); + x = vsetq_lane_u8(x15, x, 14); + x = vsetq_lane_u8(x16, x, 15); + return x; +} + +// We have to do the same work for make_int8x16_t +simdutf_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, + int8_t x5, int8_t x6, int8_t x7, int8_t x8, + int8_t x9, int8_t x10, int8_t x11, int8_t x12, + int8_t x13, int8_t x14, int8_t x15, int8_t x16) { + // Doing a load like so end ups generating worse code. + // int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, + // x9, x10,x11,x12,x13,x14,x15,x16}; + // return vld1q_s8(array); + int8x16_t x{}; + // incredibly, Visual Studio does not allow x[0] = x1 + x = vsetq_lane_s8(x1, x, 0); + x = vsetq_lane_s8(x2, x, 1); + x = vsetq_lane_s8(x3, x, 2); + x = vsetq_lane_s8(x4, x, 3); + x = vsetq_lane_s8(x5, x, 4); + x = vsetq_lane_s8(x6, x, 5); + x = vsetq_lane_s8(x7, x, 6); + x = vsetq_lane_s8(x8, x, 7); + x = vsetq_lane_s8(x9, x, 8); + x = vsetq_lane_s8(x10, x, 9); + x = vsetq_lane_s8(x11, x, 10); + x = vsetq_lane_s8(x12, x, 11); + x = vsetq_lane_s8(x13, x, 12); + x = vsetq_lane_s8(x14, x, 13); + x = vsetq_lane_s8(x15, x, 14); + x = vsetq_lane_s8(x16, x, 15); + return x; +} + +simdutf_really_inline uint16x8_t make_uint16x8_t(uint16_t x1, uint16_t x2, uint16_t x3, uint16_t x4, + uint16_t x5, uint16_t x6, uint16_t x7, uint16_t x8) { + uint16x8_t x{}; + x = vsetq_lane_u16(x1, x, 0); + x = vsetq_lane_u16(x2, x, 1); + x = vsetq_lane_u16(x3, x, 2); + x = vsetq_lane_u16(x4, x, 3); + x = vsetq_lane_u16(x5, x, 4); + x = vsetq_lane_u16(x6, x, 5); + x = vsetq_lane_u16(x7, x, 6); + x = vsetq_lane_u16(x8, x, 7);; + return x; +} + +simdutf_really_inline int16x8_t make_int16x8_t(int16_t x1, int16_t x2, int16_t x3, int16_t x4, + int16_t x5, int16_t x6, int16_t x7, int16_t x8) { + uint16x8_t x{}; + x = vsetq_lane_s16(x1, x, 0); + x = vsetq_lane_s16(x2, x, 1); + x = vsetq_lane_s16(x3, x, 2); + x = vsetq_lane_s16(x4, x, 3); + x = vsetq_lane_s16(x5, x, 4); + x = vsetq_lane_s16(x6, x, 5); + x = vsetq_lane_s16(x7, x, 6); + x = vsetq_lane_s16(x8, x, 7);; + return x; +} + + +// End of private section with Visual Studio workaround +} // namespace +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO + + + template + struct simd8; + + // + // Base class of simd8 and simd8, both of which use uint8x16_t internally. + // + template> + struct base_u8 { + uint8x16_t value; + static const int SIZE = sizeof(value); + + // Conversion from/to SIMD register + simdutf_really_inline base_u8(const uint8x16_t _value) : value(_value) {} + simdutf_really_inline operator const uint8x16_t&() const { return this->value; } + simdutf_really_inline operator uint8x16_t&() { return this->value; } + simdutf_really_inline T first() const { return vgetq_lane_u8(*this,0); } + simdutf_really_inline T last() const { return vgetq_lane_u8(*this,15); } + + // Bit operations + simdutf_really_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } + simdutf_really_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } + simdutf_really_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } + simdutf_really_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } + simdutf_really_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } + simdutf_really_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } + + simdutf_really_inline Mask operator==(const simd8 other) const { return vceqq_u8(*this, other); } + + template + simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { + return vextq_u8(prev_chunk, *this, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8: base_u8 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + static simdutf_really_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } + + simdutf_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + // False constructor + simdutf_really_inline simd8() : simd8(vdupq_n_u8(0)) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {} + simdutf_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } + + // We return uint32_t instead of uint16_t because that seems to be more efficient for most + // purposes (cutting it down to uint16_t costs performance in some compilers). + simdutf_really_inline uint32_t to_bitmask() const { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint8x16_t bit_mask = make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); +#else + const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; +#endif + auto minput = *this & bit_mask; + uint8x16_t tmp = vpaddq_u8(minput, minput); + tmp = vpaddq_u8(tmp, tmp); + tmp = vpaddq_u8(tmp, tmp); + return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); + } + simdutf_really_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdutf_really_inline bool none() const { return vmaxvq_u8(*this) == 0; } + simdutf_really_inline bool all() const { return vminvq_u8(*this) == 0xFF; } + + + }; + + // Unsigned bytes + template<> + struct simd8: base_u8 { + static simdutf_really_inline simd8 splat(uint8_t _value) { return vmovq_n_u8(_value); } + static simdutf_really_inline simd8 zero() { return vdupq_n_u8(0); } + static simdutf_really_inline simd8 load(const uint8_t* values) { return vld1q_u8(values); } + simdutf_really_inline simd8(const simd8& value) = default; + simdutf_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Splat constructor + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Member-by-member initialization +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + simdutf_really_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(make_uint8x16_t( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + )) {} +#else + simdutf_really_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(uint8x16_t{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} +#endif + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + // Store to array + simdutf_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } + + // Saturated math + simdutf_really_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } + simdutf_really_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } + simdutf_really_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } + simdutf_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdutf_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + + // Order-specific operations + simdutf_really_inline uint8_t max_val() const { return vmaxvq_u8(*this); } + simdutf_really_inline uint8_t min_val() const { return vminvq_u8(*this); } + simdutf_really_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } + simdutf_really_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } + simdutf_really_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } + simdutf_really_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } + // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. + simdutf_really_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } + // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. + simdutf_really_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } + + // Bit-specific operations + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } + simdutf_really_inline bool is_ascii() const { return this->max_val() < 0b10000000u; } + + simdutf_really_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdutf_really_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } + template + simdutf_really_inline simd8 shr() const { return vshrq_n_u8(*this, N); } + template + simdutf_really_inline simd8 shl() const { return vshlq_n_u8(*this, N); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { + return lookup_table.apply_lookup_16_to(*this); + } + + + template + simdutf_really_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + + template + simdutf_really_inline simd8 apply_lookup_16_to(const simd8 original) const { + return vqtbl1q_u8(*this, simd8(original)); + } + }; + + // Signed bytes + template<> + struct simd8 { + int8x16_t value; + + static simdutf_really_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } + static simdutf_really_inline simd8 zero() { return vdupq_n_s8(0); } + static simdutf_really_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } + simdutf_really_inline void store_ascii_as_utf16(char16_t * p) const { + vst1q_u16(reinterpret_cast(p), vmovl_u8(vget_low_u8 (vreinterpretq_u8_s8(this->value)))); + vst1q_u16(reinterpret_cast(p + 8), vmovl_high_u8(vreinterpretq_u8_s8(this->value))); + } + // Conversion from/to SIMD register + simdutf_really_inline simd8(const int8x16_t _value) : value{_value} {} + simdutf_really_inline operator const int8x16_t&() const { return this->value; } + simdutf_really_inline operator const uint8x16_t() const { return vreinterpretq_u8_s8(this->value); } + simdutf_really_inline operator int8x16_t&() { return this->value; } + + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + simdutf_really_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(make_int8x16_t( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + )) {} +#else + simdutf_really_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(int8x16_t{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} +#endif + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + // Store to array + simdutf_really_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, value); } + // Explicit conversion to/from unsigned + // + // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type. + // In theory, we could check this occurence with std::same_as and std::enabled_if but it is C++14 + // and relatively ugly and hard to read. +#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO + simdutf_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} +#endif + simdutf_really_inline operator simd8() const { return vreinterpretq_u8_s8(this->value); } + + simdutf_really_inline simd8 operator|(const simd8 other) const { return vorrq_s8(value, other.value); } + simdutf_really_inline simd8 operator&(const simd8 other) const { return vandq_s8(value, other.value); } + simdutf_really_inline simd8 operator^(const simd8 other) const { return veorq_s8(value, other.value); } + simdutf_really_inline simd8 bit_andnot(const simd8 other) const { return vbicq_s8(value, other.value); } + + // Math + simdutf_really_inline simd8 operator+(const simd8 other) const { return vaddq_s8(value, other.value); } + simdutf_really_inline simd8 operator-(const simd8 other) const { return vsubq_s8(value, other.value); } + simdutf_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdutf_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + + simdutf_really_inline int8_t max_val() const { return vmaxvq_s8(value); } + simdutf_really_inline int8_t min_val() const { return vminvq_s8(value); } + simdutf_really_inline bool is_ascii() const { return this->min_val() >= 0; } + + // Order-sensitive comparisons + simdutf_really_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(value, other.value); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return vminq_s8(value, other.value); } + simdutf_really_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(value, other.value); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return vcltq_s8(value, other.value); } + simdutf_really_inline simd8 operator==(const simd8 other) const { return vceqq_s8(value, other.value); } + + template + simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { + return vextq_s8(prev_chunk, *this, 16 - N); + } + + // Perform a lookup assuming no value is larger than 16 + template + simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { + return lookup_table.apply_lookup_16_to(*this); + } + template + simdutf_really_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + + template + simdutf_really_inline simd8 apply_lookup_16_to(const simd8 original) { + return vqtbl1q_s8(*this, simd8(original)); + } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8::load(ptr), simd8::load(ptr+sizeof(simd8)/sizeof(T)), simd8::load(ptr+2*sizeof(simd8)/sizeof(T)), simd8::load(ptr+3*sizeof(simd8)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd8)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd8)*1/sizeof(T)); + this->chunks[2].store(ptr+sizeof(simd8)*2/sizeof(T)); + this->chunks[3].store(ptr+sizeof(simd8)*3/sizeof(T)); + } + + simdutf_really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdutf_really_inline bool is_ascii() const { + return reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd8)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd8)*1); + this->chunks[2].store_ascii_as_utf16(ptr+sizeof(simd8)*2); + this->chunks[3].store_ascii_as_utf16(ptr+sizeof(simd8)*3); + } + + simdutf_really_inline uint64_t to_bitmask() const { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint8x16_t bit_mask = make_uint8x16_t( + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + ); +#else + const uint8x16_t bit_mask = { + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + }; +#endif + // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one. + uint8x16_t sum0 = vpaddq_u8(vandq_u8(uint8x16_t(this->chunks[0]), bit_mask), vandq_u8(uint8x16_t(this->chunks[1]), bit_mask)); + uint8x16_t sum1 = vpaddq_u8(vandq_u8(uint8x16_t(this->chunks[2]), bit_mask), vandq_u8(uint8x16_t(this->chunks[3]), bit_mask)); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + + return simd8x64( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + return simd8x64( + (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low), + (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low), + (this->chunks[2] > mask_high) | (this->chunks[2] < mask_low), + (this->chunks[3] > mask_high) | (this->chunks[3] < mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] < mask, + this->chunks[1] < mask, + this->chunks[2] < mask, + this->chunks[3] < mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] > mask, + this->chunks[1] > mask, + this->chunks[2] > mask, + this->chunks[3] > mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] >= mask, + this->chunks[1] >= mask, + this->chunks[2] >= mask, + this->chunks[3] >= mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + simd8(uint8x16_t(this->chunks[0])) >= mask, + simd8(uint8x16_t(this->chunks[1])) >= mask, + simd8(uint8x16_t(this->chunks[2])) >= mask, + simd8(uint8x16_t(this->chunks[3])) >= mask + ).to_bitmask(); + } + }; // struct simd8x64 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/simd16-inl.h +/* begin file src/simdutf/arm64/simd16-inl.h */ +template +struct simd16; + + template> + struct base_u16 { + uint16x8_t value; + static const int SIZE = sizeof(value); + + // Conversion from/to SIMD register + simdutf_really_inline base_u16() = default; + simdutf_really_inline base_u16(const uint16x8_t _value) : value(_value) {} + simdutf_really_inline operator const uint16x8_t&() const { return this->value; } + simdutf_really_inline operator uint16x8_t&() { return this->value; } + // Bit operations + simdutf_really_inline simd16 operator|(const simd16 other) const { return vorrq_u16(*this, other); } + simdutf_really_inline simd16 operator&(const simd16 other) const { return vandq_u16(*this, other); } + simdutf_really_inline simd16 operator^(const simd16 other) const { return veorq_u16(*this, other); } + simdutf_really_inline simd16 bit_andnot(const simd16 other) const { return vbicq_u16(*this, other); } + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline simd16& operator|=(const simd16 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } + simdutf_really_inline simd16& operator&=(const simd16 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } + simdutf_really_inline simd16& operator^=(const simd16 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } + + simdutf_really_inline Mask operator==(const simd16 other) const { return vceqq_u16(*this, other); } + + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return vextq_u18(prev_chunk, *this, 8 - N); + } + }; + +template> +struct base16: base_u16 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdutf_really_inline base16() : base_u16() {} + simdutf_really_inline base16(const uint16x8_t _value) : base_u16(_value) {} + template + simdutf_really_inline base16(const Pointer* ptr) : base16(vld1q_u16(ptr)) {} + + simdutf_really_inline Mask operator==(const simd16 other) const { return vceqq_u16(*this, other); } + + static const int SIZE = sizeof(base_u16::value); + + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return vextq_u18(prev_chunk, *this, 8 - N); + } +}; + +// SIMD byte mask type (returned by things like eq and gt) +template<> +struct simd16: base16 { + static simdutf_really_inline simd16 splat(bool _value) { return vmovq_n_u16(uint16_t(-(!!_value))); } + + simdutf_really_inline simd16() : base16() {} + simdutf_really_inline simd16(const uint16x8_t _value) : base16(_value) {} + // Splat constructor + simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} + +}; + +template +struct base16_numeric: base16 { + static simdutf_really_inline simd16 splat(T _value) { return vmovq_n_u16(_value); } + static simdutf_really_inline simd16 zero() { return vdupq_n_u16(0); } + static simdutf_really_inline simd16 load(const T values[8]) { + return vld1q_u16(reinterpret_cast(values)); + } + + simdutf_really_inline base16_numeric() : base16() {} + simdutf_really_inline base16_numeric(const uint16x8_t _value) : base16(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[8]) const { return vst1q_u16(dst, *this); } + + // Override to distinguish from bool version + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd16 operator+(const simd16 other) const { return vaddq_u8(*this, other); } + simdutf_really_inline simd16 operator-(const simd16 other) const { return vsubq_u8(*this, other); } + simdutf_really_inline simd16& operator+=(const simd16 other) { *this = *this + other; return *static_cast*>(this); } + simdutf_really_inline simd16& operator-=(const simd16 other) { *this = *this - other; return *static_cast*>(this); } +}; + +// Signed words +template<> +struct simd16 : base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} +#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO + simdutf_really_inline simd16(const uint16x8_t _value) : base16_numeric(_value) {} +#endif + simdutf_really_inline simd16(const int16x8_t _value) : base16_numeric(vreinterpretq_u16_s16(_value)) {} + + // Splat constructor + simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + simdutf_really_inline operator simd16() const; + simdutf_really_inline operator const uint16x8_t&() const { return this->value; } + simdutf_really_inline operator const int16x8_t() const { return vreinterpretq_s16_u16(this->value); } + + simdutf_really_inline int16_t max_val() const { return vmaxvq_s16(vreinterpretq_s16_u16(this->value)); } + simdutf_really_inline int16_t min_val() const { return vminvq_s16(vreinterpretq_s16_u16(this->value)); } + // Order-sensitive comparisons + simdutf_really_inline simd16 max_val(const simd16 other) const { return vmaxq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return vmaxq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); } + simdutf_really_inline simd16 operator>(const simd16 other) const { return vcgtq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return vcltq_s16(vreinterpretq_s16_u16(this->value), vreinterpretq_s16_u16(other.value)); } +}; + + + + +// Unsigned words +template<> +struct simd16: base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const uint16x8_t _value) : base16_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + + + simdutf_really_inline int16_t max_val() const { return vmaxvq_u16(*this); } + simdutf_really_inline int16_t min_val() const { return vminvq_u16(*this); } + // Saturated math + simdutf_really_inline simd16 saturating_add(const simd16 other) const { return vqaddq_u16(*this, other); } + simdutf_really_inline simd16 saturating_sub(const simd16 other) const { return vqsubq_u16(*this, other); } + + // Order-specific operations + simdutf_really_inline simd16 max_val(const simd16 other) const { return vmaxq_u16(*this, other); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return vminq_u16(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 gt_bits(const simd16 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 lt_bits(const simd16 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd16 operator<=(const simd16 other) const { return vcleq_u16(*this, other); } + simdutf_really_inline simd16 operator>=(const simd16 other) const { return vcgeq_u16(*this, other); } + simdutf_really_inline simd16 operator>(const simd16 other) const { return vcgtq_u16(*this, other); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return vcltq_u16(*this, other); } + + // Bit-specific operations + simdutf_really_inline simd16 bits_not_set() const { return *this == uint16_t(0); } + template + simdutf_really_inline simd16 shr() const { return simd16(vshrq_n_u16(*this, N)); } + template + simdutf_really_inline simd16 shl() const { return simd16(vshlq_n_u16(*this, N)); } + + // logical operations + simdutf_really_inline simd16 operator|(const simd16 other) const { return vorrq_u16(*this, other); } + simdutf_really_inline simd16 operator&(const simd16 other) const { return vandq_u16(*this, other); } + simdutf_really_inline simd16 operator^(const simd16 other) const { return veorq_u16(*this, other); } + + // Pack with the unsigned saturation two uint16_t words into single uint8_t vector + static simdutf_really_inline simd8 pack(const simd16& v0, const simd16& v1) { + return vqmovn_high_u16(vqmovn_u16(v0), v1); + } +}; +simdutf_really_inline simd16::operator simd16() const { return this->value; } + + + template + struct simd16x32 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); + static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); + const simd16 chunks[NUM_CHUNKS]; + + simd16x32(const simd16x32& o) = delete; // no copy allowed + simd16x32& operator=(const simd16 other) = delete; // no assignment allowed + simd16x32() = delete; // no default constructor allowed + + simdutf_really_inline simd16x32(const simd16 chunk0, const simd16 chunk1, const simd16 chunk2, const simd16 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16::load(ptr), simd16::load(ptr+sizeof(simd16)/sizeof(T)), simd16::load(ptr+2*sizeof(simd16)/sizeof(T)), simd16::load(ptr+3*sizeof(simd16)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd16)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd16)*1/sizeof(T)); + this->chunks[2].store(ptr+sizeof(simd16)*2/sizeof(T)); + this->chunks[3].store(ptr+sizeof(simd16)*3/sizeof(T)); + } + + simdutf_really_inline simd16 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdutf_really_inline bool is_ascii() const { + return reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd16)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd16)*1); + this->chunks[2].store_ascii_as_utf16(ptr+sizeof(simd16)*2); + this->chunks[3].store_ascii_as_utf16(ptr+sizeof(simd16)*3); + } + + simdutf_really_inline uint64_t to_bitmask() const { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint8x16_t bit_mask = make_uint8x16_t( + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + ); +#else + const uint8x16_t bit_mask = { + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + }; +#endif + // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one. + uint8x16_t sum0 = vpaddq_u8(vreinterpretq_u8_u16(this->chunks[0] & vreinterpretq_u16_u8(bit_mask)), vreinterpretq_u8_u16(this->chunks[1] & vreinterpretq_u16_u8(bit_mask))); + uint8x16_t sum1 = vpaddq_u8(vreinterpretq_u8_u16(this->chunks[2] & vreinterpretq_u16_u8(bit_mask)), vreinterpretq_u8_u16(this->chunks[3] & vreinterpretq_u16_u8(bit_mask))); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(low); + const simd16 mask_high = simd16::splat(high); + + return simd16x32( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(low); + const simd16 mask_high = simd16::splat(high); + return simd16x32( + (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low), + (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low), + (this->chunks[2] > mask_high) | (this->chunks[2] < mask_low), + (this->chunks[3] > mask_high) | (this->chunks[3] < mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] < mask, + this->chunks[1] < mask, + this->chunks[2] < mask, + this->chunks[3] < mask + ).to_bitmask(); + } + + }; // struct simd16x32 + template<> + simdutf_really_inline uint64_t simd16x32::not_in_range(const uint16_t low, const uint16_t high) const { + const simd16 mask_low = simd16::splat(low); + const simd16 mask_high = simd16::splat(high); + simd16x32 x( + simd16((this->chunks[0] > mask_high) | (this->chunks[0] < mask_low)), + simd16((this->chunks[1] > mask_high) | (this->chunks[1] < mask_low)), + simd16((this->chunks[2] > mask_high) | (this->chunks[2] < mask_low)), + simd16((this->chunks[3] > mask_high) | (this->chunks[3] < mask_low)) + ); + return x.to_bitmask(); + } +/* end file src/simdutf/arm64/simd16-inl.h */ +} // namespace simd +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf + +#endif // SIMDUTF_ARM64_SIMD_H +/* end file src/simdutf/arm64/simd.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/end.h +/* begin file src/simdutf/arm64/end.h */ +/* end file src/simdutf/arm64/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_ARM64 + +#endif // SIMDUTF_ARM64_H +/* end file src/simdutf/arm64.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell.h +/* begin file src/simdutf/haswell.h */ +#ifndef SIMDUTF_HASWELL_H +#define SIMDUTF_HASWELL_H + +#ifdef SIMDUTF_WESTMERE_H +#error "haswell.h must be included before westmere.h" +#endif +#ifdef SIMDUTF_FALLBACK_H +#error "haswell.h must be included before fallback.h" +#endif + + +// Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected +// at runtime. +#ifndef SIMDUTF_IMPLEMENTATION_HASWELL +// +// You do not want to restrict it like so: SIMDUTF_IS_X86_64 && __AVX2__ +// because we want to rely on *runtime dispatch*. +// +#define SIMDUTF_IMPLEMENTATION_HASWELL (SIMDUTF_IS_X86_64) +#endif +// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see +// https://github.com/simdutf/simdutf/issues/1247 +#define SIMDUTF_CAN_ALWAYS_RUN_HASWELL ((SIMDUTF_IMPLEMENTATION_HASWELL) && (SIMDUTF_IS_X86_64) && (__AVX2__)) + +#if SIMDUTF_IMPLEMENTATION_HASWELL + +#define SIMDUTF_TARGET_HASWELL SIMDUTF_TARGET_REGION("avx2,bmi,pclmul,lzcnt") + +namespace simdutf { +/** + * Implementation for Haswell (Intel AVX2). + */ +namespace haswell { +} // namespace haswell +} // namespace simdutf + +// +// These two need to be included outside SIMDUTF_TARGET_REGION +// +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/implementation.h +/* begin file src/simdutf/haswell/implementation.h */ +#ifndef SIMDUTF_HASWELL_IMPLEMENTATION_H +#define SIMDUTF_HASWELL_IMPLEMENTATION_H + + +// The constructor may be executed on any host, so we take care not to use SIMDUTF_TARGET_REGION +namespace simdutf { +namespace haswell { + +using namespace simdutf; + +class implementation final : public simdutf::implementation { +public: + simdutf_really_inline implementation() : simdutf::implementation( + "haswell", + "Intel/AMD AVX2", + internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 + ) {} + simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; + simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t length) const noexcept; + simdutf_warn_unused size_t count_utf8(const char * buf, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept; +}; + +} // namespace haswell +} // namespace simdutf + +#endif // SIMDUTF_HASWELL_IMPLEMENTATION_H +/* end file src/simdutf/haswell/implementation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/intrinsics.h +/* begin file src/simdutf/haswell/intrinsics.h */ +#ifndef SIMDUTF_HASWELL_INTRINSICS_H +#define SIMDUTF_HASWELL_INTRINSICS_H + + +#ifdef SIMDUTF_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDUTF_VISUAL_STUDIO + +#ifdef SIMDUTF_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdutf, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 +// unfortunately, we may not get _blsr_u64, but, thankfully, clang +// has it as a macro. +#ifndef _blsr_u64 +// we roll our own +SIMDUTF_TARGET_HASWELL +static simdutf_really_inline uint64_t _blsr_u64(uint64_t n) { + return (n - 1) & n; +} +SIMDUTF_UNTARGET_REGION +#endif // _blsr_u64 +#endif // SIMDUTF_CLANG_VISUAL_STUDIO + +#endif // SIMDUTF_HASWELL_INTRINSICS_H +/* end file src/simdutf/haswell/intrinsics.h */ + +// +// The rest need to be inside the region +// +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/begin.h +/* begin file src/simdutf/haswell/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "haswell" +// #define SIMDUTF_IMPLEMENTATION haswell +SIMDUTF_TARGET_HASWELL +/* end file src/simdutf/haswell/begin.h */ + +// Declarations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/bitmanipulation.h +/* begin file src/simdutf/haswell/bitmanipulation.h */ +#ifndef SIMDUTF_HASWELL_BITMANIPULATION_H +#define SIMDUTF_HASWELL_BITMANIPULATION_H + +namespace simdutf { +namespace haswell { +namespace { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +NO_SANITIZE_UNDEFINED +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int leading_zeroes(uint64_t input_num) { + return int(_lzcnt_u64(input_num)); +} + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +simdutf_really_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num);// Visual Studio wants two underscores +} +#else +simdutf_really_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); +} +#endif + +simdutf_really_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t *result) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif +} + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf + +#endif // SIMDUTF_HASWELL_BITMANIPULATION_H +/* end file src/simdutf/haswell/bitmanipulation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/bitmask.h +/* begin file src/simdutf/haswell/bitmask.h */ +#ifndef SIMDUTF_HASWELL_BITMASK_H +#define SIMDUTF_HASWELL_BITMASK_H + +namespace simdutf { +namespace haswell { +namespace { + +// +// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. +// +// For example, prefix_xor(00100100) == 00011100 +// +simdutf_really_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); +} + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf + +#endif // SIMDUTF_HASWELL_BITMASK_H +/* end file src/simdutf/haswell/bitmask.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/simd.h +/* begin file src/simdutf/haswell/simd.h */ +#ifndef SIMDUTF_HASWELL_SIMD_H +#define SIMDUTF_HASWELL_SIMD_H + + +namespace simdutf { +namespace haswell { +namespace { +namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdutf_really_inline base() : value{__m256i()} {} + + // Conversion from SIMD register + simdutf_really_inline base(const __m256i _value) : value(_value) {} + // Conversion to SIMD register + simdutf_really_inline operator const __m256i&() const { return this->value; } + simdutf_really_inline operator __m256i&() { return this->value; } + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), _mm256_cvtepu8_epi16(_mm256_castsi256_si128(*this))); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 16), _mm256_cvtepu8_epi16(_mm256_extractf128_si256(*this,1))); + } + // Bit operations + simdutf_really_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdutf_really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdutf_really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdutf_really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdutf_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdutf_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdutf_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8: base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdutf_really_inline base8() : base>() {} + simdutf_really_inline base8(const __m256i _value) : base>(_value) {} + simdutf_really_inline T first() const { return _mm256_extract_epi8(*this,0); } + simdutf_really_inline T last() const { return _mm256_extract_epi8(*this,31); } + simdutf_really_inline Mask operator==(const simd8 other) const { return _mm256_cmpeq_epi8(*this, other); } + + static const int SIZE = sizeof(base::value); + + template + simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { + return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8: base8 { + static simdutf_really_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + + simdutf_really_inline simd8() : base8() {} + simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} + + simdutf_really_inline uint32_t to_bitmask() const { return uint32_t(_mm256_movemask_epi8(*this)); } + simdutf_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdutf_really_inline bool none() const { return _mm256_testz_si256(*this, *this); } + simdutf_really_inline bool all() const { return static_cast(_mm256_movemask_epi8(*this)) == 0xFFFFFFFF; } + simdutf_really_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric: base8 { + static simdutf_really_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdutf_really_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdutf_really_inline simd8 load(const T values[32]) { + return _mm256_loadu_si256(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdutf_really_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + simdutf_really_inline base8_numeric() : base8() {} + simdutf_really_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdutf_really_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdutf_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdutf_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm256_shuffle_epi8(lookup_table, *this); + } + + template + simdutf_really_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + simdutf_really_inline operator simd8() const; + // Member-by-member initialization + simdutf_really_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + simdutf_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + // Order-sensitive comparisons + simdutf_really_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdutf_really_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8: base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + + // Saturated math + simdutf_really_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdutf_really_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + + // Order-specific operations + simdutf_really_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdutf_really_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdutf_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdutf_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdutf_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdutf_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdutf_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdutf_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdutf_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdutf_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdutf_really_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdutf_really_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdutf_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } + }; + simdutf_really_inline simd8::operator simd8() const { return this->value; } + + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8::load(ptr), simd8::load(ptr+sizeof(simd8)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd8)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd8)*1/sizeof(T)); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + + simdutf_really_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd8)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd8)); + } + + simdutf_really_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask, + this->chunks[1] | mask + ); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t eq(const simd8x64 &other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + + return simd8x64( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + return simd8x64( + (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low), + (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] < mask, + this->chunks[1] < mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t gt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] > mask, + this->chunks[1] > mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] >= mask, + this->chunks[1] >= mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + (simd8(__m256i(this->chunks[0])) >= mask), + (simd8(__m256i(this->chunks[1])) >= mask) + ).to_bitmask(); + } + }; // struct simd8x64 + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/simd16-inl.h +/* begin file src/simdutf/haswell/simd16-inl.h */ +#ifdef __GNUC__ +#if __GNUC__ < 8 +#define _mm256_set_m128i(xmm1, xmm2) _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), _mm256_castsi128_si256(xmm2), 2) +#define _mm256_setr_m128i(xmm2, xmm1) _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), _mm256_castsi128_si256(xmm2), 2) +#endif +#endif + +template +struct simd16; + +template> +struct base16: base> { + using bitmask_type = uint32_t; + + simdutf_really_inline base16() : base>() {} + simdutf_really_inline base16(const __m256i _value) : base>(_value) {} + template + simdutf_really_inline base16(const Pointer* ptr) : base16(_mm256_loadu_si256(reinterpret_cast(ptr))) {} + + simdutf_really_inline Mask operator==(const simd16 other) const { return _mm256_cmpeq_epi16(*this, other); } + + /// the size of vector in bytes + static const int SIZE = sizeof(base>::value); + + /// the number of elements of type T a vector can hold + static const int ELEMENTS = SIZE / sizeof(T); + + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return _mm256_alignr_epi8(*this, prev_chunk, 16 - N); + } +}; + +// SIMD byte mask type (returned by things like eq and gt) +template<> +struct simd16: base16 { + static simdutf_really_inline simd16 splat(bool _value) { return _mm256_set1_epi16(uint16_t(-(!!_value))); } + + simdutf_really_inline simd16() : base16() {} + simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} + // Splat constructor + simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} + + simdutf_really_inline bitmask_type to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdutf_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdutf_really_inline simd16 operator~() const { return *this ^ true; } +}; + +template +struct base16_numeric: base16 { + static simdutf_really_inline simd16 splat(T _value) { return _mm256_set1_epi16(_value); } + static simdutf_really_inline simd16 zero() { return _mm256_setzero_si256(); } + static simdutf_really_inline simd16 load(const T values[8]) { + return _mm256_loadu_si256(reinterpret_cast(values)); + } + + simdutf_really_inline base16_numeric() : base16() {} + simdutf_really_inline base16_numeric(const __m256i _value) : base16(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[8]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } + + // Override to distinguish from bool version + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd16 operator+(const simd16 other) const { return _mm256_add_epi16(*this, other); } + simdutf_really_inline simd16 operator-(const simd16 other) const { return _mm256_sub_epi16(*this, other); } + simdutf_really_inline simd16& operator+=(const simd16 other) { *this = *this + other; return *static_cast*>(this); } + simdutf_really_inline simd16& operator-=(const simd16 other) { *this = *this - other; return *static_cast*>(this); } +}; + +// Signed words +template<> +struct simd16 : base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m256i _value) : base16_numeric(_value) {} + // Splat constructor + simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + // Order-sensitive comparisons + simdutf_really_inline simd16 max_val(const simd16 other) const { return _mm256_max_epi16(*this, other); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return _mm256_min_epi16(*this, other); } + simdutf_really_inline simd16 operator>(const simd16 other) const { return _mm256_cmpgt_epi16(*this, other); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return _mm256_cmpgt_epi16(other, *this); } +}; + +// Unsigned words +template<> +struct simd16: base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m256i _value) : base16_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + + // Saturated math + simdutf_really_inline simd16 saturating_add(const simd16 other) const { return _mm256_adds_epu16(*this, other); } + simdutf_really_inline simd16 saturating_sub(const simd16 other) const { return _mm256_subs_epu16(*this, other); } + + // Order-specific operations + simdutf_really_inline simd16 max_val(const simd16 other) const { return _mm256_max_epu16(*this, other); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return _mm256_min_epu16(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 gt_bits(const simd16 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 lt_bits(const simd16 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd16 operator<=(const simd16 other) const { return other.max_val(*this) == other; } + simdutf_really_inline simd16 operator>=(const simd16 other) const { return other.min_val(*this) == other; } + simdutf_really_inline simd16 operator>(const simd16 other) const { return this->gt_bits(other).any_bits_set(); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdutf_really_inline simd16 bits_not_set() const { return *this == uint16_t(0); } + simdutf_really_inline simd16 bits_not_set(simd16 bits) const { return (*this & bits).bits_not_set(); } + simdutf_really_inline simd16 any_bits_set() const { return ~this->bits_not_set(); } + simdutf_really_inline simd16 any_bits_set(simd16 bits) const { return ~this->bits_not_set(bits); } + + simdutf_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdutf_really_inline bool bits_not_set_anywhere(simd16 bits) const { return _mm256_testz_si256(*this, bits); } + simdutf_really_inline bool any_bits_set_anywhere(simd16 bits) const { return !bits_not_set_anywhere(bits); } + template + simdutf_really_inline simd16 shr() const { return simd16(_mm256_srli_epi16(*this, N)); } + template + simdutf_really_inline simd16 shl() const { return simd16(_mm256_slli_epi16(*this, N)); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdutf_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 15-N)); } + + // Pack with the unsigned saturation two uint16_t words into single uint8_t vector + static simdutf_really_inline simd8 pack(const simd16& v0, const simd16& v1) { + // Note: the AVX2 variant of pack operates on 128-bit lanes, thus + // we have to shuffle lanes in order to produce bytes in the + // correct order. + + // get the 0th lanes + const __m128i lo_0 = _mm256_extracti128_si256(v0, 0); + const __m128i lo_1 = _mm256_extracti128_si256(v1, 0); + + // get the 1st lanes + const __m128i hi_0 = _mm256_extracti128_si256(v0, 1); + const __m128i hi_1 = _mm256_extracti128_si256(v1, 1); + + // build new vectors (shuffle lanes) + const __m256i t0 = _mm256_set_m128i(lo_1, lo_0); + const __m256i t1 = _mm256_set_m128i(hi_1, hi_0); + + // pack words in linear order from v0 and v1 + return _mm256_packus_epi16(t0, t1); + } +}; + + + template + struct simd16x32 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); + const simd16 chunks[NUM_CHUNKS]; + + simd16x32(const simd16x32& o) = delete; // no copy allowed + simd16x32& operator=(const simd16 other) = delete; // no assignment allowed + simd16x32() = delete; // no default constructor allowed + + simdutf_really_inline simd16x32(const simd16 chunk0, const simd16 chunk1) : chunks{chunk0, chunk1} {} + simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16::load(ptr), simd16::load(ptr+sizeof(simd16)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd16)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd16)*1/sizeof(T)); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + + simdutf_really_inline simd16 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd16)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd16)); + } + + simdutf_really_inline simd16x32 bit_or(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] | mask, + this->chunks[1] | mask + ); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t eq(const simd16x32 &other) const { + return simd16x32( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(low); + const simd16 mask_high = simd16::splat(high); + + return simd16x32( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(static_cast(low-1)); + const simd16 mask_high = simd16::splat(static_cast(high+1)); + return simd16x32( + (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low), + (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] < mask, + this->chunks[1] < mask + ).to_bitmask(); + } + }; // struct simd16x32 +/* end file src/simdutf/haswell/simd16-inl.h */ + +} // namespace simd + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf + +#endif // SIMDUTF_HASWELL_SIMD_H +/* end file src/simdutf/haswell/simd.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/end.h +/* begin file src/simdutf/haswell/end.h */ +SIMDUTF_UNTARGET_REGION +/* end file src/simdutf/haswell/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_HASWELL +#endif // SIMDUTF_HASWELL_COMMON_H +/* end file src/simdutf/haswell.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere.h +/* begin file src/simdutf/westmere.h */ +#ifndef SIMDUTF_WESTMERE_H +#define SIMDUTF_WESTMERE_H + +#ifdef SIMDUTF_FALLBACK_H +#error "westmere.h must be included before fallback.h" +#endif + + +// Default Westmere to on if this is x86-64, unless we'll always select Haswell. +#ifndef SIMDUTF_IMPLEMENTATION_WESTMERE +// +// You do not want to set it to (SIMDUTF_IS_X86_64 && !SIMDUTF_REQUIRES_HASWELL) +// because you want to rely on runtime dispatch! +// +#define SIMDUTF_IMPLEMENTATION_WESTMERE (SIMDUTF_IS_X86_64) +#endif +#define SIMDUTF_CAN_ALWAYS_RUN_WESTMERE (SIMDUTF_IMPLEMENTATION_WESTMERE && SIMDUTF_IS_X86_64 && __SSE4_2__ && __PCLMUL__) + +#if SIMDUTF_IMPLEMENTATION_WESTMERE + +#define SIMDUTF_TARGET_WESTMERE SIMDUTF_TARGET_REGION("sse4.2,pclmul") + +namespace simdutf { +/** + * Implementation for Westmere (Intel SSE4.2). + */ +namespace westmere { +} // namespace westmere +} // namespace simdutf + +// +// These two need to be included outside SIMDUTF_TARGET_REGION +// +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/implementation.h +/* begin file src/simdutf/westmere/implementation.h */ +#ifndef SIMDUTF_WESTMERE_IMPLEMENTATION_H +#define SIMDUTF_WESTMERE_IMPLEMENTATION_H + + +// The constructor may be executed on any host, so we take care not to use SIMDUTF_TARGET_REGION +namespace simdutf { +namespace westmere { + +namespace { +using namespace simdutf; +} + +class implementation final : public simdutf::implementation { +public: + simdutf_really_inline implementation() : simdutf::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} + simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; + simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t length) const noexcept; + simdutf_warn_unused size_t count_utf8(const char * buf, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept; +}; + +} // namespace westmere +} // namespace simdutf + +#endif // SIMDUTF_WESTMERE_IMPLEMENTATION_H +/* end file src/simdutf/westmere/implementation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/intrinsics.h +/* begin file src/simdutf/westmere/intrinsics.h */ +#ifndef SIMDUTF_WESTMERE_INTRINSICS_H +#define SIMDUTF_WESTMERE_INTRINSICS_H + +#ifdef SIMDUTF_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDUTF_VISUAL_STUDIO + + +#ifdef SIMDUTF_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + */ +#include // for _mm_alignr_epi8 +#include // for _mm_clmulepi64_si128 +#endif + + + +#endif // SIMDUTF_WESTMERE_INTRINSICS_H +/* end file src/simdutf/westmere/intrinsics.h */ + +// +// The rest need to be inside the region +// +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/begin.h +/* begin file src/simdutf/westmere/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "westmere" +// #define SIMDUTF_IMPLEMENTATION westmere +SIMDUTF_TARGET_WESTMERE +/* end file src/simdutf/westmere/begin.h */ + +// Declarations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/bitmanipulation.h +/* begin file src/simdutf/westmere/bitmanipulation.h */ +#ifndef SIMDUTF_WESTMERE_BITMANIPULATION_H +#define SIMDUTF_WESTMERE_BITMANIPULATION_H + +namespace simdutf { +namespace westmere { +namespace { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +NO_SANITIZE_UNDEFINED +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num-1); +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int leading_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDUTF_REGULAR_VISUAL_STUDIO +} + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +simdutf_really_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num);// Visual Studio wants two underscores +} +#else +simdutf_really_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); +} +#endif + +simdutf_really_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t *result) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif +} + +} // unnamed namespace +} // namespace westmere +} // namespace simdutf + +#endif // SIMDUTF_WESTMERE_BITMANIPULATION_H +/* end file src/simdutf/westmere/bitmanipulation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/bitmask.h +/* begin file src/simdutf/westmere/bitmask.h */ +#ifndef SIMDUTF_WESTMERE_BITMASK_H +#define SIMDUTF_WESTMERE_BITMASK_H + +namespace simdutf { +namespace westmere { +namespace { + +// +// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. +// +// For example, prefix_xor(00100100) == 00011100 +// +simdutf_really_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processing supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); +} + +} // unnamed namespace +} // namespace westmere +} // namespace simdutf + +#endif // SIMDUTF_WESTMERE_BITMASK_H +/* end file src/simdutf/westmere/bitmask.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/simd.h +/* begin file src/simdutf/westmere/simd.h */ +#ifndef SIMDUTF_WESTMERE_SIMD_H +#define SIMDUTF_WESTMERE_SIMD_H + +namespace simdutf { +namespace westmere { +namespace { +namespace simd { + + template + struct base { + __m128i value; + + // Zero constructor + simdutf_really_inline base() : value{__m128i()} {} + + // Conversion from SIMD register + simdutf_really_inline base(const __m128i _value) : value(_value) {} + // Conversion to SIMD register + simdutf_really_inline operator const __m128i&() const { return this->value; } + simdutf_really_inline operator __m128i&() { return this->value; } + simdutf_really_inline void store_ascii_as_utf16(char16_t * p) const { + _mm_storeu_si128(reinterpret_cast<__m128i *>(p), _mm_cvtepu8_epi16(*this)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(p+8), _mm_cvtepu8_epi16(_mm_srli_si128(*this,8))); + } + // Bit operations + simdutf_really_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdutf_really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdutf_really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdutf_really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdutf_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdutf_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdutf_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8: base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdutf_really_inline T first() const { return _mm_extract_epi8(*this,0); } + simdutf_really_inline T last() const { return _mm_extract_epi8(*this,15); } + simdutf_really_inline base8() : base>() {} + simdutf_really_inline base8(const __m128i _value) : base>(_value) {} + + simdutf_really_inline Mask operator==(const simd8 other) const { return _mm_cmpeq_epi8(*this, other); } + + static const int SIZE = sizeof(base>::value); + + template + simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8: base8 { + static simdutf_really_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + + simdutf_really_inline simd8() : base8() {} + simdutf_really_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} + + simdutf_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdutf_really_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdutf_really_inline bool none() const { return _mm_testz_si128(*this, *this); } + simdutf_really_inline bool all() const { return _mm_movemask_epi8(*this) == 0xFFFF; } + simdutf_really_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric: base8 { + static simdutf_really_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdutf_really_inline simd8 zero() { return _mm_setzero_si128(); } + static simdutf_really_inline simd8 load(const T values[16]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdutf_really_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + simdutf_really_inline base8_numeric() : base8() {} + simdutf_really_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } + + // Override to distinguish from bool version + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdutf_really_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdutf_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdutf_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm_shuffle_epi8(lookup_table, *this); + } + + template + simdutf_really_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + simdutf_really_inline operator simd8() const; + simdutf_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + + // Order-sensitive comparisons + simdutf_really_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdutf_really_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8: base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + // Saturated math + simdutf_really_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdutf_really_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + + // Order-specific operations + simdutf_really_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdutf_really_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdutf_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdutf_really_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdutf_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdutf_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdutf_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdutf_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + + simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdutf_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdutf_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdutf_really_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdutf_really_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdutf_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } + }; + simdutf_really_inline simd8::operator simd8() const { return this->value; } + + // Unsigned bytes + template<> + struct simd8: base { + static simdutf_really_inline simd8 splat(uint16_t _value) { return _mm_set1_epi16(_value); } + static simdutf_really_inline simd8 load(const uint16_t values[8]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + + simdutf_really_inline simd8() : base() {} + simdutf_really_inline simd8(const __m128i _value) : base(_value) {} + // Splat constructor + simdutf_really_inline simd8(uint16_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const uint16_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline simd8( + uint16_t v0, uint16_t v1, uint16_t v2, uint16_t v3, uint16_t v4, uint16_t v5, uint16_t v6, uint16_t v7 + ) : simd8(_mm_setr_epi16( + v0, v1, v2, v3, v4, v5, v6, v7 + )) {} + + // Saturated math + simdutf_really_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu16(*this, other); } + simdutf_really_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu16(*this, other); } + + // Order-specific operations + simdutf_really_inline simd8 max_val(const simd8 other) const { return _mm_max_epu16(*this, other); } + simdutf_really_inline simd8 min_val(const simd8 other) const { return _mm_min_epu16(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdutf_really_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdutf_really_inline simd8 operator==(const simd8 other) const { return _mm_cmpeq_epi16(*this, other); } + simdutf_really_inline simd8 operator&(const simd8 other) const { return _mm_and_si128(*this, other); } + simdutf_really_inline simd8 operator|(const simd8 other) const { return _mm_or_si128(*this, other); } + + // Bit-specific operations + simdutf_really_inline simd8 bits_not_set() const { return *this == uint16_t(0); } + simdutf_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + + simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdutf_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdutf_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + }; + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8::load(ptr), simd8::load(ptr+sizeof(simd8)/sizeof(T)), simd8::load(ptr+2*sizeof(simd8)/sizeof(T)), simd8::load(ptr+3*sizeof(simd8)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd8)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd8)*1/sizeof(T)); + this->chunks[2].store(ptr+sizeof(simd8)*2/sizeof(T)); + this->chunks[3].store(ptr+sizeof(simd8)*3/sizeof(T)); + } + + simdutf_really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd8)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd8)*1); + this->chunks[2].store_ascii_as_utf16(ptr+sizeof(simd8)*2); + this->chunks[3].store_ascii_as_utf16(ptr+sizeof(simd8)*3); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); + uint64_t r1 = this->chunks[1].to_bitmask() ; + uint64_t r2 = this->chunks[2].to_bitmask() ; + uint64_t r3 = this->chunks[3].to_bitmask() ; + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t eq(const simd8x64 &other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + + return simd8x64( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low-1); + const simd8 mask_high = simd8::splat(high+1); + return simd8x64( + (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low), + (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low), + (this->chunks[2] >= mask_high) | (this->chunks[2] <= mask_low), + (this->chunks[3] >= mask_high) | (this->chunks[3] <= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] < mask, + this->chunks[1] < mask, + this->chunks[2] < mask, + this->chunks[3] < mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t gt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] > mask, + this->chunks[1] > mask, + this->chunks[2] > mask, + this->chunks[3] > mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] >= mask, + this->chunks[1] >= mask, + this->chunks[2] >= mask, + this->chunks[3] >= mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + simd8(__m128i(this->chunks[0])) >= mask, + simd8(__m128i(this->chunks[1])) >= mask, + simd8(__m128i(this->chunks[2])) >= mask, + simd8(__m128i(this->chunks[3])) >= mask + ).to_bitmask(); + } + }; // struct simd8x64 + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/simd16-inl.h +/* begin file src/simdutf/westmere/simd16-inl.h */ +template +struct simd16; + +template> +struct base16: base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdutf_really_inline base16() : base>() {} + simdutf_really_inline base16(const __m128i _value) : base>(_value) {} + template + simdutf_really_inline base16(const Pointer* ptr) : base16(_mm_loadu_si128(reinterpret_cast(ptr))) {} + + simdutf_really_inline Mask operator==(const simd16 other) const { return _mm_cmpeq_epi16(*this, other); } + + static const int SIZE = sizeof(base>::value); + + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } +}; + +// SIMD byte mask type (returned by things like eq and gt) +template<> +struct simd16: base16 { + static simdutf_really_inline simd16 splat(bool _value) { return _mm_set1_epi16(uint16_t(-(!!_value))); } + + simdutf_really_inline simd16() : base16() {} + simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} + // Splat constructor + simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} + + simdutf_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdutf_really_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdutf_really_inline simd16 operator~() const { return *this ^ true; } +}; + +template +struct base16_numeric: base16 { + static simdutf_really_inline simd16 splat(T _value) { return _mm_set1_epi16(_value); } + static simdutf_really_inline simd16 zero() { return _mm_setzero_si128(); } + static simdutf_really_inline simd16 load(const T values[8]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + + simdutf_really_inline base16_numeric() : base16() {} + simdutf_really_inline base16_numeric(const __m128i _value) : base16(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[8]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } + + // Override to distinguish from bool version + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd16 operator+(const simd16 other) const { return _mm_add_epi16(*this, other); } + simdutf_really_inline simd16 operator-(const simd16 other) const { return _mm_sub_epi16(*this, other); } + simdutf_really_inline simd16& operator+=(const simd16 other) { *this = *this + other; return *static_cast*>(this); } + simdutf_really_inline simd16& operator-=(const simd16 other) { *this = *this - other; return *static_cast*>(this); } +}; + +// Signed words +template<> +struct simd16 : base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m128i _value) : base16_numeric(_value) {} + // Splat constructor + simdutf_really_inline simd16(int16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const int16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + // Member-by-member initialization + simdutf_really_inline simd16( + int16_t v0, int16_t v1, int16_t v2, int16_t v3, int16_t v4, int16_t v5, int16_t v6, int16_t v7) + : simd16(_mm_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7)) {} + simdutf_really_inline operator simd16() const; + + // Order-sensitive comparisons + simdutf_really_inline simd16 max_val(const simd16 other) const { return _mm_max_epi16(*this, other); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return _mm_min_epi16(*this, other); } + simdutf_really_inline simd16 operator>(const simd16 other) const { return _mm_cmpgt_epi16(*this, other); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return _mm_cmpgt_epi16(other, *this); } +}; + +// Unsigned words +template<> +struct simd16: base16_numeric { + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m128i _value) : base16_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} + // Array constructor + simdutf_really_inline simd16(const uint16_t* values) : simd16(load(values)) {} + simdutf_really_inline simd16(const char16_t* values) : simd16(load(reinterpret_cast(values))) {} + // Member-by-member initialization + simdutf_really_inline simd16( + uint16_t v0, uint16_t v1, uint16_t v2, uint16_t v3, uint16_t v4, uint16_t v5, uint16_t v6, uint16_t v7) + : simd16(_mm_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7)) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd16 repeat_16( + uint16_t v0, uint16_t v1, uint16_t v2, uint16_t v3, uint16_t v4, uint16_t v5, uint16_t v6, uint16_t v7 + ) { + return simd16(v0, v1, v2, v3, v4, v5, v6, v7); + } + + // Saturated math + simdutf_really_inline simd16 saturating_add(const simd16 other) const { return _mm_adds_epu16(*this, other); } + simdutf_really_inline simd16 saturating_sub(const simd16 other) const { return _mm_subs_epu16(*this, other); } + + // Order-specific operations + simdutf_really_inline simd16 max_val(const simd16 other) const { return _mm_max_epu16(*this, other); } + simdutf_really_inline simd16 min_val(const simd16 other) const { return _mm_min_epu16(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 gt_bits(const simd16 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd16 lt_bits(const simd16 other) const { return other.saturating_sub(*this); } + simdutf_really_inline simd16 operator<=(const simd16 other) const { return other.max_val(*this) == other; } + simdutf_really_inline simd16 operator>=(const simd16 other) const { return other.min_val(*this) == other; } + simdutf_really_inline simd16 operator>(const simd16 other) const { return this->gt_bits(other).any_bits_set(); } + simdutf_really_inline simd16 operator<(const simd16 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdutf_really_inline simd16 bits_not_set() const { return *this == uint16_t(0); } + simdutf_really_inline simd16 bits_not_set(simd16 bits) const { return (*this & bits).bits_not_set(); } + simdutf_really_inline simd16 any_bits_set() const { return ~this->bits_not_set(); } + simdutf_really_inline simd16 any_bits_set(simd16 bits) const { return ~this->bits_not_set(bits); } + + simdutf_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdutf_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdutf_really_inline bool bits_not_set_anywhere(simd16 bits) const { return _mm_testz_si128(*this, bits); } + simdutf_really_inline bool any_bits_set_anywhere(simd16 bits) const { return !bits_not_set_anywhere(bits); } + template + simdutf_really_inline simd16 shr() const { return simd16(_mm_srli_epi16(*this, N)); } + template + simdutf_really_inline simd16 shl() const { return simd16(_mm_slli_epi16(*this, N)); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdutf_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } + + // Pack with the unsigned saturation two uint16_t words into single uint8_t vector + static simdutf_really_inline simd8 pack(const simd16& v0, const simd16& v1) { + return _mm_packus_epi16(v0, v1); + } +}; +simdutf_really_inline simd16::operator simd16() const { return this->value; } + +template + struct simd16x32 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd16 chunks[NUM_CHUNKS]; + + simd16x32(const simd16x32& o) = delete; // no copy allowed + simd16x32& operator=(const simd16 other) = delete; // no assignment allowed + simd16x32() = delete; // no default constructor allowed + + simdutf_really_inline simd16x32(const simd16 chunk0, const simd16 chunk1, const simd16 chunk2, const simd16 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd16x32(const T* ptr) : chunks{simd16::load(ptr), simd16::load(ptr+sizeof(simd16)/sizeof(T)), simd16::load(ptr+2*sizeof(simd16)/sizeof(T)), simd16::load(ptr+3*sizeof(simd16)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr+sizeof(simd16)*0/sizeof(T)); + this->chunks[1].store(ptr+sizeof(simd16)*1/sizeof(T)); + this->chunks[2].store(ptr+sizeof(simd16)*2/sizeof(T)); + this->chunks[3].store(ptr+sizeof(simd16)*3/sizeof(T)); + } + + simdutf_really_inline simd16 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } + + simdutf_really_inline void store_ascii_as_utf16(char16_t * ptr) const { + this->chunks[0].store_ascii_as_utf16(ptr+sizeof(simd16)*0); + this->chunks[1].store_ascii_as_utf16(ptr+sizeof(simd16)*1); + this->chunks[2].store_ascii_as_utf16(ptr+sizeof(simd16)*2); + this->chunks[3].store_ascii_as_utf16(ptr+sizeof(simd16)*3); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); + uint64_t r1 = this->chunks[1].to_bitmask() ; + uint64_t r2 = this->chunks[2].to_bitmask() ; + uint64_t r3 = this->chunks[3].to_bitmask() ; + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t eq(const simd16x32 &other) const { + return simd16x32( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(low); + const simd16 mask_high = simd16::splat(high); + + return simd16x32( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd16 mask_low = simd16::splat(static_cast(low-1)); + const simd16 mask_high = simd16::splat(static_cast(high+1)); + return simd16x32( + (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low), + (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low), + (this->chunks[2] >= mask_high) | (this->chunks[2] <= mask_low), + (this->chunks[3] >= mask_high) | (this->chunks[3] <= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32( + this->chunks[0] < mask, + this->chunks[1] < mask, + this->chunks[2] < mask, + this->chunks[3] < mask + ).to_bitmask(); + } + }; // struct simd16x32 +/* end file src/simdutf/westmere/simd16-inl.h */ + +} // namespace simd +} // unnamed namespace +} // namespace westmere +} // namespace simdutf + +#endif // SIMDUTF_WESTMERE_SIMD_INPUT_H +/* end file src/simdutf/westmere/simd.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/end.h +/* begin file src/simdutf/westmere/end.h */ +SIMDUTF_UNTARGET_REGION +/* end file src/simdutf/westmere/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_WESTMERE +#endif // SIMDUTF_WESTMERE_COMMON_H +/* end file src/simdutf/westmere.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64.h +/* begin file src/simdutf/ppc64.h */ +#ifndef SIMDUTF_PPC64_H +#define SIMDUTF_PPC64_H + +#ifdef SIMDUTF_FALLBACK_H +#error "ppc64.h must be included before fallback.h" +#endif + + +#ifndef SIMDUTF_IMPLEMENTATION_PPC64 +#define SIMDUTF_IMPLEMENTATION_PPC64 (SIMDUTF_IS_PPC64) +#endif +#define SIMDUTF_CAN_ALWAYS_RUN_PPC64 SIMDUTF_IMPLEMENTATION_PPC64 && SIMDUTF_IS_PPC64 + + + +#if SIMDUTF_IMPLEMENTATION_PPC64 + +namespace simdutf { +/** + * Implementation for ALTIVEC (PPC64). + */ +namespace ppc64 { +} // namespace ppc64 +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/implementation.h +/* begin file src/simdutf/ppc64/implementation.h */ +#ifndef SIMDUTF_PPC64_IMPLEMENTATION_H +#define SIMDUTF_PPC64_IMPLEMENTATION_H + + +namespace simdutf { +namespace ppc64 { + +namespace { +using namespace simdutf; +} // namespace + +class implementation final : public simdutf::implementation { +public: + simdutf_really_inline implementation() + : simdutf::implementation("ppc64", "PPC64 ALTIVEC", + internal::instruction_set::ALTIVEC) {} + simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; + simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t length) const noexcept; + simdutf_warn_unused size_t count_utf8(const char * buf, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept; +}; + +} // namespace ppc64 +} // namespace simdutf + +#endif // SIMDUTF_PPC64_IMPLEMENTATION_H +/* end file src/simdutf/ppc64/implementation.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/begin.h +/* begin file src/simdutf/ppc64/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "ppc64" +// #define SIMDUTF_IMPLEMENTATION ppc64 +/* end file src/simdutf/ppc64/begin.h */ + +// Declarations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/intrinsics.h +/* begin file src/simdutf/ppc64/intrinsics.h */ +#ifndef SIMDUTF_PPC64_INTRINSICS_H +#define SIMDUTF_PPC64_INTRINSICS_H + + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +// These are defined by altivec.h in GCC toolchain, it is safe to undef them. +#ifdef bool +#undef bool +#endif + +#ifdef vector +#undef vector +#endif + +#endif // SIMDUTF_PPC64_INTRINSICS_H +/* end file src/simdutf/ppc64/intrinsics.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/bitmanipulation.h +/* begin file src/simdutf/ppc64/bitmanipulation.h */ +#ifndef SIMDUTF_PPC64_BITMANIPULATION_H +#define SIMDUTF_PPC64_BITMANIPULATION_H + +namespace simdutf { +namespace ppc64 { +namespace { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +NO_SANITIZE_UNDEFINED +simdutf_really_inline int trailing_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); +} + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int leading_zeroes(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif // SIMDUTF_REGULAR_VISUAL_STUDIO +} + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +simdutf_really_inline int count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num); // Visual Studio wants two underscores +} +#else +simdutf_really_inline int count_ones(uint64_t input_num) { + return __builtin_popcountll(input_num); +} +#endif + +simdutf_really_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t *result) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + *result = value1 + value2; + return *result < value1; +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif +} + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf + +#endif // SIMDUTF_PPC64_BITMANIPULATION_H +/* end file src/simdutf/ppc64/bitmanipulation.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/bitmask.h +/* begin file src/simdutf/ppc64/bitmask.h */ +#ifndef SIMDUTF_PPC64_BITMASK_H +#define SIMDUTF_PPC64_BITMASK_H + +namespace simdutf { +namespace ppc64 { +namespace { + +// +// Perform a "cumulative bitwise xor," flipping bits each time a 1 is +// encountered. +// +// For example, prefix_xor(00100100) == 00011100 +// +simdutf_really_inline uint64_t prefix_xor(uint64_t bitmask) { + // You can use the version below, however gcc sometimes miscompiles + // vec_pmsum_be, it happens somewhere around between 8 and 9th version. + // The performance boost was not noticeable, falling back to a usual + // implementation. + // __vector unsigned long long all_ones = {~0ull, ~0ull}; + // __vector unsigned long long mask = {bitmask, 0}; + // // Clang and GCC return different values for pmsum for ull so cast it to one. + // // Generally it is not specified by ALTIVEC ISA what is returned by + // // vec_pmsum_be. + // #if defined(__LITTLE_ENDIAN__) + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); + // #else + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); + // #endif + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; +} + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf + +#endif +/* end file src/simdutf/ppc64/bitmask.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/simd.h +/* begin file src/simdutf/ppc64/simd.h */ +#ifndef SIMDUTF_PPC64_SIMD_H +#define SIMDUTF_PPC64_SIMD_H + +#include + +namespace simdutf { +namespace ppc64 { +namespace { +namespace simd { + +using __m128i = __vector unsigned char; + +template struct base { + __m128i value; + + // Zero constructor + simdutf_really_inline base() : value{__m128i()} {} + + // Conversion from SIMD register + simdutf_really_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdutf_really_inline operator const __m128i &() const { + return this->value; + } + simdutf_really_inline operator __m128i &() { return this->value; } + + // Bit operations + simdutf_really_inline Child operator|(const Child other) const { + return vec_or(this->value, (__m128i)other); + } + simdutf_really_inline Child operator&(const Child other) const { + return vec_and(this->value, (__m128i)other); + } + simdutf_really_inline Child operator^(const Child other) const { + return vec_xor(this->value, (__m128i)other); + } + simdutf_really_inline Child bit_andnot(const Child other) const { + return vec_andc(this->value, (__m128i)other); + } + simdutf_really_inline Child &operator|=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast | other; + return *this_cast; + } + simdutf_really_inline Child &operator&=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast & other; + return *this_cast; + } + simdutf_really_inline Child &operator^=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast ^ other; + return *this_cast; + } +}; + +// Forward-declared so they can be used by splat and friends. +template struct simd8; + +template > +struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdutf_really_inline base8() : base>() {} + simdutf_really_inline base8(const __m128i _value) : base>(_value) {} + + simdutf_really_inline Mask operator==(const simd8 other) const { + return (__m128i)vec_cmpeq(this->value, (__m128i)other); + } + + static const int SIZE = sizeof(base>::value); + + template + simdutf_really_inline simd8 prev(simd8 prev_chunk) const { + __m128i chunk = this->value; +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve(this->value); + prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); +#endif + chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve((__m128i)chunk); +#endif + return chunk; + } +}; + +// SIMD byte mask type (returned by things like eq and gt) +template <> struct simd8 : base8 { + static simdutf_really_inline simd8 splat(bool _value) { + return (__m128i)vec_splats((unsigned char)(-(!!_value))); + } + + simdutf_really_inline simd8() : base8() {} + simdutf_really_inline simd8(const __m128i _value) + : base8(_value) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) + : base8(splat(_value)) {} + + simdutf_really_inline int to_bitmask() const { + __vector unsigned long long result; + const __m128i perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, + 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; + + result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, + (__m128i)perm_mask)); +#ifdef __LITTLE_ENDIAN__ + return static_cast(result[1]); +#else + return static_cast(result[0]); +#endif + } + simdutf_really_inline bool any() const { + return !vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdutf_really_inline simd8 operator~() const { + return this->value ^ (__m128i)splat(true); + } +}; + +template struct base8_numeric : base8 { + static simdutf_really_inline simd8 splat(T value) { + (void)value; + return (__m128i)vec_splats(value); + } + static simdutf_really_inline simd8 zero() { return splat(0); } + static simdutf_really_inline simd8 load(const T values[16]) { + return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdutf_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + T v5, T v6, T v7, T v8, T v9, + T v10, T v11, T v12, T v13, + T v14, T v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15); + } + + simdutf_really_inline base8_numeric() : base8() {} + simdutf_really_inline base8_numeric(const __m128i _value) + : base8(_value) {} + + // Store to array + simdutf_really_inline void store(T dst[16]) const { + vec_vsx_st(this->value, 0, reinterpret_cast<__m128i *>(dst)); + } + + // Override to distinguish from bool version + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdutf_really_inline simd8 operator+(const simd8 other) const { + return (__m128i)((__m128i)this->value + (__m128i)other); + } + simdutf_really_inline simd8 operator-(const simd8 other) const { + return (__m128i)((__m128i)this->value - (__m128i)other); + } + simdutf_really_inline simd8 &operator+=(const simd8 other) { + *this = *this + other; + return *static_cast *>(this); + } + simdutf_really_inline simd8 &operator-=(const simd8 other) { + *this = *this - other; + return *static_cast *>(this); + } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior + // for out of range values) + template + simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { + return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); + } + + template + simdutf_really_inline simd8 + lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, + L replace5, L replace6, L replace7, L replace8, L replace9, + L replace10, L replace11, L replace12, L replace13, L replace14, + L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, replace4, replace5, replace6, + replace7, replace8, replace9, replace10, replace11, replace12, + replace13, replace14, replace15)); + } +}; + +// Signed bytes +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) + : simd8((__m128i)(__vector signed char){v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, + v15}) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 + repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, + int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Order-sensitive comparisons + simdutf_really_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdutf_really_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return (__m128i)vec_cmpgt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdutf_really_inline simd8 + operator<(const simd8 other) const { + return (__m128i)vec_cmplt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } +}; + +// Unsigned bytes +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + // Splat constructor + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t *values) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) + : simd8((__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15}) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 + repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, + uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, + uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, + uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Saturated math + simdutf_really_inline simd8 + saturating_add(const simd8 other) const { + return (__m128i)vec_adds(this->value, (__m128i)other); + } + simdutf_really_inline simd8 + saturating_sub(const simd8 other) const { + return (__m128i)vec_subs(this->value, (__m128i)other); + } + + // Order-specific operations + simdutf_really_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max(this->value, (__m128i)other); + } + simdutf_really_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min(this->value, (__m128i)other); + } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 + gt_bits(const simd8 other) const { + return this->saturating_sub(other); + } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 + lt_bits(const simd8 other) const { + return other.saturating_sub(*this); + } + simdutf_really_inline simd8 + operator<=(const simd8 other) const { + return other.max_val(*this) == other; + } + simdutf_really_inline simd8 + operator>=(const simd8 other) const { + return other.min_val(*this) == other; + } + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + simdutf_really_inline simd8 + operator<(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + + // Bit-specific operations + simdutf_really_inline simd8 bits_not_set() const { + return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); + } + simdutf_really_inline simd8 bits_not_set(simd8 bits) const { + return (*this & bits).bits_not_set(); + } + simdutf_really_inline simd8 any_bits_set() const { + return ~this->bits_not_set(); + } + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { + return ~this->bits_not_set(bits); + } + + simdutf_really_inline bool is_ascii() const { + return this->saturating_sub(0b01111111u).bits_not_set_anywhere(); + } + + simdutf_really_inline bool bits_not_set_anywhere() const { + return vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdutf_really_inline bool any_bits_set_anywhere() const { + return !bits_not_set_anywhere(); + } + simdutf_really_inline bool bits_not_set_anywhere(simd8 bits) const { + return vec_all_eq(vec_and(this->value, (__m128i)bits), + (__m128i)vec_splats(0)); + } + simdutf_really_inline bool any_bits_set_anywhere(simd8 bits) const { + return !bits_not_set_anywhere(bits); + } + template simdutf_really_inline simd8 shr() const { + return simd8( + (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); + } + template simdutf_really_inline simd8 shl() const { + return simd8( + (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); + } +}; + +template struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, + "PPC64 kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64 &o) = delete; // no copy allowed + simd8x64 & + operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} + + simdutf_really_inline simd8x64(const T* ptr) : chunks{simd8::load(ptr), simd8::load(ptr+sizeof(simd8)/sizeof(T)), simd8::load(ptr+2*sizeof(simd8)/sizeof(T)), simd8::load(ptr+3*sizeof(simd8)/sizeof(T))} {} + + simdutf_really_inline void store(T* ptr) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0/sizeof(T)); + this->chunks[1].store(ptr + sizeof(simd8) * 1/sizeof(T)); + this->chunks[2].store(ptr + sizeof(simd8) * 2/sizeof(T)); + this->chunks[3].store(ptr + sizeof(simd8) * 3/sizeof(T)); + } + + simdutf_really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); + } + + + simdutf_really_inline bool is_ascii() const { + return input.reduce_or().is_ascii(); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdutf_really_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, + this->chunks[2] == mask, this->chunks[3] == mask) + .to_bitmask(); + } + + simdutf_really_inline uint64_t eq(const simd8x64 &other) const { + return simd8x64(this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3]) + .to_bitmask(); + } + + simdutf_really_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) + .to_bitmask(); + } + + simdutf_really_inline uint64_t in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + + return simd8x64( + (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), + (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), + (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { + const simd8 mask_low = simd8::splat(low); + const simd8 mask_high = simd8::splat(high); + return simd8x64( + (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low), + (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low), + (this->chunks[2] > mask_high) | (this->chunks[2] < mask_low), + (this->chunks[3] > mask_high) | (this->chunks[3] < mask_low) + ).to_bitmask(); + } + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, + this->chunks[2] < mask, this->chunks[3] < mask) + .to_bitmask(); + } + + simdutf_really_inline uint64_t gt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] > mask, + this->chunks[1] > mask, + this->chunks[2] > mask, + this->chunks[3] > mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] >= mask, + this->chunks[1] >= mask, + this->chunks[2] >= mask, + this->chunks[3] >= mask + ).to_bitmask(); + } + simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + simd8(this->chunks[0]) >= mask, + simd8(this->chunks[1]) >= mask, + simd8(this->chunks[2]) >= mask, + simd8(this->chunks[3]) >= mask + ).to_bitmask(); + } +}; // struct simd8x64 + +} // namespace simd +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf + +#endif // SIMDUTF_PPC64_SIMD_INPUT_H +/* end file src/simdutf/ppc64/simd.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/end.h +/* begin file src/simdutf/ppc64/end.h */ +/* end file src/simdutf/ppc64/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_PPC64 + +#endif // SIMDUTF_PPC64_H +/* end file src/simdutf/ppc64.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback.h +/* begin file src/simdutf/fallback.h */ +#ifndef SIMDUTF_FALLBACK_H +#define SIMDUTF_FALLBACK_H + + +// Default Fallback to on unless a builtin implementation has already been selected. +#ifndef SIMDUTF_IMPLEMENTATION_FALLBACK +#define SIMDUTF_IMPLEMENTATION_FALLBACK 1 // (!SIMDUTF_CAN_ALWAYS_RUN_ARM64 && !SIMDUTF_CAN_ALWAYS_RUN_HASWELL && !SIMDUTF_CAN_ALWAYS_RUN_WESTMERE && !SIMDUTF_CAN_ALWAYS_RUN_PPC64) +#endif +#define SIMDUTF_CAN_ALWAYS_RUN_FALLBACK SIMDUTF_IMPLEMENTATION_FALLBACK + +#if SIMDUTF_IMPLEMENTATION_FALLBACK + +namespace simdutf { +/** + * Fallback implementation (runs on any machine). + */ +namespace fallback { +} // namespace fallback +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/implementation.h +/* begin file src/simdutf/fallback/implementation.h */ +#ifndef SIMDUTF_FALLBACK_IMPLEMENTATION_H +#define SIMDUTF_FALLBACK_IMPLEMENTATION_H + + +namespace simdutf { +namespace fallback { + +namespace { +using namespace simdutf; +} + +class implementation final : public simdutf::implementation { +public: + simdutf_really_inline implementation() : simdutf::implementation( + "fallback", + "Generic fallback implementation", + 0 + ) {} + simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; + simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) const noexcept final; + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t length) const noexcept; + simdutf_warn_unused size_t count_utf8(const char * buf, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept; +}; + +} // namespace fallback +} // namespace simdutf + +#endif // SIMDUTF_FALLBACK_IMPLEMENTATION_H +/* end file src/simdutf/fallback/implementation.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/begin.h +/* begin file src/simdutf/fallback/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "fallback" +// #define SIMDUTF_IMPLEMENTATION fallback +/* end file src/simdutf/fallback/begin.h */ + +// Declarations +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/bitmanipulation.h +/* begin file src/simdutf/fallback/bitmanipulation.h */ +#ifndef SIMDUTF_FALLBACK_BITMANIPULATION_H +#define SIMDUTF_FALLBACK_BITMANIPULATION_H + +#include + +namespace simdutf { +namespace fallback { +namespace { + +#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) +static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { + unsigned long x0 = (unsigned long)x, top, bottom; + _BitScanForward(&top, (unsigned long)(x >> 32)); + _BitScanForward(&bottom, x0); + *ret = x0 ? bottom : 32 + top; + return x != 0; +} +static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { + unsigned long x1 = (unsigned long)(x >> 32), top, bottom; + _BitScanReverse(&top, x1); + _BitScanReverse(&bottom, (unsigned long)x); + *ret = x1 ? top + 32 : bottom; + return x != 0; +} +#endif + +/* result might be undefined when input_num is zero */ +simdutf_really_inline int leading_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// _MSC_VER +} + +} // unnamed namespace +} // namespace fallback +} // namespace simdutf + +#endif // SIMDUTF_FALLBACK_BITMANIPULATION_H +/* end file src/simdutf/fallback/bitmanipulation.h */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/end.h +/* begin file src/simdutf/fallback/end.h */ +/* end file src/simdutf/fallback/end.h */ + +#endif // SIMDUTF_IMPLEMENTATION_FALLBACK +#endif // SIMDUTF_FALLBACK_H +/* end file src/simdutf/fallback.h */ + + +namespace simdutf { +bool implementation::supported_by_runtime_system() const { + uint32_t required_instruction_sets = this->required_instruction_sets(); + uint32_t supported_instruction_sets = internal::detect_supported_architectures(); + return ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets); +} + +simdutf_warn_unused encoding_type implementation::autodetect_encoding(const char * input, size_t length) const noexcept { + // If there is a BOM, then we trust it. + auto bom_encoding = simdutf::BOM::check_bom(input, length); + if(bom_encoding != encoding_type::unspecified) { return bom_encoding; } + // UTF8 is common, it includes ASCII, and is commonly represented + // without a BOM, so if it fits, go with that. Note that it is still + // possible to get it wrong, we are only 'guessing'. If some has UTF-16 + // data without a BOM, it could pass as UTF-8. + // + // An interesting twist might be to check for UTF-16 ASCII first (every + // other byte is zero). + if(validate_utf8(input, length)) { return encoding_type::UTF8; } + // The next most common encoding that might appear without BOM is probably + // UTF-16LE, so try that next. + if((length % 2) == 0) { + if(validate_utf16(reinterpret_cast(input), length)) { return encoding_type::UTF16_LE; } + } + return encoding_type::unspecified; +} + +namespace internal { + +// Static array of known implementations. We're hoping these get baked into the executable +// without requiring a static initializer. + +#if SIMDUTF_IMPLEMENTATION_HASWELL +const haswell::implementation haswell_singleton{}; +#endif +#if SIMDUTF_IMPLEMENTATION_WESTMERE +const westmere::implementation westmere_singleton{}; +#endif // SIMDUTF_IMPLEMENTATION_WESTMERE +#if SIMDUTF_IMPLEMENTATION_ARM64 +const arm64::implementation arm64_singleton{}; +#endif // SIMDUTF_IMPLEMENTATION_ARM64 +#if SIMDUTF_IMPLEMENTATION_PPC64 +const ppc64::implementation ppc64_singleton{}; +#endif // SIMDUTF_IMPLEMENTATION_PPC64 +#if SIMDUTF_IMPLEMENTATION_FALLBACK +const fallback::implementation fallback_singleton{}; +#endif // SIMDUTF_IMPLEMENTATION_FALLBACK + +/** + * @private Detects best supported implementation on first use, and sets it + */ +class detect_best_supported_implementation_on_first_use final : public implementation { +public: + const std::string &name() const noexcept final { return set_best()->name(); } + const std::string &description() const noexcept final { return set_best()->description(); } + uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); } + + simdutf_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override { + return set_best()->validate_utf8(buf, len); + } + + simdutf_warn_unused bool validate_utf16(const char16_t * buf, size_t len) const noexcept final override { + return set_best()->validate_utf16(buf, len); + } + + simdutf_warn_unused size_t convert_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final override { + return set_best()->convert_utf8_to_utf16(buf, len, utf16_output); + } + + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * buf, size_t len, char16_t* utf16_output) const noexcept final override { + return set_best()->convert_valid_utf8_to_utf16(buf, len, utf16_output); + } + + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_output) const noexcept final override { + return set_best()->convert_utf16_to_utf8(buf, len, utf8_output); + } + + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_output) const noexcept final override { + return set_best()->convert_valid_utf16_to_utf8(buf, len, utf8_output); + } + + simdutf_warn_unused size_t count_utf16(const char16_t * buf, size_t len) const noexcept final override { + return set_best()->count_utf16(buf, len); + } + + simdutf_warn_unused size_t count_utf8(const char * buf, size_t len) const noexcept final override { + return set_best()->count_utf8(buf, len); + } + + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * buf, size_t len) const noexcept override { + return set_best()->utf8_length_from_utf16(buf, len); + } + + simdutf_warn_unused size_t utf16_length_from_utf8(const char * buf, size_t len) const noexcept override { + return set_best()->utf16_length_from_utf8(buf, len); + } + + simdutf_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} + +private: + const implementation *set_best() const noexcept; +}; + +const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; + +const std::initializer_list available_implementation_pointers { +#if SIMDUTF_IMPLEMENTATION_HASWELL + &haswell_singleton, +#endif +#if SIMDUTF_IMPLEMENTATION_WESTMERE + &westmere_singleton, +#endif +#if SIMDUTF_IMPLEMENTATION_ARM64 + &arm64_singleton, +#endif +#if SIMDUTF_IMPLEMENTATION_PPC64 + &ppc64_singleton, +#endif +#if SIMDUTF_IMPLEMENTATION_FALLBACK + &fallback_singleton, +#endif +}; // available_implementation_pointers + +// So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support +class unsupported_implementation final : public implementation { +public: + simdutf_warn_unused bool validate_utf8(const char *, size_t) const noexcept final override { + return false; // Just refuse to validate. Given that we have a fallback implementation + // it seems unlikely that unsupported_implementation will ever be used. If it is used, + // then it will flag all strings as invalid. The alternative is to return an error_code + // from which the user has to figure out whether the string is valid UTF-8... which seems + // like a lot of work just to handle the very unlikely case that we have an unsupported + // implementation. And, when it does happen (that we have an unsupported implementation), + // what are the chances that the programmer has a fallback? Given that *we* provide the + // fallback, it implies that the programmer would need a fallback for our fallback. + } + + simdutf_warn_unused bool validate_utf16(const char16_t*, size_t) const noexcept final override { + return false; + } + + simdutf_warn_unused size_t convert_utf8_to_utf16(const char*, size_t, char16_t*) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char*, size_t, char16_t*) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t*, size_t, char*) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t*, size_t, char*) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t count_utf16(const char16_t *, size_t) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t count_utf8(const char *, size_t) const noexcept final override { + return 0; + } + + simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *, size_t) const noexcept override { + return 0; + } + + simdutf_warn_unused size_t utf16_length_from_utf8(const char *, size_t) const noexcept override { + return 0; + } + + unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {} +}; + +const unsupported_implementation unsupported_singleton{}; + +size_t available_implementation_list::size() const noexcept { + return internal::available_implementation_pointers.size(); +} +const implementation * const *available_implementation_list::begin() const noexcept { + return internal::available_implementation_pointers.begin(); +} +const implementation * const *available_implementation_list::end() const noexcept { + return internal::available_implementation_pointers.end(); +} +const implementation *available_implementation_list::detect_best_supported() const noexcept { + // They are prelisted in priority order, so we just go down the list + uint32_t supported_instruction_sets = internal::detect_supported_architectures(); + for (const implementation *impl : internal::available_implementation_pointers) { + uint32_t required_instruction_sets = impl->required_instruction_sets(); + if ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets) { return impl; } + } + return &unsupported_singleton; // this should never happen? +} + +const implementation *detect_best_supported_implementation_on_first_use::set_best() const noexcept { + SIMDUTF_PUSH_DISABLE_WARNINGS + SIMDUTF_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + char *force_implementation_name = getenv("SIMDUTF_FORCE_IMPLEMENTATION"); + SIMDUTF_POP_DISABLE_WARNINGS + + if (force_implementation_name) { + auto force_implementation = available_implementations[force_implementation_name]; + if (force_implementation) { + return active_implementation = force_implementation; + } else { + // Note: abort() and stderr usage within the library is forbidden. + return active_implementation = &unsupported_singleton; + } + } + return active_implementation = available_implementations.detect_best_supported(); +} + +} // namespace internal + +SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{}; +SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton}; + +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { + return active_implementation->validate_utf8(buf, len); +} +simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_output) noexcept { + return active_implementation->convert_utf8_to_utf16(input, length, utf16_output); +} +simdutf_warn_unused bool validate_utf16(const char16_t * buf, size_t len) noexcept { + return active_implementation->validate_utf16(buf, len); +} +simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept { + return active_implementation->convert_valid_utf8_to_utf16(input, length, utf16_buffer); +} +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { + return active_implementation->convert_utf16_to_utf8(buf, len, utf8_buffer); +} +simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { + return active_implementation->convert_valid_utf16_to_utf8(buf, len, utf8_buffer); +} +simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept { + return active_implementation->count_utf16(input, length); +} +simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept { + return active_implementation->count_utf8(input, length); +} +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept { + return active_implementation->utf8_length_from_utf16(input, length); +} +simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept { + return active_implementation->utf16_length_from_utf8(input, length); +} +simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * buf, size_t length) noexcept { + return active_implementation->autodetect_encoding(buf, length); +} + +const implementation * builtin_implementation() { + static const implementation * builtin_impl = available_implementations[STRINGIFY(SIMDUTF_BUILTIN_IMPLEMENTATION)]; + return builtin_impl; +} + + +} // namespace simdutf + +/* end file src/implementation.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=encoding_types.cpp +/* begin file src/encoding_types.cpp */ + +namespace simdutf { +std::string to_string(encoding_type bom) { + switch (bom) { + case UTF16_LE: return "UTF16 little-endian"; + case UTF16_BE: return "UTF16 big-endian"; + case UTF32_LE: return "UTF32 little-endian"; + case UTF32_BE: return "UTF32 big-endian"; + case UTF8: return "UTF8"; + case unspecified: return "unknown"; + default: return "error"; + } +} + +namespace BOM { +// Note that BOM for UTF8 is discouraged. +encoding_type check_bom(const uint8_t* byte, size_t length) { + if (length >= 2 && byte[0] == 0xff and byte[1] == 0xfe) { + if (length >= 4 && byte[2] == 0x00 and byte[3] == 0x0) { + return encoding_type::UTF32_LE; + } else { + return encoding_type::UTF16_LE; + } + } else if (length >= 2 && byte[0] == 0xfe and byte[1] == 0xff) { + return encoding_type::UTF16_BE; + } else if (length >= 4 && byte[0] == 0x00 and byte[1] == 0x00 and byte[2] == 0xfe and byte[3] == 0xff) { + return encoding_type::UTF32_BE; + } else if (length >= 4 && byte[0] == 0xef and byte[1] == 0xbb and byte[3] == 0xbf) { + return encoding_type::UTF8; + } + return encoding_type::unspecified; + } + +encoding_type check_bom(const char* byte, size_t length) { + return check_bom(reinterpret_cast(byte), length); + } + + size_t bom_byte_size(encoding_type bom) { + switch (bom) { + case UTF16_LE: return 2; + case UTF16_BE: return 2; + case UTF32_LE: return 4; + case UTF32_BE: return 4; + case UTF8: return 3; + case unspecified: return 0; + default: return 0; + } +} + +} +} +/* end file src/encoding_types.cpp */ +// The large tables should be included once and they +// should not depend on a kernel. +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=tables/utf8_to_utf16_tables.h +/* begin file src/tables/utf8_to_utf16_tables.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_TABLES_H +#define SIMDUTF_UTF8_TO_UTF16_TABLES_H +#include + +namespace simdutf { +namespace { +namespace tables { +namespace utf8_to_utf16 { +/** + * utf8bigindex uses about 8 kB + * shufutf8 uses about 3344 B + * + * So we use a bit over 11 kB. It would be + * easy to save about 4 kB by only + * storing the index in utf8bigindex, and + * deriving the consumed bytes otherwise. + * However, this may come at a significant (10% to 20%) + * performance penalty. + */ + +const uint8_t shufutf8[209][16] = +{ {0, 255, 1, 255, 2, 255, 3, 255, 4, 255, 5, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 3, 255, 4, 255, 6, 5, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 3, 255, 5, 4, 6, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 3, 255, 5, 4, 7, 6, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 4, 3, 5, 255, 6, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 4, 3, 5, 255, 7, 6, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 4, 3, 6, 5, 7, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 2, 255, 4, 3, 6, 5, 8, 7, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 4, 255, 5, 255, 6, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 4, 255, 5, 255, 7, 6, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 4, 255, 6, 5, 7, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 4, 255, 6, 5, 8, 7, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 5, 4, 6, 255, 7, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 5, 4, 6, 255, 8, 7, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 5, 4, 7, 6, 8, 255, 0, 0, 0, 0}, + {0, 255, 1, 255, 3, 2, 5, 4, 7, 6, 9, 8, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 4, 255, 5, 255, 6, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 4, 255, 5, 255, 7, 6, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 4, 255, 6, 5, 7, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 4, 255, 6, 5, 8, 7, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 5, 4, 6, 255, 7, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 5, 4, 6, 255, 8, 7, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 5, 4, 7, 6, 8, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 3, 255, 5, 4, 7, 6, 9, 8, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 5, 255, 6, 255, 7, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 5, 255, 6, 255, 8, 7, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 5, 255, 7, 6, 8, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 5, 255, 7, 6, 9, 8, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 6, 5, 7, 255, 8, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 6, 5, 7, 255, 9, 8, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 6, 5, 8, 7, 9, 255, 0, 0, 0, 0}, + {0, 255, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 4, 255, 5, 255, 6, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 4, 255, 5, 255, 7, 6, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 4, 255, 6, 5, 7, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 4, 255, 6, 5, 8, 7, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 5, 4, 6, 255, 7, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 5, 4, 6, 255, 8, 7, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 5, 4, 7, 6, 8, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 3, 255, 5, 4, 7, 6, 9, 8, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 5, 255, 6, 255, 7, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 5, 255, 6, 255, 8, 7, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 5, 255, 7, 6, 8, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 5, 255, 7, 6, 9, 8, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 6, 5, 7, 255, 8, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 6, 5, 7, 255, 9, 8, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 6, 5, 8, 7, 9, 255, 0, 0, 0, 0}, + {1, 0, 2, 255, 4, 3, 6, 5, 8, 7, 10, 9, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 5, 255, 6, 255, 7, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 5, 255, 6, 255, 8, 7, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 5, 255, 7, 6, 8, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 5, 255, 7, 6, 9, 8, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 6, 5, 7, 255, 8, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 6, 5, 7, 255, 9, 8, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 6, 5, 8, 7, 9, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 4, 255, 6, 5, 8, 7, 10, 9, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 6, 255, 7, 255, 8, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 6, 255, 7, 255, 9, 8, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 6, 255, 8, 7, 9, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 6, 255, 8, 7, 10, 9, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 7, 6, 8, 255, 9, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 7, 6, 8, 255, 10, 9, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 10, 255, 0, 0, 0, 0}, + {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 0, 0, 0, 0}, + {0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 4, 3, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 5, 4, 3, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 3, 2, 255, 255, 4, 255, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 3, 2, 255, 255, 5, 4, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 3, 2, 255, 255, 6, 5, 4, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 4, 3, 2, 255, 5, 255, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 4, 3, 2, 255, 6, 5, 255, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 4, 3, 2, 255, 7, 6, 5, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 3, 255, 255, 255, 4, 255, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 3, 255, 255, 255, 5, 4, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 3, 255, 255, 255, 6, 5, 4, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 4, 3, 255, 255, 5, 255, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 4, 3, 255, 255, 6, 5, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 4, 3, 255, 255, 7, 6, 5, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 5, 4, 3, 255, 6, 255, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 5, 4, 3, 255, 7, 6, 255, 255}, + {0, 255, 255, 255, 2, 1, 255, 255, 5, 4, 3, 255, 8, 7, 6, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 4, 255, 255, 255, 5, 255, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 4, 255, 255, 255, 6, 5, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 4, 255, 255, 255, 7, 6, 5, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 5, 4, 255, 255, 6, 255, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 5, 4, 255, 255, 7, 6, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 5, 4, 255, 255, 8, 7, 6, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 6, 5, 4, 255, 7, 255, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 6, 5, 4, 255, 8, 7, 255, 255}, + {0, 255, 255, 255, 3, 2, 1, 255, 6, 5, 4, 255, 9, 8, 7, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255, 4, 255, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255, 5, 4, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255, 6, 5, 4, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 4, 3, 255, 255, 5, 255, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 4, 3, 255, 255, 6, 5, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 4, 3, 255, 255, 7, 6, 5, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 5, 4, 3, 255, 6, 255, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 5, 4, 3, 255, 7, 6, 255, 255}, + {1, 0, 255, 255, 2, 255, 255, 255, 5, 4, 3, 255, 8, 7, 6, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 4, 255, 255, 255, 5, 255, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 4, 255, 255, 255, 6, 5, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 4, 255, 255, 255, 7, 6, 5, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 5, 4, 255, 255, 6, 255, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 5, 4, 255, 255, 7, 6, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 5, 4, 255, 255, 8, 7, 6, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 6, 5, 4, 255, 7, 255, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 6, 5, 4, 255, 8, 7, 255, 255}, + {1, 0, 255, 255, 3, 2, 255, 255, 6, 5, 4, 255, 9, 8, 7, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 5, 255, 255, 255, 6, 255, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 5, 255, 255, 255, 7, 6, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 5, 255, 255, 255, 8, 7, 6, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 6, 5, 255, 255, 7, 255, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 6, 5, 255, 255, 8, 7, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 6, 5, 255, 255, 9, 8, 7, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 7, 6, 5, 255, 8, 255, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 7, 6, 5, 255, 9, 8, 255, 255}, + {1, 0, 255, 255, 4, 3, 2, 255, 7, 6, 5, 255, 10, 9, 8, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 4, 255, 255, 255, 5, 255, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 4, 255, 255, 255, 6, 5, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 4, 255, 255, 255, 7, 6, 5, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 5, 4, 255, 255, 6, 255, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 5, 4, 255, 255, 7, 6, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 5, 4, 255, 255, 8, 7, 6, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 6, 5, 4, 255, 7, 255, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 6, 5, 4, 255, 8, 7, 255, 255}, + {2, 1, 0, 255, 3, 255, 255, 255, 6, 5, 4, 255, 9, 8, 7, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 5, 255, 255, 255, 6, 255, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 5, 255, 255, 255, 7, 6, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 5, 255, 255, 255, 8, 7, 6, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 6, 5, 255, 255, 7, 255, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 6, 5, 255, 255, 8, 7, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 6, 5, 255, 255, 9, 8, 7, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 7, 6, 5, 255, 8, 255, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 7, 6, 5, 255, 9, 8, 255, 255}, + {2, 1, 0, 255, 4, 3, 255, 255, 7, 6, 5, 255, 10, 9, 8, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 6, 255, 255, 255, 7, 255, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 6, 255, 255, 255, 8, 7, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 6, 255, 255, 255, 9, 8, 7, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 7, 6, 255, 255, 8, 255, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 7, 6, 255, 255, 9, 8, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 7, 6, 255, 255, 10, 9, 8, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 9, 255, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 10, 9, 255, 255}, + {2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 11, 10, 9, 255}, + {0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 1, 255, 255, 255, 3, 2, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 1, 255, 255, 255, 4, 3, 2, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 1, 255, 255, 255, 5, 4, 3, 2, 0, 0, 0, 0}, + {0, 255, 255, 255, 2, 1, 255, 255, 3, 255, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 2, 1, 255, 255, 4, 3, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 2, 1, 255, 255, 5, 4, 3, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 2, 1, 255, 255, 6, 5, 4, 3, 0, 0, 0, 0}, + {0, 255, 255, 255, 3, 2, 1, 255, 4, 255, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 3, 2, 1, 255, 5, 4, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 3, 2, 1, 255, 6, 5, 4, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 3, 2, 1, 255, 7, 6, 5, 4, 0, 0, 0, 0}, + {0, 255, 255, 255, 4, 3, 2, 1, 5, 255, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 4, 3, 2, 1, 6, 5, 255, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 4, 3, 2, 1, 7, 6, 5, 255, 0, 0, 0, 0}, + {0, 255, 255, 255, 4, 3, 2, 1, 8, 7, 6, 5, 0, 0, 0, 0}, + {1, 0, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 2, 255, 255, 255, 4, 3, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 2, 255, 255, 255, 5, 4, 3, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 2, 255, 255, 255, 6, 5, 4, 3, 0, 0, 0, 0}, + {1, 0, 255, 255, 3, 2, 255, 255, 4, 255, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 3, 2, 255, 255, 5, 4, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 3, 2, 255, 255, 6, 5, 4, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 3, 2, 255, 255, 7, 6, 5, 4, 0, 0, 0, 0}, + {1, 0, 255, 255, 4, 3, 2, 255, 5, 255, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 4, 3, 2, 255, 6, 5, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 4, 3, 2, 255, 7, 6, 5, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 4, 3, 2, 255, 8, 7, 6, 5, 0, 0, 0, 0}, + {1, 0, 255, 255, 5, 4, 3, 2, 6, 255, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 5, 4, 3, 2, 7, 6, 255, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 5, 4, 3, 2, 8, 7, 6, 255, 0, 0, 0, 0}, + {1, 0, 255, 255, 5, 4, 3, 2, 9, 8, 7, 6, 0, 0, 0, 0}, + {2, 1, 0, 255, 3, 255, 255, 255, 4, 255, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 3, 255, 255, 255, 5, 4, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 3, 255, 255, 255, 6, 5, 4, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 3, 255, 255, 255, 7, 6, 5, 4, 0, 0, 0, 0}, + {2, 1, 0, 255, 4, 3, 255, 255, 5, 255, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 4, 3, 255, 255, 6, 5, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 4, 3, 255, 255, 7, 6, 5, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 4, 3, 255, 255, 8, 7, 6, 5, 0, 0, 0, 0}, + {2, 1, 0, 255, 5, 4, 3, 255, 6, 255, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 5, 4, 3, 255, 7, 6, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 5, 4, 3, 255, 9, 8, 7, 6, 0, 0, 0, 0}, + {2, 1, 0, 255, 6, 5, 4, 3, 7, 255, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 6, 5, 4, 3, 8, 7, 255, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 6, 5, 4, 3, 9, 8, 7, 255, 0, 0, 0, 0}, + {2, 1, 0, 255, 6, 5, 4, 3, 10, 9, 8, 7, 0, 0, 0, 0}, + {3, 2, 1, 0, 4, 255, 255, 255, 5, 255, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 4, 255, 255, 255, 6, 5, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 4, 255, 255, 255, 7, 6, 5, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 4, 255, 255, 255, 8, 7, 6, 5, 0, 0, 0, 0}, + {3, 2, 1, 0, 5, 4, 255, 255, 6, 255, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 5, 4, 255, 255, 7, 6, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 5, 4, 255, 255, 8, 7, 6, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 5, 4, 255, 255, 9, 8, 7, 6, 0, 0, 0, 0}, + {3, 2, 1, 0, 6, 5, 4, 255, 7, 255, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 6, 5, 4, 255, 8, 7, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 6, 5, 4, 255, 9, 8, 7, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 6, 5, 4, 255, 10, 9, 8, 7, 0, 0, 0, 0}, + {3, 2, 1, 0, 7, 6, 5, 4, 8, 255, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 7, 6, 5, 4, 9, 8, 255, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 7, 6, 5, 4, 10, 9, 8, 255, 0, 0, 0, 0}, + {3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 0, 0, 0, 0}}; +/* number of two bytes : 64 */ +/* number of two + three bytes : 145 */ +/* number of two + three + four bytes : 209 */ +const uint8_t utf8bigindex[4096][2] = +{ {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {148, 6}, + {0, 12}, + {151, 6}, + {163, 6}, + {66, 6}, + {0, 12}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {152, 7}, + {164, 7}, + {145, 3}, + {0, 12}, + {155, 7}, + {167, 7}, + {69, 7}, + {179, 7}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {170, 7}, + {71, 7}, + {182, 7}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {185, 7}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {171, 8}, + {72, 8}, + {183, 8}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {186, 8}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {68, 6}, + {122, 8}, + {74, 6}, + {92, 6}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {76, 6}, + {94, 6}, + {5, 8}, + {193, 6}, + {82, 6}, + {100, 6}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {77, 7}, + {95, 7}, + {6, 8}, + {194, 7}, + {83, 7}, + {101, 7}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {187, 9}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {77, 7}, + {95, 7}, + {7, 9}, + {194, 7}, + {83, 7}, + {101, 7}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {176, 10}, + {148, 6}, + {188, 10}, + {151, 6}, + {163, 6}, + {66, 6}, + {200, 10}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {191, 10}, + {152, 7}, + {164, 7}, + {145, 3}, + {203, 10}, + {90, 10}, + {108, 10}, + {69, 7}, + {126, 10}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {114, 10}, + {71, 7}, + {132, 10}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {138, 10}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {206, 10}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {116, 10}, + {72, 8}, + {134, 10}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {140, 10}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {15, 10}, + {122, 8}, + {23, 10}, + {39, 10}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {27, 10}, + {43, 10}, + {5, 8}, + {193, 6}, + {82, 6}, + {51, 10}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {29, 10}, + {45, 10}, + {6, 8}, + {194, 7}, + {83, 7}, + {53, 10}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {57, 10}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {142, 10}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {30, 10}, + {46, 10}, + {7, 9}, + {194, 7}, + {83, 7}, + {54, 10}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {58, 10}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {60, 10}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {148, 6}, + {0, 12}, + {151, 6}, + {163, 6}, + {66, 6}, + {0, 12}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {192, 11}, + {152, 7}, + {164, 7}, + {145, 3}, + {204, 11}, + {155, 7}, + {167, 7}, + {69, 7}, + {179, 7}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {170, 7}, + {71, 7}, + {182, 7}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {185, 7}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {207, 11}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {117, 11}, + {72, 8}, + {135, 11}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {141, 11}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {68, 6}, + {122, 8}, + {74, 6}, + {92, 6}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {76, 6}, + {94, 6}, + {5, 8}, + {193, 6}, + {82, 6}, + {100, 6}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {77, 7}, + {95, 7}, + {6, 8}, + {194, 7}, + {83, 7}, + {101, 7}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {143, 11}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {31, 11}, + {47, 11}, + {7, 9}, + {194, 7}, + {83, 7}, + {55, 11}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {59, 11}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {61, 11}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {176, 10}, + {148, 6}, + {188, 10}, + {151, 6}, + {163, 6}, + {66, 6}, + {200, 10}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {191, 10}, + {152, 7}, + {164, 7}, + {145, 3}, + {203, 10}, + {90, 10}, + {108, 10}, + {69, 7}, + {126, 10}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {114, 10}, + {71, 7}, + {132, 10}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {138, 10}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {206, 10}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {116, 10}, + {72, 8}, + {134, 10}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {140, 10}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {62, 11}, + {15, 10}, + {122, 8}, + {23, 10}, + {39, 10}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {27, 10}, + {43, 10}, + {5, 8}, + {193, 6}, + {82, 6}, + {51, 10}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {29, 10}, + {45, 10}, + {6, 8}, + {194, 7}, + {83, 7}, + {53, 10}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {57, 10}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {142, 10}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {30, 10}, + {46, 10}, + {7, 9}, + {194, 7}, + {83, 7}, + {54, 10}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {58, 10}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {60, 10}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {148, 6}, + {0, 12}, + {151, 6}, + {163, 6}, + {66, 6}, + {0, 12}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {152, 7}, + {164, 7}, + {145, 3}, + {0, 12}, + {155, 7}, + {167, 7}, + {69, 7}, + {179, 7}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {170, 7}, + {71, 7}, + {182, 7}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {185, 7}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {208, 12}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {171, 8}, + {72, 8}, + {183, 8}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {186, 8}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {68, 6}, + {122, 8}, + {74, 6}, + {92, 6}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {76, 6}, + {94, 6}, + {5, 8}, + {193, 6}, + {82, 6}, + {100, 6}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {77, 7}, + {95, 7}, + {6, 8}, + {194, 7}, + {83, 7}, + {101, 7}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {144, 12}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {77, 7}, + {95, 7}, + {7, 9}, + {194, 7}, + {83, 7}, + {101, 7}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {176, 10}, + {148, 6}, + {188, 10}, + {151, 6}, + {163, 6}, + {66, 6}, + {200, 10}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {191, 10}, + {152, 7}, + {164, 7}, + {145, 3}, + {203, 10}, + {90, 10}, + {108, 10}, + {69, 7}, + {126, 10}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {114, 10}, + {71, 7}, + {132, 10}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {138, 10}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {206, 10}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {116, 10}, + {72, 8}, + {134, 10}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {140, 10}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {63, 12}, + {15, 10}, + {122, 8}, + {23, 10}, + {39, 10}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {27, 10}, + {43, 10}, + {5, 8}, + {193, 6}, + {82, 6}, + {51, 10}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {29, 10}, + {45, 10}, + {6, 8}, + {194, 7}, + {83, 7}, + {53, 10}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {57, 10}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {142, 10}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {30, 10}, + {46, 10}, + {7, 9}, + {194, 7}, + {83, 7}, + {54, 10}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {58, 10}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {60, 10}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {148, 6}, + {0, 12}, + {151, 6}, + {163, 6}, + {66, 6}, + {0, 12}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {192, 11}, + {152, 7}, + {164, 7}, + {145, 3}, + {204, 11}, + {155, 7}, + {167, 7}, + {69, 7}, + {179, 7}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {170, 7}, + {71, 7}, + {182, 7}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {185, 7}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {207, 11}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {117, 11}, + {72, 8}, + {135, 11}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {141, 11}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {104, 8}, + {68, 6}, + {122, 8}, + {74, 6}, + {92, 6}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {76, 6}, + {94, 6}, + {5, 8}, + {193, 6}, + {82, 6}, + {100, 6}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {77, 7}, + {95, 7}, + {6, 8}, + {194, 7}, + {83, 7}, + {101, 7}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {143, 11}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {31, 11}, + {47, 11}, + {7, 9}, + {194, 7}, + {83, 7}, + {55, 11}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {59, 11}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {61, 11}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {0, 12}, + {0, 12}, + {147, 5}, + {0, 12}, + {150, 5}, + {162, 5}, + {65, 5}, + {0, 12}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {176, 10}, + {148, 6}, + {188, 10}, + {151, 6}, + {163, 6}, + {66, 6}, + {200, 10}, + {154, 6}, + {166, 6}, + {68, 6}, + {178, 6}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {169, 6}, + {70, 6}, + {181, 6}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {191, 10}, + {152, 7}, + {164, 7}, + {145, 3}, + {203, 10}, + {90, 10}, + {108, 10}, + {69, 7}, + {126, 10}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {114, 10}, + {71, 7}, + {132, 10}, + {77, 7}, + {95, 7}, + {65, 5}, + {194, 7}, + {83, 7}, + {101, 7}, + {67, 5}, + {119, 7}, + {73, 5}, + {91, 5}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {138, 10}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {103, 7}, + {68, 6}, + {121, 7}, + {74, 6}, + {92, 6}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {76, 6}, + {94, 6}, + {4, 7}, + {193, 6}, + {82, 6}, + {100, 6}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {206, 10}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {116, 10}, + {72, 8}, + {134, 10}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {140, 10}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {62, 11}, + {15, 10}, + {122, 8}, + {23, 10}, + {39, 10}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {27, 10}, + {43, 10}, + {5, 8}, + {193, 6}, + {82, 6}, + {51, 10}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {29, 10}, + {45, 10}, + {6, 8}, + {194, 7}, + {83, 7}, + {53, 10}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {57, 10}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {0, 12}, + {0, 12}, + {0, 12}, + {146, 4}, + {0, 12}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {160, 9}, + {172, 9}, + {147, 5}, + {184, 9}, + {150, 5}, + {162, 5}, + {65, 5}, + {196, 9}, + {153, 5}, + {165, 5}, + {67, 5}, + {177, 5}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {175, 9}, + {148, 6}, + {142, 10}, + {81, 9}, + {99, 9}, + {66, 6}, + {199, 9}, + {87, 9}, + {105, 9}, + {68, 6}, + {123, 9}, + {74, 6}, + {92, 6}, + {64, 4}, + {0, 12}, + {157, 6}, + {111, 9}, + {70, 6}, + {129, 9}, + {76, 6}, + {94, 6}, + {65, 5}, + {193, 6}, + {82, 6}, + {100, 6}, + {67, 5}, + {118, 6}, + {73, 5}, + {91, 5}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {190, 9}, + {152, 7}, + {164, 7}, + {145, 3}, + {202, 9}, + {89, 9}, + {107, 9}, + {69, 7}, + {125, 9}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {113, 9}, + {71, 7}, + {131, 9}, + {30, 10}, + {46, 10}, + {7, 9}, + {194, 7}, + {83, 7}, + {54, 10}, + {11, 9}, + {119, 7}, + {19, 9}, + {35, 9}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {137, 9}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {58, 10}, + {13, 9}, + {121, 7}, + {21, 9}, + {37, 9}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {25, 9}, + {41, 9}, + {4, 7}, + {193, 6}, + {82, 6}, + {49, 9}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {145, 3}, + {205, 9}, + {156, 8}, + {168, 8}, + {146, 4}, + {180, 8}, + {149, 4}, + {161, 4}, + {64, 4}, + {0, 12}, + {159, 8}, + {115, 9}, + {72, 8}, + {133, 9}, + {78, 8}, + {96, 8}, + {65, 5}, + {195, 8}, + {84, 8}, + {102, 8}, + {67, 5}, + {120, 8}, + {73, 5}, + {91, 5}, + {64, 4}, + {0, 12}, + {0, 12}, + {174, 8}, + {148, 6}, + {139, 9}, + {80, 8}, + {98, 8}, + {66, 6}, + {198, 8}, + {86, 8}, + {60, 10}, + {14, 9}, + {122, 8}, + {22, 9}, + {38, 9}, + {3, 8}, + {0, 12}, + {157, 6}, + {110, 8}, + {70, 6}, + {128, 8}, + {26, 9}, + {42, 9}, + {5, 8}, + {193, 6}, + {82, 6}, + {50, 9}, + {9, 8}, + {118, 6}, + {17, 8}, + {33, 8}, + {0, 6}, + {0, 12}, + {0, 12}, + {0, 12}, + {0, 12}, + {189, 8}, + {152, 7}, + {164, 7}, + {145, 3}, + {201, 8}, + {88, 8}, + {106, 8}, + {69, 7}, + {124, 8}, + {75, 7}, + {93, 7}, + {64, 4}, + {0, 12}, + {158, 7}, + {112, 8}, + {71, 7}, + {130, 8}, + {28, 9}, + {44, 9}, + {6, 8}, + {194, 7}, + {83, 7}, + {52, 9}, + {10, 8}, + {119, 7}, + {18, 8}, + {34, 8}, + {1, 7}, + {0, 12}, + {0, 12}, + {173, 7}, + {148, 6}, + {136, 8}, + {79, 7}, + {97, 7}, + {66, 6}, + {197, 7}, + {85, 7}, + {56, 9}, + {12, 8}, + {121, 7}, + {20, 8}, + {36, 8}, + {2, 7}, + {0, 12}, + {157, 6}, + {109, 7}, + {70, 6}, + {127, 7}, + {24, 8}, + {40, 8}, + {4, 7}, + {193, 6}, + {82, 6}, + {48, 8}, + {8, 7}, + {118, 6}, + {16, 7}, + {32, 7}, + {0, 6}}; +} // utf8_to_utf16 namespace +} // tables namespace +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_UTF8_TO_UTF16_TABLES_H +/* end file src/tables/utf8_to_utf16_tables.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=tables/utf16_to_utf8_tables.h +/* begin file src/tables/utf16_to_utf8_tables.h */ +// file generated by scripts/sse_convert_utf16_to_utf8.py +#ifndef SIMDUTF_UTF16_TO_UTF8_TABLES_H +#define SIMDUTF_UTF16_TO_UTF8_TABLES_H + +namespace simdutf { +namespace { +namespace tables { +namespace utf16_to_utf8 { + + // 1 byte for length, 16 bytes for mask + const uint8_t pack_1_2_utf8_bytes[256][17] = { + {16,1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14}, + {15,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14,0x80}, + {15,1,0,3,2,5,4,7,6,8,11,10,13,12,15,14,0x80}, + {14,0,3,2,5,4,7,6,8,11,10,13,12,15,14,0x80,0x80}, + {15,1,0,2,5,4,7,6,9,8,11,10,13,12,15,14,0x80}, + {14,0,2,5,4,7,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {14,1,0,2,5,4,7,6,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,2,5,4,7,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {15,1,0,3,2,5,4,7,6,9,8,10,13,12,15,14,0x80}, + {14,0,3,2,5,4,7,6,9,8,10,13,12,15,14,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,8,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80}, + {14,1,0,2,5,4,7,6,9,8,10,13,12,15,14,0x80,0x80}, + {13,0,2,5,4,7,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {15,1,0,3,2,4,7,6,9,8,11,10,13,12,15,14,0x80}, + {14,0,3,2,4,7,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {14,1,0,3,2,4,7,6,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,4,7,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {14,1,0,2,4,7,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,2,4,7,6,9,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,2,4,7,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,4,7,6,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,4,7,6,9,8,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,4,7,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,4,7,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,4,7,6,9,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {15,1,0,3,2,5,4,7,6,9,8,11,10,12,15,14,0x80}, + {14,0,3,2,5,4,7,6,9,8,11,10,12,15,14,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,8,11,10,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80}, + {14,1,0,2,5,4,7,6,9,8,11,10,12,15,14,0x80,0x80}, + {13,0,2,5,4,7,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,9,8,10,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,7,6,8,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,4,7,6,9,8,11,10,12,15,14,0x80,0x80}, + {13,0,3,2,4,7,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,4,7,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,2,4,7,6,9,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,7,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {15,1,0,3,2,5,4,6,9,8,11,10,13,12,15,14,0x80}, + {14,0,3,2,5,4,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {14,1,0,3,2,5,4,6,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {14,1,0,2,5,4,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,2,5,4,6,9,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,2,5,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,6,9,8,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,4,6,9,8,11,10,13,12,15,14,0x80,0x80}, + {13,0,3,2,4,6,9,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,4,6,9,8,11,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,2,4,6,9,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,6,8,11,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,6,9,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,8,10,13,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,6,9,8,11,10,12,15,14,0x80,0x80}, + {13,0,3,2,5,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,2,5,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,9,8,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,5,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,5,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,5,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,5,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80}, + {12,0,3,2,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,6,9,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,8,11,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,9,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,4,6,8,10,12,15,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {15,1,0,3,2,5,4,7,6,9,8,11,10,13,12,14,0x80}, + {14,0,3,2,5,4,7,6,9,8,11,10,13,12,14,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,8,11,10,13,12,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80}, + {14,1,0,2,5,4,7,6,9,8,11,10,13,12,14,0x80,0x80}, + {13,0,2,5,4,7,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,9,8,10,13,12,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,7,6,8,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,4,7,6,9,8,11,10,13,12,14,0x80,0x80}, + {13,0,3,2,4,7,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,4,7,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,2,4,7,6,9,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,7,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,7,6,9,8,11,10,12,14,0x80,0x80}, + {13,0,3,2,5,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,7,6,8,11,10,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80}, + {12,0,2,5,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,7,6,9,8,10,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,5,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,5,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,5,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,5,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80}, + {12,0,3,2,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,7,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,7,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,7,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,4,7,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {14,1,0,3,2,5,4,6,9,8,11,10,13,12,14,0x80,0x80}, + {13,0,3,2,5,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {13,1,0,2,5,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,2,5,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,9,8,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,5,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,5,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,5,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,5,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80}, + {12,0,3,2,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,4,6,9,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,8,11,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,9,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,4,6,8,10,13,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {13,1,0,3,2,5,4,6,9,8,11,10,12,14,0x80,0x80,0x80}, + {12,0,3,2,5,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,5,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,5,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,2,5,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,2,5,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,5,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,5,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,5,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,5,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,5,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,5,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,5,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,5,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,5,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,5,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {12,1,0,3,2,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80}, + {11,0,3,2,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,2,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,2,4,6,9,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,4,6,8,11,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,1,0,3,2,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80}, + {10,0,3,2,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,3,2,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,3,2,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,1,0,2,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,2,4,6,9,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,1,0,2,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,2,4,6,8,10,12,14,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80} + }; + + // 1 byte for length, 16 bytes for mask + const uint8_t pack_1_2_3_utf8_bytes[256][17] = { + {12,2,3,1,6,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80}, + {9,6,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,3,1,6,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80}, + {10,0,6,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,2,3,1,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80}, + {8,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,3,1,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,7,5,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,4,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,4,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,4,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,4,10,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,6,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,6,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,6,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,6,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,7,5,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,4,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,4,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,4,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,4,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,2,3,1,6,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80}, + {8,6,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,3,1,6,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,6,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,7,5,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,4,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,4,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,4,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,4,11,9,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,6,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,6,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,6,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,6,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,7,5,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,4,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,4,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,4,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,4,8,14,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,6,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,6,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,6,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,6,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,7,5,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,4,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,4,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,4,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,4,10,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,6,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,6,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,6,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,6,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,2,3,1,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {0,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,3,1,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {1,0,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,7,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,2,3,1,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {1,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,3,1,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,0,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,6,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,6,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,6,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,6,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,7,5,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,4,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,4,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,4,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,4,11,9,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,6,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,6,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,6,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,6,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,2,3,1,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {1,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,3,1,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,0,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,7,5,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,4,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,4,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,4,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,4,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {11,2,3,1,6,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80}, + {8,6,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,3,1,6,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,0,6,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,7,5,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,4,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,4,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,4,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,4,10,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,6,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,6,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,6,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,6,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,7,5,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,4,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,4,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,4,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,4,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,6,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,6,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,6,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,6,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,7,5,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,4,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,4,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,4,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,4,11,9,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,6,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,6,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,6,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,6,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,7,5,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,4,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,4,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,4,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,4,8,15,13,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {10,2,3,1,6,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,6,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,3,1,6,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,0,6,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,7,5,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,4,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,4,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,4,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,4,10,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,6,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,6,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,6,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,6,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,2,3,1,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {1,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,3,1,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,0,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,7,5,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,4,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,4,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,4,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,4,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {9,2,3,1,6,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,6,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,3,1,6,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,0,6,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,7,5,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,4,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,4,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,4,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,4,11,9,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {8,2,3,1,6,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,6,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,3,1,6,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,0,6,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,2,3,1,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {2,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,3,1,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,0,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {7,2,3,1,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,3,1,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,0,7,5,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {6,2,3,1,4,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {3,4,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {5,3,1,4,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80}, + {4,0,4,8,12,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80} + }; + +} // utf16_to_utf8 namespace +} // tables namespace +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_UTF16_TO_UTF8_TABLES_H +/* end file src/tables/utf16_to_utf8_tables.h */ +// End of tables. + +// The scalar routines should be included once. +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf16_to_utf8/valid_utf16_to_utf8.h +/* begin file src/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +inline size_t convert_valid(const char16_t* buf, size_t len, char* utf8_output) { + const uint16_t *data = reinterpret_cast(buf); + size_t pos = 0; + char* start{utf8_output}; + while (pos < len) { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while(pos < final_pos) { + *utf8_output++ = char(buf[pos]); + pos++; + } + continue; + } + } + uint16_t word = data[pos]; + if((word & 0xFF80)==0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if((word & 0xF800)==0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word>>6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if((word &0xF800 ) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word>>12) | 0b11100000); + *utf8_output++ = char(((word>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if(pos + 1 >= len) { return 0; } // minimal bound checking + uint16_t next_word = data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value>>18) | 0b11110000); + *utf8_output++ = char(((value>>12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf16_to_utf8/utf16_to_utf8.h +/* begin file src/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +inline size_t convert(const char16_t* buf, size_t len, char* utf8_output) { + const uint16_t *data = reinterpret_cast(buf); + size_t pos = 0; + char* start{utf8_output}; + while (pos < len) { + // try to convert the next block of 8 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while(pos < final_pos) { + *utf8_output++ = char(buf[pos]); + pos++; + } + continue; + } + } + uint16_t word = data[pos]; + if((word & 0xFF80)==0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if((word & 0xF800)==0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word>>6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if((word &0xF800 ) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word>>12) | 0b11100000); + *utf8_output++ = char(((word>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if(pos + 1 >= len) { return 0; } + uint16_t diff = uint16_t(word - 0xD800); + if(diff > 0x3FF) { return 0; } + uint16_t next_word = data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if(diff2 > 0x3FF) { return 0; } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value>>18) | 0b11110000); + *utf8_output++ = char(((value>>12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf16_to_utf8/utf16_to_utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf8_to_utf16/valid_utf8_to_utf16.h +/* begin file src/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +inline size_t convert_valid(const char* buf, size_t len, char16_t* utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t* start{utf16_output}; + while (pos < len) { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while(pos < final_pos) { + *utf16_output++ = char16_t(buf[pos]); + pos++; + } + continue; + } + } + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if(pos + 1 >= len) { break; } // minimal bound checking + *utf16_output++ = char16_t(((leading_byte &0b00011111) << 6) | (data[pos + 1] &0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if(pos + 2 >= len) { break; } // minimal bound checking + *utf16_output++ = char16_t(((leading_byte &0b00001111) << 12) | ((data[pos + 1] &0b00111111) << 6) | (data[pos + 2] &0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if(pos + 3 >= len) { break; } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18 )| ((data[pos + 1] &0b00111111) << 12) + | ((data[pos + 2] &0b00111111) << 6) | (data[pos + 3] &0b00111111); + code_word -= 0x10000; + *utf16_output++ = char16_t(0xD800 + (code_word >> 10)); + *utf16_output++ = char16_t(0xDC00 + (code_word & 0x3FF)); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf8_to_utf16/utf8_to_utf16.h +/* begin file src/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +inline size_t convert(const char* buf, size_t len, char16_t* utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t* start{utf16_output}; + while (pos < len) { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while(pos < final_pos) { + *utf16_output++ = char16_t(buf[pos]); + pos++; + } + continue; + } + } + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if(pos + 1 >= len) { return 0; } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return 0; } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0x7ff < code_point) { return 0; } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if(pos + 2 >= len) { return 0; } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return 0; } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { return 0; } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || 0xffff < code_point || + (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if(pos + 3 >= len) { return 0; } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return 0; } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { return 0; } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { return 0; } + + // range check + uint32_t code_point = + (leading_byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { return 0; } + code_point -= 0x10000; + *utf16_output++ = char16_t(0xD800 + (code_point >> 10)); + *utf16_output++ = char16_t(0xDC00 + (code_point & 0x3FF)); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf8_to_utf16/utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf8.h +/* begin file src/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { +// credit: based on code from Google Fuchsia (Apache Licensed) +inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { + const uint8_t *data = reinterpret_cast(buf); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 8 bytes are ascii. + uint64_t next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 8 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + if (byte < 0b10000000) { + pos++; + continue; + } else if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { return false; } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if ((code_point < 0x80) || (0x7ff < code_point)) { return false; } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { return false; } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || (0xffff < code_point) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { return false; } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { return false; } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +inline size_t count_code_points(const char* buf, size_t len) { + const int8_t * p = reinterpret_cast(buf); + size_t counter{0}; + for(size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new code point. + if(p[i] > -65) { counter++; } + } + return counter; +} + +inline size_t utf16_length_from_utf8(const char* buf, size_t len) { + const int8_t * p = reinterpret_cast(buf); + size_t counter{0}; + for(size_t i = 0; i < len; i++) { + if(p[i] > -65) { counter++; } + if(uint8_t(p[i]) >= 240) { counter++; } + } + return counter; +} + +} // utf8 namespace +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=scalar/utf16.h +/* begin file src/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16 { + +inline simdutf_warn_unused bool validate(const char16_t *buf, size_t len) noexcept { + const uint16_t *data = reinterpret_cast(buf); + uint64_t pos = 0; + while (pos < len) { + uint16_t word = data[pos]; + if((word &0xF800) == 0xD800) { + if(pos + 1 >= len) { return false; } + uint16_t diff = uint16_t(word - 0xD800); + if(diff > 0x3FF) { return false; } + uint16_t next_word = data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if(diff2 > 0x3FF) { return false; } + pos += 2; + } else { + pos++; + } + } + return true; +} + + +inline size_t count_code_points(const char16_t* buf, size_t len) { + // We are not BOM aware. + const uint16_t * p = reinterpret_cast(buf); + size_t counter{0}; + for(size_t i = 0; i < len; i++) { + counter += ((p[i] & 0xFC00) != 0xDC00); + } + return counter; +} + +inline size_t utf8_length_from_utf16(const char16_t* buf, size_t len) { + // We are not BOM aware. + const uint16_t * p = reinterpret_cast(buf); + size_t counter{0}; + for(size_t i = 0; i < len; i++) { + /** ASCII **/ + if(p[i] <= 0x7F) { counter++; } + /** two-byte **/ + else if(p[i] <= 0x7FF) { counter += 2; } + /** three-byte **/ + else if((p[i] <= 0xD7FF) || (p[i] >= 0xE000)) { counter += 3; } + /** surrogates -- 4 bytes **/ + else { counter += 2; } + } + return counter; +} + +} // utf16 namespace +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file src/scalar/utf16.h */ +// + + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + + +#if SIMDUTF_IMPLEMENTATION_ARM64 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=arm64/implementation.cpp +/* begin file src/arm64/implementation.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/begin.h +/* begin file src/simdutf/arm64/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "arm64" +// #define SIMDUTF_IMPLEMENTATION arm64 +/* end file src/simdutf/arm64/begin.h */ +namespace simdutf { +namespace arm64 { +namespace { +#ifndef SIMDUTF_ARM64_H +#error "arm64.h must be included" +#endif +using namespace simd; + +simdutf_really_inline bool is_ascii(const simd8x64& input) { + simd8 bits = input.reduce_or(); + return bits.max_val() < 0b10000000u; +} + +simdutf_unused simdutf_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); + simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); + simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + // Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller is using ^ as well. + // This will work fine because we only have to report errors for cases with 0-1 lead bytes. + // Multiple lead bytes implies 2 overlapping multibyte characters, and if that happens, there is + // guaranteed to be at least *one* lead byte that is part of only 1 other multibyte character. + // The error will be detected there. + return is_second_byte ^ is_third_byte ^ is_fourth_byte; +} + +simdutf_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); + simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + return is_third_byte ^ is_fourth_byte; +} +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=arm64/arm_convert_utf16_to_utf8.cpp +/* begin file src/arm64/arm_convert_utf16_to_utf8.cpp */ +/* + The vectorized algorithm works on single SSE register i.e., it + loads eight 16-bit words. + + We consider three cases: + 1. an input register contains no surrogates and each value + is in range 0x0000 .. 0x07ff. + 2. an input register contains no surrogates and values are + is in range 0x0000 .. 0xffff. + 3. an input register contains surrogates --- i.e. codepoints + can have 16 or 32 bits. + + Ad 1. + + When values are less than 0x0800, it means that a 16-bit words + can be converted into: 1) single UTF8 byte (when it's an ASCII + char) or 2) two UTF8 bytes. + + For this case we do only some shuffle to obtain these 2-byte + codes and finally compress the whole SSE register with a single + shuffle. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + Ad 2. + + When values fit in 16-bit words, but are above 0x07ff, then + a single word may produce one, two or three UTF8 bytes. + + We prepare data for all these three cases in two registers. + The first register contains lower two UTF8 bytes (used in all + cases), while the second one contains just the third byte for + the three-UTF8-bytes case. + + Finally these two registers are interleaved forming eight-element + array of 32-bit values. The array spans two SSE registers. + The bytes from the registers are compressed using two shuffles. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + + To summarize: + - We need two 256-entry tables that have 8704 bytes in total. +*/ +/* + Returns a pair: the first unprocessed byte from buf and utf8_output + A scalar routing should carry on the conversion of the tail. +*/ +std::pair arm_convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_out) { + uint8_t * utf8_output = reinterpret_cast(utf8_out); + const char16_t* end = buf + len; + + const uint16x8_t v_f800 = vmovq_n_u16((uint16_t)0xf800); + const uint16x8_t v_d800 = vmovq_n_u16((uint16_t)0xd800); + const uint16x8_t v_c080 = vmovq_n_u16((uint16_t)0xc080); + + while (buf + 16 <= end) { + uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); + if(vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!! + // It is common enough that we have sequences of 16 consecutive ASCII characters. + uint16x8_t nextin = vld1q_u16(reinterpret_cast(buf) + 8); + if(vmaxvq_u16(nextin) > 0x7F) { + // 1. pack the bytes + // obviously suboptimal. + uint8x8_t utf8_packed = vmovn_u16(in); + // 2. store (8 bytes) + vst1_u8(utf8_output, utf8_packed); + // 3. adjust pointers + buf += 8; + utf8_output += 8; + in = nextin; + } else { + // 1. pack the bytes + // obviously suboptimal. + uint8x16_t utf8_packed = vmovn_high_u16(vmovn_u16(in), nextin); + // 2. store (16 bytes) + vst1q_u8(utf8_output, utf8_packed); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } + } + + if (vmaxvq_u16(in) <= 0x7FF) { + // 1. prepare 2-byte values + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 + // expected output : [110a|aaaa|10bb|bbbb] x 8 + const uint16x8_t v_1f00 = vmovq_n_u16((int16_t)0x1f00); + const uint16x8_t v_003f = vmovq_n_u16((int16_t)0x003f); + + // t0 = [000a|aaaa|bbbb|bb00] + const uint16x8_t t0 = vshlq_n_u16(in, 2); + // t1 = [000a|aaaa|0000|0000] + const uint16x8_t t1 = vandq_u16(t0, v_1f00); + // t2 = [0000|0000|00bb|bbbb] + const uint16x8_t t2 = vandq_u16(in, v_003f); + // t3 = [000a|aaaa|00bb|bbbb] + const uint16x8_t t3 = vorrq_u16(t1, t2); + // t4 = [110a|aaaa|10bb|bbbb] + const uint16x8_t t4 = vorrq_u16(t3, v_c080); + // 2. merge ASCII and 2-byte codewords + const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F); + const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f); + const uint8x16_t utf8_unpacked = vreinterpretq_u8_u16(vbslq_u16(one_byte_bytemask, in, t4)); + // 3. prepare bitmask for 8-bit lookup +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint16x8_t mask = make_uint16x8_t(0x0001, 0x0004, + 0x0010, 0x0040, + 0x0002, 0x0008, + 0x0020, 0x0080); +#else + const uint16x8_t mask = { 0x0001, 0x0004, + 0x0010, 0x0040, + 0x0002, 0x0008, + 0x0020, 0x0080 }; +#endif + uint16_t m2 = vaddvq_u16(vandq_u16(one_byte_bytemask, mask)); + // 4. pack the bytes + const uint8_t* row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0]; + const uint8x16_t shuffle = vld1q_u8(row + 1); + const uint8x16_t utf8_packed = vqtbl1q_u8(utf8_unpacked, shuffle); + + // 5. store bytes + vst1q_u8(utf8_output, utf8_packed); + + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; + continue; + + } + const uint16x8_t surrogates_bytemask = vceqq_u16(vandq_u16(in, v_f800), v_d800); + // It might seem like checking for surrogates_bitmask == 0xc000 could help. However, + // it is likely an uncommon occurrence. + if (vmaxvq_u16(surrogates_bytemask) == 0) { + // case: words from register produce either 1, 2 or 3 UTF-8 bytes +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint16x8_t dup_even = make_uint16x8_t(0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); +#else + const uint16x8_t dup_even = {0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0a0a, 0x0c0c, 0x0e0e}; +#endif + /* In this branch we handle three cases: + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three UTF-8 bytes + + We expand the input word (16-bit) into two words (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. + + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. + + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. + + Finally from these two words we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ + /** + * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: + * t2 => [0ccc|cccc] [10cc|cccc] + * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) + */ +#define vec(x) vmovq_n_u16(static_cast(x)) + // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] + const uint16x8_t t0 = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(in), vreinterpretq_u8_u16(dup_even))); + // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] + const uint16x8_t t1 = vandq_u16(t0, vec(0b0011111101111111)); + // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] + const uint16x8_t t2 = vorrq_u16 (t1, vec(0b1000000000000000)); + + // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] + const uint16x8_t s0 = vshrq_n_u16(in, 12); + // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] + const uint16x8_t s1 = vandq_u16(in, vec(0b0000111111000000)); + // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000] + const uint16x8_t s1s = vshlq_n_u16(s1, 2); + // [00bb|bbbb|0000|aaaa] + const uint16x8_t s2 = vorrq_u16(s0, s1s); + // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] + const uint16x8_t s3 = vorrq_u16(s2, vec(0b1100000011100000)); + const uint16x8_t v_07ff = vmovq_n_u16((uint16_t)0x07FF); + const uint16x8_t one_or_two_bytes_bytemask = vcleq_u16(in, v_07ff); + const uint16x8_t m0 = vbicq_u16(vec(0b0100000000000000), one_or_two_bytes_bytemask); + const uint16x8_t s4 = veorq_u16(s3, m0); +#undef vec + + // 4. expand words 16-bit => 32-bit + const uint8x16_t out0 = vreinterpretq_u8_u16(vzip1q_u16(t2, s4)); + const uint8x16_t out1 = vreinterpretq_u8_u16(vzip2q_u16(t2, s4)); + + // 5. compress 32-bit words into 1, 2 or 3 bytes -- 2 x shuffle + const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F); + const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f); +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint16x8_t onemask = make_uint16x8_t(0x0001, 0x0004, + 0x0010, 0x0040, + 0x0100, 0x0400, + 0x1000, 0x4000 ); + const uint16x8_t twomask = make_uint16x8_t(0x0002, 0x0008, + 0x0020, 0x0080, + 0x0200, 0x0800, + 0x2000, 0x8000 ); +#else + const uint16x8_t onemask = { 0x0001, 0x0004, + 0x0010, 0x0040, + 0x0100, 0x0400, + 0x1000, 0x4000 }; + const uint16x8_t twomask = { 0x0002, 0x0008, + 0x0020, 0x0080, + 0x0200, 0x0800, + 0x2000, 0x8000 }; +#endif + const uint16x8_t combined = vorrq_u16(vandq_u16(one_byte_bytemask, onemask), vandq_u16(one_or_two_bytes_bytemask, twomask)); + const uint16_t mask = vaddvq_u16(combined); + // The following fast path may or may not be beneficial. + /*if(mask == 0) { + // We only have three-byte words. Use fast path. + const uint8x16_t shuffle = {2,3,1,6,7,5,10,11,9,14,15,13,0,0,0,0}; + const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle); + const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle); + vst1q_u8(utf8_output, utf8_0); + utf8_output += 12; + vst1q_u8(utf8_output, utf8_1); + utf8_output += 12; + buf += 8; + continue; + }*/ + const uint8_t mask0 = uint8_t(mask); + + const uint8_t* row0 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + const uint8x16_t shuffle0 = vld1q_u8(row0 + 1); + const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle0); + + const uint8_t mask1 = static_cast(mask >> 8); + const uint8_t* row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + const uint8x16_t shuffle1 = vld1q_u8(row1 + 1); + const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle1); + + vst1q_u8(utf8_output, utf8_0); + utf8_output += row0[0]; + vst1q_u8(utf8_output, utf8_1); + utf8_output += row1[0]; + + buf += 8; + // surrogate pair(s) in a register + } else { + // Let us do a scalar fallback. + // It may seem wasteful to use scalar code, but being efficient with SIMD + // in the presence of surrogate pairs may require non-trivial tables. + size_t forward = 15; + size_t k = 0; + if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} + for(; k < forward; k++) { + uint16_t word = buf[k]; + if((word & 0xFF80)==0) { + *utf8_output++ = char(word); + } else if((word & 0xF800)==0) { + *utf8_output++ = char((word>>6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else if((word &0xF800 ) != 0xD800) { + *utf8_output++ = char((word>>12) | 0b11100000); + *utf8_output++ = char(((word>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = buf[k+1]; + k++; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if((diff | diff2) > 0x3FF) { return std::make_pair(nullptr, reinterpret_cast(utf8_output)); } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf8_output++ = char((value>>18) | 0b11110000); + *utf8_output++ = char(((value>>12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + } + } + buf += k; + } + } // while + + return std::make_pair(buf, reinterpret_cast(utf8_output)); +} +/* end file src/arm64/arm_convert_utf16_to_utf8.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=arm64/arm_convert_utf8_to_utf16.cpp +/* begin file src/arm64/arm_convert_utf8_to_utf16.cpp */ +// Convert up to 12 bytes from utf8 to utf16 using a mask indicating the +// end of the code points. Only the least significant 12 bits of the mask +// are accessed. +// It returns how many bytes were consumed (up to 12). +size_t convert_masked_utf8_to_utf16(const char *input, + uint64_t utf8_end_of_code_point_mask, + char16_t *&utf16_output) { + // we use an approach where we try to process up to 12 input bytes. + // Why 12 input bytes and not 16? Because we are concerned with the size of + // the lookup tables. Also 12 is nicely divisible by two and three. + // + uint8x16_t in = vld1q_u8(reinterpret_cast(input)); + const uint16_t input_utf8_end_of_code_point_mask = + utf8_end_of_code_point_mask & 0xFFF; + // + // Optimization note: our main path below is load-latency dependent. Thus it is maybe + // beneficial to have fast paths that depend on branch prediction but have less latency. + // This results in more instructions but, potentially, also higher speeds. + // + // We first try a few fast paths. + if((utf8_end_of_code_point_mask & 0xFFFF) == 0xFFFF) { + // We process in chunks of 16 bytes + vst1q_u16(reinterpret_cast(utf16_output), vmovl_u8(vget_low_u8 (in))); + vst1q_u16(reinterpret_cast(utf16_output) + 8, vmovl_high_u8(in)); + utf16_output += 16; // We wrote 16 16-bit characters. + return 16; // We consumed 16 bytes. + } + if((utf8_end_of_code_point_mask & 0xFFFF) == 0xaaaa) { + // We want to take 8 2-byte UTF-8 words and turn them into 8 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint8x16_t sh = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); +#else + const uint8x16_t sh = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}; +#endif + uint8x16_t perm = vqtbl1q_u8(in, sh); + uint8x16_t ascii = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x7f))); + uint8x16_t highbyte = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x1f00))); + uint8x16_t composed = vorrq_u8(ascii, vreinterpretq_u8_u16(vshrq_n_u16(vreinterpretq_u16_u8(highbyte), 2))); + vst1q_u8(reinterpret_cast(utf16_output), composed); + utf16_output += 8; // We wrote 16 bytes, 8 code points. + return 16; + } + if(input_utf8_end_of_code_point_mask == 0x924) { + // We want to take 4 3-byte UTF-8 words and turn them into 4 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO + const uint8x16_t sh = make_uint8x16_t(2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 11, 10, 9, 255); +#else + const uint8x16_t sh = {2, 1, 0, 255, 5, 4, 3, 255, 8, 7, 6, 255, 11, 10, 9, 255}; +#endif + uint8x16_t perm = vqtbl1q_u8(in, sh); + uint8x16_t ascii = + vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x7f))); // 7 or 6 bits + uint8x16_t middlebyte = + vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x3f00))); // 5 or 6 bits + uint8x16_t middlebyte_shifted = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(middlebyte), 2)); + uint32x4_t highbyte = + vreinterpretq_u32_u8(vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x0f0000)))); // 4 bits + uint32x4_t highbyte_shifted = vshrq_n_u32(highbyte, 4); + uint32x4_t composed = + vorrq_u32(vorrq_u32(vreinterpretq_u32_u8(ascii), vreinterpretq_u32_u8(middlebyte_shifted)), highbyte_shifted); + uint16x8_t composed_repacked = vmovn_high_u32(vmovn_u32(composed), composed); + vst1q_u16(reinterpret_cast(utf16_output), composed_repacked); + utf16_output += 4; + return 12; + } + /// We do not have a fast path available, so we fallback. + + const uint8_t idx = + simdutf::tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; + const uint8_t consumed = + simdutf::tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; + + + if (idx < 64) { + // SIX (6) input code-words + // this is a relatively easy scenario + // we process SIX (6) input code-words. The max length in bytes of six code + // words spanning between 1 and 2 bytes each is 12 bytes. + uint8x16_t sh = vld1q_u8(reinterpret_cast(simdutf::tables::utf8_to_utf16::shufutf8[idx])); + uint8x16_t perm = vqtbl1q_u8(in, sh); + uint8x16_t ascii = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x7f))); + uint8x16_t highbyte = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x1f00))); + uint8x16_t composed = vorrq_u8(ascii, vreinterpretq_u8_u16(vshrq_n_u16(vreinterpretq_u16_u8(highbyte), 2))); + vst1q_u8(reinterpret_cast(utf16_output), composed); + utf16_output += 6; // We wrote 12 bytes, 6 code points. + } else if (idx < 145) { + // FOUR (4) input code-words + uint8x16_t sh = vld1q_u8(reinterpret_cast(simdutf::tables::utf8_to_utf16::shufutf8[idx])); + uint8x16_t perm = vqtbl1q_u8(in, sh); + uint8x16_t ascii = + vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x7f))); // 7 or 6 bits + uint8x16_t middlebyte = + vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x3f00))); // 5 or 6 bits + uint8x16_t middlebyte_shifted = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(middlebyte), 2)); + uint32x4_t highbyte = + vreinterpretq_u32_u8(vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x0f0000)))); // 4 bits + uint32x4_t highbyte_shifted = vshrq_n_u32(highbyte, 4); + uint32x4_t composed = + vorrq_u32(vorrq_u32(vreinterpretq_u32_u8(ascii), vreinterpretq_u32_u8(middlebyte_shifted)), highbyte_shifted); + uint16x8_t composed_repacked = vmovn_high_u32(vmovn_u32(composed), composed); + vst1q_u16(reinterpret_cast(utf16_output), composed_repacked); + utf16_output += 4; + } else if (idx < 209) { + // TWO (2) input code-words + uint8x16_t sh = vld1q_u8(reinterpret_cast(simdutf::tables::utf8_to_utf16::shufutf8[idx])); + uint8x16_t perm = vqtbl1q_u8(in, sh); + uint8x16_t ascii = vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x7f))); + uint8x16_t middlebyte = vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x3f00))); + uint8x16_t middlebyte_shifted = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(middlebyte), 2)); + uint8x16_t middlehighbyte = vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x3f0000))); + // correct for spurious high bit + uint8x16_t correct = + vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x400000)))), 1)); + middlehighbyte = veorq_u8(correct, middlehighbyte); + uint8x16_t middlehighbyte_shifted = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(middlehighbyte), 4)); + uint8x16_t highbyte = vandq_u8(perm, vreinterpretq_u8_u32(vmovq_n_u32(0x07000000))); + uint8x16_t highbyte_shifted =vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(highbyte), 6)); + uint8x16_t composed = + vorrq_u8(vorrq_u8(ascii, middlebyte_shifted), + vorrq_u8(highbyte_shifted, middlehighbyte_shifted)); + uint32x4_t composedminus = + vsubq_u32(vreinterpretq_u32_u8(composed), vmovq_n_u32(0x10000)); + uint32x4_t lowtenbits = + vandq_u32(composedminus, vmovq_n_u32(0x3ff)); + uint32x4_t hightenbits = vshrq_n_u32(composedminus, 10); + uint32x4_t lowtenbitsadd = + vaddq_u32(lowtenbits, vmovq_n_u32(0xDC00)); + uint32x4_t hightenbitsadd = + vaddq_u32(hightenbits, vmovq_n_u32(0xD800)); + uint32x4_t lowtenbitsaddshifted = vshlq_n_u32(lowtenbitsadd, 16); + uint32x4_t surrogates = + vorrq_u32(hightenbitsadd, lowtenbitsaddshifted); + uint32_t basic_buffer[4]; + vst1q_u32(basic_buffer, vreinterpretq_u32_u8(composed)); + uint32_t surrogate_buffer[4]; + vst1q_u32(surrogate_buffer, surrogates); + for (size_t i = 0; i < 3; i++) { + if (basic_buffer[i] < 65536) { + utf16_output[0] = uint16_t(basic_buffer[i]); + utf16_output++; + } else { + utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xFFFF); + utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); + utf16_output += 2; + } + } + } else { + // here we know that there is an error but we do not handle errors + } + return consumed; +} +/* end file src/arm64/arm_convert_utf8_to_utf16.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=arm64/arm_validate_utf16le.cpp +/* begin file src/arm64/arm_validate_utf16le.cpp */ + +const char16_t* arm_validate_utf16le(const char16_t* input, size_t size) { + const char16_t* end = input + size; + const auto v_d8 = simd8::splat(0xd8); + const auto v_f8 = simd8::splat(0xf8); + const auto v_fc = simd8::splat(0xfc); + const auto v_dc = simd8::splat(0xdc); + while (input + 16 < end) { + // 0. Load data: since the validation takes into account only higher + // byte of each word, we compress the two vectors into one which + // consists only the higher bytes. + const auto in0 = simd16(input); + const auto in1 = simd16(input + simd16::SIZE / sizeof(char16_t)); + const auto t0 = in0.shr<8>(); + const auto t1 = in1.shr<8>(); + const simd8 in = simd16::pack(t0, t1); + // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). + const auto surrogates_wordmask = ((in & v_f8) == v_d8); + if(surrogates_wordmask.none()) { + input += 16; + } else { + const auto vH = simd8((in & v_fc) == v_dc); + const auto vL = simd8(surrogates_wordmask).bit_andnot(vH); + // We are going to need these later: + const uint8_t low_vh = vH.first(); + const uint8_t high_vl = vL.last(); + // We shift vH down, possibly killing low_vh + const auto sh = simd8({1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,0xFF}); + const auto vHshifteddown = vH.apply_lookup_16_to(sh); + const auto match = vHshifteddown == vL; + // We need to handle the fact that high_vl is unmatched. + // We could use this... + // const uint8x16_t allbutlast = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xFF}; + // match = vorrq_u8(match, allbutlast); + // but sh will do: + const auto fmatch = simd8(simd8(match) | sh); + // We deliberately take these two lines out of the following branchy code + // so that they are always s + if (fmatch.all() && low_vh == 0) { + input += (high_vl == 0) ? 16 : 15; + } else { + return nullptr; + } + } + } + return input; +} +/* end file src/arm64/arm_validate_utf16le.cpp */ + +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/buf_block_reader.h +/* begin file src/generic/buf_block_reader.h */ +namespace simdutf { +namespace arm64 { +namespace { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdutf_really_inline size_t block_index(); + simdutf_really_inline bool has_full_block() const; + simdutf_really_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdutf_really_inline size_t get_remainder(uint8_t *dst) const; + simdutf_really_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text_64(const uint8_t *text) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text(const simd8x64& in) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdutf_unused static char * format_mask(uint64_t mask) { + static char *buf = reinterpret_cast(malloc(64 + 1)); + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdutf_really_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdutf_really_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdutf_really_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdutf_really_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/buf_block_reader.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_lookup4_algorithm.h +/* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +namespace simdutf { +namespace arm64 { +namespace { +namespace utf8_validation { + +using namespace simd; + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdutf_really_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + }; + const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdutf_really_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdutf_really_inline void check_next_input(const simd8x64& input) { + if(simdutf_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; + + } + } + // do not forget to call check_eof! + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // namespace utf8_validation + +using utf8_validation::utf8_checker; + +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_validator.h +/* begin file src/generic/utf8_validation/utf8_validator.h */ +namespace simdutf { +namespace arm64 { +namespace { +namespace utf8_validation { + +/** + * Validates that the string is actual UTF-8. + */ +template +bool generic_validate_utf8(const uint8_t * input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return !c.errors(); +} + +bool generic_validate_utf8(const char * input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input),length); +} + +} // namespace utf8_validation +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_validator.h */ +// transcoding from UTF-8 to UTF-16 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/valid_utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ + + +namespace simdutf { +namespace arm64 { +namespace { +namespace utf8_to_utf16 { + +using namespace simd; + + +simdutf_warn_unused size_t convert_valid(const char* input, size_t size, + char16_t* utf16_output) noexcept { + // The implementation is not specific to haswell and should be moved to the generic directory. + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + // this loop could be unrolled further. For example, we could process the mask + // far more than 64 bytes. + // + // For pure ASCII inputs, this function is not optimally fast because they are + // faster ways to just check for ASCII than to compute the continuation mask. + // However, the continuation mask is more informative. There might be a trade-off + // involved. + // + simd8x64 in(reinterpret_cast(input + pos)); + uint64_t utf8_continuation_mask = in.lt(-65 + 1); + // -65 is 0b10111111 in two-complement's, so largest possible continuation byte + if(utf8_continuation_mask != 0) { + // Slow path. We hope that the compiler will recognize that this is a slow path. + // Anything that is not a continuation mask is a 'leading byte', that is, the + // start of a new code point. + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + // The *start* of code points is not so useful, rather, we want the *end* of code points. + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times when using solely + // the slow/regular path, and at least four times if there are fast paths. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + // + // Thus we may allow convert_masked_utf8_to_utf16 to process + // more bytes at a time under a fast-path mode where 16 bytes + // are consumed at once (e.g., when encountering ASCII). + size_t consumed = convert_masked_utf8_to_utf16(input + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } else { + in.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } + } + utf16_output += scalar::utf8_to_utf16::convert_valid(input + pos, size - pos, utf16_output); + return utf16_output - start; +} + + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ + + +namespace simdutf { +namespace arm64 { +namespace { +namespace utf8_to_utf16 { +using namespace simd; + + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + + struct validating_transcoder { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + + validating_transcoder() : error(uint8_t(0)) {} + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + + + simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) { + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + simd8x64 input(reinterpret_cast(in + pos)); + if(input.is_ascii()) { + input.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + auto zero = simd8{uint8_t(0)}; + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + size_t consumed = convert_masked_utf8_to_utf16(in + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } + } + if(errors()) { return 0; } + if(pos < size) { + size_t howmany = scalar::utf8_to_utf16::convert(in + pos, size - pos, utf16_output); + if(howmany == 0) { return 0; } + utf16_output += howmany; + } + return utf16_output - start; + } + + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ +// other functions +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8.h +/* begin file src/generic/utf8.h */ + +namespace simdutf { +namespace arm64 { +namespace { +namespace utf8 { + +using namespace simd; + +simdutf_really_inline size_t count_code_points(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + count += 64 - count_ones(utf8_continuation_mask); + } + return count + scalar::utf8::count_code_points(in + pos, size - pos); +} + + +simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + // We count one word for anything that is not a continuation (so + // leading bytes). + count += 64 - count_ones(utf8_continuation_mask); + int64_t utf8_4byte = input.gteq_unsigned(240); + count += count_ones(utf8_4byte); + } + return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); +} +} // utf8 namespace +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf16.h +/* begin file src/generic/utf16.h */ +#include +namespace simdutf { +namespace arm64 { +namespace { +namespace utf16 { + +simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); + count += count_ones(not_pair) / 2; + } + return count + scalar::utf16::count_code_points(in + pos, size - pos); +} +simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t ascii_mask = input.lteq(0x7F); + uint64_t twobyte_mask = input.lteq(0x7FF); + uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); + + size_t ascii_count = count_ones(ascii_mask) / 2; + size_t twobyte_count = count_ones(twobyte_mask & ~ ascii_mask) / 2; + size_t threebyte_count = count_ones(not_pair_mask & ~ twobyte_mask) / 2; + size_t fourbyte_count = 32 - count_ones(not_pair_mask) / 2; + count += 2 * fourbyte_count + 3 * threebyte_count + 2 * twobyte_count + ascii_count; + } + return count + scalar::utf16::utf8_length_from_utf16(in + pos, size - pos); +} +} // utf16 +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf16.h */ +// +// Implementation-specific overrides +// +namespace simdutf { +namespace arm64 { + +simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return arm64::utf8_validation::generic_validate_utf8(buf,len); +} + +simdutf_warn_unused bool implementation::validate_utf16(const char16_t *buf, size_t len) const noexcept { + const char16_t* tail = arm_validate_utf16le(buf, len); + if (tail) { + return scalar::utf16::validate(tail, len - (tail - buf)); + } else { + return false; + } +} + +simdutf_warn_unused size_t implementation::convert_utf8_to_utf16(const char* buf, size_t len, char16_t* utf16_output) const noexcept { + utf8_to_utf16::validating_transcoder converter; + return converter.convert(buf, len, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf16(const char* input, size_t size, + char16_t* utf16_output) const noexcept { + return utf8_to_utf16::convert_valid(input, size, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + std::pair ret = arm_convert_utf16_to_utf8(buf, len, utf8_output); + if (ret.first == nullptr) { return 0; } + size_t saved_bytes = ret.second - utf8_output; + if (ret.first != buf + len) { + const size_t scalar_saved_bytes = scalar::utf16_to_utf8::convert( + ret.first, len - (ret.first - buf), ret.second); + if (scalar_saved_bytes == 0) { return 0; } + saved_bytes += scalar_saved_bytes; + } + return saved_bytes; +} + +simdutf_warn_unused size_t implementation::convert_valid_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return convert_utf16_to_utf8(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::count_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::count_utf8(const char * input, size_t length) const noexcept { + return utf8::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::utf8_length_from_utf16(input, length); +} + +simdutf_warn_unused size_t implementation::utf16_length_from_utf8(const char * input, size_t length) const noexcept { + return utf8::utf16_length_from_utf8(input, length); +} + +} // namespace arm64 +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/arm64/end.h +/* begin file src/simdutf/arm64/end.h */ +/* end file src/simdutf/arm64/end.h */ +/* end file src/arm64/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_FALLBACK +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=fallback/implementation.cpp +/* begin file src/fallback/implementation.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/begin.h +/* begin file src/simdutf/fallback/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "fallback" +// #define SIMDUTF_IMPLEMENTATION fallback +/* end file src/simdutf/fallback/begin.h */ + + +namespace simdutf { +namespace fallback { + +simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return scalar::utf8::validate(buf, len); +} + +simdutf_warn_unused bool implementation::validate_utf16(const char16_t *buf, size_t len) const noexcept { + return scalar::utf16::validate(buf, len); +} + +simdutf_warn_unused size_t implementation::convert_utf8_to_utf16(const char* buf, size_t len, char16_t* utf16_output) const noexcept { + return scalar::utf8_to_utf16::convert(buf, len, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf16(const char* buf, size_t len, char16_t* utf16_output) const noexcept { + return scalar::utf8_to_utf16::convert_valid(buf, len, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return scalar::utf16_to_utf8::convert(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return scalar::utf16_to_utf8::convert_valid(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::count_utf16(const char16_t * input, size_t length) const noexcept { + return scalar::utf16::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::count_utf8(const char * input, size_t length) const noexcept { + return scalar::utf8::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept { + return scalar::utf16::utf8_length_from_utf16(input, length); +} + +simdutf_warn_unused size_t implementation::utf16_length_from_utf8(const char * input, size_t length) const noexcept { + return scalar::utf8::utf16_length_from_utf8(input, length); +} + +} // namespace fallback +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/fallback/end.h +/* begin file src/simdutf/fallback/end.h */ +/* end file src/simdutf/fallback/end.h */ +/* end file src/fallback/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_HASWELL +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=haswell/implementation.cpp +/* begin file src/haswell/implementation.cpp */ + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/begin.h +/* begin file src/simdutf/haswell/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "haswell" +// #define SIMDUTF_IMPLEMENTATION haswell +SIMDUTF_TARGET_HASWELL +/* end file src/simdutf/haswell/begin.h */ +namespace simdutf { +namespace haswell { +namespace { +#ifndef SIMDUTF_HASWELL_H +#error "haswell.h must be included" +#endif +using namespace simd; + + +simdutf_really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); +} + +simdutf_unused simdutf_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); +} + +simdutf_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=haswell/avx2_convert_utf8_to_utf16.cpp +/* begin file src/haswell/avx2_convert_utf8_to_utf16.cpp */ +// depends on "tables/utf8_to_utf16_tables.h" + + +// Convert up to 12 bytes from utf8 to utf16 using a mask indicating the +// end of the code points. Only the least significant 12 bits of the mask +// are accessed. +// It returns how many bytes were consumed (up to 12). +size_t convert_masked_utf8_to_utf16(const char *input, + uint64_t utf8_end_of_code_point_mask, + char16_t *&utf16_output) { + // we use an approach where we try to process up to 12 input bytes. + // Why 12 input bytes and not 16? Because we are concerned with the size of + // the lookup tables. Also 12 is nicely divisible by two and three. + // + // + // Optimization note: our main path below is load-latency dependent. Thus it is maybe + // beneficial to have fast paths that depend on branch prediction but have less latency. + // This results in more instructions but, potentially, also higher speeds. + // + // We first try a few fast paths. + const __m128i in = _mm_loadu_si128((__m128i *)input); + const uint16_t input_utf8_end_of_code_point_mask = + utf8_end_of_code_point_mask & 0xFFF; + if(((utf8_end_of_code_point_mask & 0xFFFF) == 0xFFFF)) { + // We process the data in chunks of 16 bytes. + _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf16_output), _mm256_cvtepu8_epi16(in)); + utf16_output += 16; // We wrote 16 16-bit characters. + return 16; // We consumed 16 bytes. + } + if(((utf8_end_of_code_point_mask & 0xFFFF) == 0xaaaa)) { + // We want to take 8 2-byte UTF-8 words and turn them into 8 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. + const __m128i sh = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); + const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); + _mm_storeu_si128((__m128i *)utf16_output, composed); + utf16_output += 8; // We wrote 16 bytes, 8 code points. + return 16; + } + if(input_utf8_end_of_code_point_mask == 0x924) { + // We want to take 4 3-byte UTF-8 words and turn them into 4 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. + const __m128i sh = _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = + _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits + const __m128i middlebyte = + _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + const __m128i highbyte = + _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); + const __m128i composed_repacked = _mm_packus_epi32(composed, composed); + _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); + utf16_output += 4; + return 12; + } + + const uint8_t idx = + simdutf::tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; + const uint8_t consumed = + simdutf::tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; + if (idx < 64) { + // SIX (6) input code-words + // this is a relatively easy scenario + // we process SIX (6) input code-words. The max length in bytes of six code + // words spanning between 1 and 2 bytes each is 12 bytes. On processors + // where pdep/pext is fast, we might be able to use a small lookup table. + const __m128i sh = + _mm_loadu_si128((const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); + const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); + _mm_storeu_si128((__m128i *)utf16_output, composed); + utf16_output += 6; // We wrote 12 bytes, 6 code points. + } else if (idx < 145) { + // FOUR (4) input code-words + const __m128i sh = + _mm_loadu_si128((const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = + _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits + const __m128i middlebyte = + _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + const __m128i highbyte = + _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); + const __m128i composed_repacked = _mm_packus_epi32(composed, composed); + _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); + utf16_output += 4; + } else if (idx < 209) { + // TWO (2) input code-words + const __m128i sh = + _mm_loadu_si128((const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); + const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); + // correct for spurious high bit + const __m128i correct = + _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); + middlehighbyte = _mm_xor_si128(correct, middlehighbyte); + const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0x07000000)); + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), + _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); + const __m128i composedminus = + _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); + const __m128i lowtenbits = + _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); + const __m128i hightenbits = _mm_srli_epi32(composedminus, 10); + const __m128i lowtenbitsadd = + _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); + const __m128i hightenbitsadd = + _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); + const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); + const __m128i surrogates = + _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); + uint32_t basic_buffer[4]; + _mm_storeu_si128((__m128i *)basic_buffer, composed); + uint32_t surrogate_buffer[4]; + _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); + for (size_t i = 0; i < 3; i++) { + if (basic_buffer[i] < 65536) { + utf16_output[0] = uint16_t(basic_buffer[i]); + utf16_output++; + } else { + utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xFFFF); + utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); + utf16_output += 2; + } + } + } else { + // here we know that there is an error but we do not handle errors + } + return consumed; +} +/* end file src/haswell/avx2_convert_utf8_to_utf16.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=haswell/avx2_validate_utf16le.cpp +/* begin file src/haswell/avx2_validate_utf16le.cpp */ +/* + In UTF-16 words in range 0xD800 to 0xDFFF have special meaning. + + In a vectorized algorithm we want to examine the most significant + nibble in order to select a fast path. If none of highest nibbles + are 0xD (13), than we are sure that UTF-16 chunk in a vector + register is valid. + + Let us analyze what we need to check if the nibble is 0xD. The + value of the preceding nibble determines what we have: + + 0xd000 .. 0xd7ff - a valid word + 0xd800 .. 0xdbff - low surrogate + 0xdc00 .. 0xdfff - high surrogate + + Other constraints we have to consider: + - there must not be two consecutive low surrogates (0xd800 .. 0xdbff) + - there must not be two consecutive high surrogates (0xdc00 .. 0xdfff) + - there must not be sole low surrogate nor high surrogate + + We're going to build three bitmasks based on the 3rd nibble: + - V = valid word, + - L = low surrogate (0xd800 .. 0xdbff) + - H = high surrogate (0xdc00 .. 0xdfff) + + 0 1 2 3 4 5 6 7 <--- word index + [ V | L | H | L | H | V | V | L ] + 1 0 0 0 0 1 1 0 - V = valid masks + 0 1 0 1 0 0 0 1 - L = low surrogate + 0 0 1 0 1 0 0 0 - H high surrogate + + + 1 0 0 0 0 1 1 0 V = valid masks + 0 1 0 1 0 0 0 0 a = L & (H >> 1) + 0 0 1 0 1 0 0 0 b = a << 1 + 1 1 1 1 1 1 1 0 c = V | a | b + ^ + the last bit can be zero, we just consume 7 words + and recheck this word in the next iteration +*/ + +/* Returns: + - pointer to the last unprocessed character (a scalar fallback should check the rest); + - nullptr if an error was detected. +*/ +const char16_t* avx2_validate_utf16le(const char16_t* input, size_t size) { + const char16_t* end = input + size; + + const auto v_d8 = simd8::splat(0xd8); + const auto v_f8 = simd8::splat(0xf8); + const auto v_fc = simd8::splat(0xfc); + const auto v_dc = simd8::splat(0xdc); + + while (input + simd16::ELEMENTS * 2 < end) { + // 0. Load data: since the validation takes into account only higher + // byte of each word, we compress the two vectors into one which + // consists only the higher bytes. + const auto in0 = simd16(input); + const auto in1 = simd16(input + simd16::ELEMENTS); + + const auto t0 = in0.shr<8>(); + const auto t1 = in1.shr<8>(); + + const auto in = simd16::pack(t0, t1); + + // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). + const auto surrogates_wordmask = (in & v_f8) == v_d8; + const uint32_t surrogates_bitmask = surrogates_wordmask.to_bitmask(); + if (surrogates_bitmask == 0x0) { + input += simd16::ELEMENTS * 2; + } else { + // 2. We have some surrogates that have to be distinguished: + // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) + // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) + // + // Fact: high surrogate has 11th bit set (3rd bit in the higher word) + + // V - non-surrogate words + // V = not surrogates_wordmask + const uint32_t V = ~surrogates_bitmask; + + // H - word-mask for high surrogates: the six highest bits are 0b1101'11 + const auto vH = (in & v_fc) == v_dc; + const uint32_t H = vH.to_bitmask(); + + // L - word mask for low surrogates + // L = not H and surrogates_wordmask + const uint32_t L = ~H & surrogates_bitmask; + + const uint32_t a = L & (H >> 1); // A low surrogate must be followed by high one. + // (A low surrogate placed in the 7th register's word + // is an exception we handle.) + const uint32_t b = a << 1; // Just mark that the opposite fact is hold, + // thanks to that we have only two masks for valid case. + const uint32_t c = V | a | b; // Combine all the masks into the final one. + + if (c == 0xffffffff) { + // The whole input register contains valid UTF-16, i.e., + // either single words or proper surrogate pairs. + input += simd16::ELEMENTS * 2; + } else if (c == 0x7fffffff) { + // The 31 lower words of the input register contains valid UTF-16. + // The 31 word may be either a low or high surrogate. It the next + // iteration we 1) check if the low surrogate is followed by a high + // one, 2) reject sole high surrogate. + input += simd16::ELEMENTS * 2 - 1; + } else { + return nullptr; + } + } + } + + return input; +} +/* end file src/haswell/avx2_validate_utf16le.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=haswell/avx2_convert_utf16_to_utf8.cpp +/* begin file src/haswell/avx2_convert_utf16_to_utf8.cpp */ +/* + The vectorized algorithm works on single SSE register i.e., it + loads eight 16-bit words. + + We consider three cases: + 1. an input register contains no surrogates and each value + is in range 0x0000 .. 0x07ff. + 2. an input register contains no surrogates and values are + is in range 0x0000 .. 0xffff. + 3. an input register contains surrogates --- i.e. codepoints + can have 16 or 32 bits. + + Ad 1. + + When values are less than 0x0800, it means that a 16-bit words + can be converted into: 1) single UTF8 byte (when it's an ASCII + char) or 2) two UTF8 bytes. + + For this case we do only some shuffle to obtain these 2-byte + codes and finally compress the whole SSE register with a single + shuffle. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + Ad 2. + + When values fit in 16-bit words, but are above 0x07ff, then + a single word may produce one, two or three UTF8 bytes. + + We prepare data for all these three cases in two registers. + The first register contains lower two UTF8 bytes (used in all + cases), while the second one contains just the third byte for + the three-UTF8-bytes case. + + Finally these two registers are interleaved forming eight-element + array of 32-bit values. The array spans two SSE registers. + The bytes from the registers are compressed using two shuffles. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + + To summarize: + - We need two 256-entry tables that have 8704 bytes in total. +*/ + + +/* + Returns a pair: the first unprocessed byte from buf and utf8_output + A scalar routing should carry on the conversion of the tail. +*/ +std::pair sse_convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) { + const char16_t* end = buf + len; + const __m256i v_0000 = _mm256_setzero_si256(); + const __m256i v_f800 = _mm256_set1_epi16((int16_t)0xf800); + const __m256i v_d800 = _mm256_set1_epi16((int16_t)0xd800); + const __m256i v_c080 = _mm256_set1_epi16((int16_t)0xc080); + const size_t safety_margin = 8; // to avoid overruns + + while (buf + 16 + safety_margin <= end) { + __m256i in = _mm256_loadu_si256((__m256i*)buf); + // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes + const __m256i v_ff80 = _mm256_set1_epi16((int16_t)0xff80); + if(_mm256_testz_si256(in, v_ff80)) { // ASCII fast path!!!! + // 1. pack the bytes + const __m128i utf8_packed = _mm_packus_epi16(_mm256_castsi256_si128(in),_mm256_extractf128_si256(in,1)); + // 2. store (16 bytes) + _mm_storeu_si128((__m128i*)utf8_output, utf8_packed); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } + // no bits set above 7th bit + const __m256i one_byte_bytemask = _mm256_cmpeq_epi16(_mm256_and_si256(in, v_ff80), v_0000); + const uint32_t one_byte_bitmask = static_cast(_mm256_movemask_epi8(one_byte_bytemask)); + + // no bits set above 11th bit + const __m256i one_or_two_bytes_bytemask = _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_0000); + const uint32_t one_or_two_bytes_bitmask = static_cast(_mm256_movemask_epi8(one_or_two_bytes_bytemask)); + if (one_or_two_bytes_bitmask == 0xffffffff) { + + // 1. prepare 2-byte values + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 + // expected output : [110a|aaaa|10bb|bbbb] x 8 + const __m256i v_1f00 = _mm256_set1_epi16((int16_t)0x1f00); + const __m256i v_003f = _mm256_set1_epi16((int16_t)0x003f); + + // t0 = [000a|aaaa|bbbb|bb00] + const __m256i t0 = _mm256_slli_epi16(in, 2); + // t1 = [000a|aaaa|0000|0000] + const __m256i t1 = _mm256_and_si256(t0, v_1f00); + // t2 = [0000|0000|00bb|bbbb] + const __m256i t2 = _mm256_and_si256(in, v_003f); + // t3 = [000a|aaaa|00bb|bbbb] + const __m256i t3 = _mm256_or_si256(t1, t2); + // t4 = [110a|aaaa|10bb|bbbb] + const __m256i t4 = _mm256_or_si256(t3, v_c080); + + // 2. merge ASCII and 2-byte codewords + const __m256i utf8_unpacked = _mm256_blendv_epi8(t4, in, one_byte_bytemask); + + // 3. prepare bitmask for 8-bit lookup + const uint32_t M0 = one_byte_bitmask & 0x55555555; + const uint32_t M1 = M0 >> 7; + const uint32_t M2 = (M1 | M0) & 0x00ff00ff; + // 4. pack the bytes + + const uint8_t* row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2)][0]; + const uint8_t* row_2 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2>>16)][0]; + + const __m128i shuffle = _mm_loadu_si128((__m128i*)(row + 1)); + const __m128i shuffle_2 = _mm_loadu_si128((__m128i*)(row_2 + 1)); + + const __m256i utf8_packed = _mm256_shuffle_epi8(utf8_unpacked, _mm256_setr_m128i(shuffle,shuffle_2)); + // 5. store bytes + _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_packed)); + utf8_output += row[0]; + _mm_storeu_si128((__m128i*)utf8_output, _mm256_extractf128_si256(utf8_packed,1)); + utf8_output += row_2[0]; + + // 6. adjust pointers + buf += 16; + continue; + } + // 1. Check if there are any surrogate word in the input chunk. + // We have also deal with situation when there is a suggogate word + // at the end of a chunk. + const __m256i surrogates_bytemask = _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_d800); + + // bitmask = 0x0000 if there are no surrogates + // = 0xc000 if the last word is a surrogate + const uint32_t surrogates_bitmask = static_cast(_mm256_movemask_epi8(surrogates_bytemask)); + // It might seem like checking for surrogates_bitmask == 0xc000 could help. However, + // it is likely an uncommon occurrence. + if (surrogates_bitmask == 0x00000000) { + // case: words from register produce either 1, 2 or 3 UTF-8 bytes + const __m256i dup_even = _mm256_setr_epi16(0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0a0a, 0x0c0c, 0x0e0e, + 0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); + + /* In this branch we handle three cases: + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three UTF-8 bytes + + We expand the input word (16-bit) into two words (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. + + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. + + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. + + Finally from these two words we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ + /** + * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: + * t2 => [0ccc|cccc] [10cc|cccc] + * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) + */ +#define vec(x) _mm256_set1_epi16(static_cast(x)) + // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] + const __m256i t0 = _mm256_shuffle_epi8(in, dup_even); + // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] + const __m256i t1 = _mm256_and_si256(t0, vec(0b0011111101111111)); + // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] + const __m256i t2 = _mm256_or_si256 (t1, vec(0b1000000000000000)); + + // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] + const __m256i s0 = _mm256_srli_epi16(in, 4); + // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] + const __m256i s1 = _mm256_and_si256(s0, vec(0b0000111111111100)); + // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] + const __m256i s2 = _mm256_maddubs_epi16(s1, vec(0x0140)); + // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] + const __m256i s3 = _mm256_or_si256(s2, vec(0b1100000011100000)); + const __m256i m0 = _mm256_andnot_si256(one_or_two_bytes_bytemask, vec(0b0100000000000000)); + const __m256i s4 = _mm256_xor_si256(s3, m0); +#undef vec + + // 4. expand words 16-bit => 32-bit + const __m256i out0 = _mm256_unpacklo_epi16(t2, s4); + const __m256i out1 = _mm256_unpackhi_epi16(t2, s4); + + // 5. compress 32-bit words into 1, 2 or 3 bytes -- 2 x shuffle + const uint32_t mask = (one_byte_bitmask & 0x55555555) | + (one_or_two_bytes_bitmask & 0xaaaaaaaa); + // Due to the wider registers, the following path is less likely to be useful. + /*if(mask == 0) { + // We only have three-byte words. Use fast path. + const __m256i shuffle = _mm256_setr_epi8(2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1, 2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1); + const __m256i utf8_0 = _mm256_shuffle_epi8(out0, shuffle); + const __m256i utf8_1 = _mm256_shuffle_epi8(out1, shuffle); + _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_0)); + utf8_output += 12; + _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_1)); + utf8_output += 12; + _mm_storeu_si128((__m128i*)utf8_output, _mm256_extractf128_si256(utf8_0,1)); + utf8_output += 12; + _mm_storeu_si128((__m128i*)utf8_output, _mm256_extractf128_si256(utf8_1,1)); + utf8_output += 12; + buf += 16; + continue; + }*/ + const uint8_t mask0 = uint8_t(mask); + const uint8_t* row0 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + const __m128i shuffle0 = _mm_loadu_si128((__m128i*)(row0 + 1)); + const __m128i utf8_0 = _mm_shuffle_epi8(_mm256_castsi256_si128(out0), shuffle0); + + const uint8_t mask1 = static_cast(mask >> 8); + const uint8_t* row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + const __m128i shuffle1 = _mm_loadu_si128((__m128i*)(row1 + 1)); + const __m128i utf8_1 = _mm_shuffle_epi8(_mm256_castsi256_si128(out1), shuffle1); + + const uint8_t mask2 = static_cast(mask >> 16); + const uint8_t* row2 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask2][0]; + const __m128i shuffle2 = _mm_loadu_si128((__m128i*)(row2 + 1)); + const __m128i utf8_2 = _mm_shuffle_epi8(_mm256_extractf128_si256(out0,1), shuffle2); + + + const uint8_t mask3 = static_cast(mask >> 24); + const uint8_t* row3 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask3][0]; + const __m128i shuffle3 = _mm_loadu_si128((__m128i*)(row3 + 1)); + const __m128i utf8_3 = _mm_shuffle_epi8(_mm256_extractf128_si256(out1,1), shuffle3); + + _mm_storeu_si128((__m128i*)utf8_output, utf8_0); + utf8_output += row0[0]; + _mm_storeu_si128((__m128i*)utf8_output, utf8_1); + utf8_output += row1[0]; + _mm_storeu_si128((__m128i*)utf8_output, utf8_2); + utf8_output += row2[0]; + _mm_storeu_si128((__m128i*)utf8_output, utf8_3); + utf8_output += row3[0]; + buf += 16; + // surrogate pair(s) in a register + } else { + // Let us do a scalar fallback. + // It may seem wasteful to use scalar code, but being efficient with SIMD + // in the presence of surrogate pairs may require non-trivial tables. + size_t forward = 15; + size_t k = 0; + if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} + for(; k < forward; k++) { + uint16_t word = buf[k]; + if((word & 0xFF80)==0) { + *utf8_output++ = char(word); + } else if((word & 0xF800)==0) { + *utf8_output++ = char((word>>6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else if((word &0xF800 ) != 0xD800) { + *utf8_output++ = char((word>>12) | 0b11100000); + *utf8_output++ = char(((word>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = buf[k+1]; + k++; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if((diff | diff2) > 0x3FF) { return std::make_pair(nullptr, utf8_output); } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf8_output++ = char((value>>18) | 0b11110000); + *utf8_output++ = char(((value>>12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + } + } + buf += k; + } + } // while + return std::make_pair(buf, utf8_output); +} +/* end file src/haswell/avx2_convert_utf16_to_utf8.cpp */ + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/buf_block_reader.h +/* begin file src/generic/buf_block_reader.h */ +namespace simdutf { +namespace haswell { +namespace { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdutf_really_inline size_t block_index(); + simdutf_really_inline bool has_full_block() const; + simdutf_really_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdutf_really_inline size_t get_remainder(uint8_t *dst) const; + simdutf_really_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text_64(const uint8_t *text) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text(const simd8x64& in) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdutf_unused static char * format_mask(uint64_t mask) { + static char *buf = reinterpret_cast(malloc(64 + 1)); + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdutf_really_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdutf_really_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdutf_really_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdutf_really_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/buf_block_reader.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_lookup4_algorithm.h +/* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +namespace simdutf { +namespace haswell { +namespace { +namespace utf8_validation { + +using namespace simd; + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdutf_really_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + }; + const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdutf_really_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdutf_really_inline void check_next_input(const simd8x64& input) { + if(simdutf_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; + + } + } + // do not forget to call check_eof! + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // namespace utf8_validation + +using utf8_validation::utf8_checker; + +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_validator.h +/* begin file src/generic/utf8_validation/utf8_validator.h */ +namespace simdutf { +namespace haswell { +namespace { +namespace utf8_validation { + +/** + * Validates that the string is actual UTF-8. + */ +template +bool generic_validate_utf8(const uint8_t * input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return !c.errors(); +} + +bool generic_validate_utf8(const char * input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input),length); +} + +} // namespace utf8_validation +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_validator.h */ +// transcoding from UTF-8 to UTF-16 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/valid_utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ + + +namespace simdutf { +namespace haswell { +namespace { +namespace utf8_to_utf16 { + +using namespace simd; + + +simdutf_warn_unused size_t convert_valid(const char* input, size_t size, + char16_t* utf16_output) noexcept { + // The implementation is not specific to haswell and should be moved to the generic directory. + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + // this loop could be unrolled further. For example, we could process the mask + // far more than 64 bytes. + // + // For pure ASCII inputs, this function is not optimally fast because they are + // faster ways to just check for ASCII than to compute the continuation mask. + // However, the continuation mask is more informative. There might be a trade-off + // involved. + // + simd8x64 in(reinterpret_cast(input + pos)); + uint64_t utf8_continuation_mask = in.lt(-65 + 1); + // -65 is 0b10111111 in two-complement's, so largest possible continuation byte + if(utf8_continuation_mask != 0) { + // Slow path. We hope that the compiler will recognize that this is a slow path. + // Anything that is not a continuation mask is a 'leading byte', that is, the + // start of a new code point. + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + // The *start* of code points is not so useful, rather, we want the *end* of code points. + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times when using solely + // the slow/regular path, and at least four times if there are fast paths. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + // + // Thus we may allow convert_masked_utf8_to_utf16 to process + // more bytes at a time under a fast-path mode where 16 bytes + // are consumed at once (e.g., when encountering ASCII). + size_t consumed = convert_masked_utf8_to_utf16(input + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } else { + in.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } + } + utf16_output += scalar::utf8_to_utf16::convert_valid(input + pos, size - pos, utf16_output); + return utf16_output - start; +} + + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ + + +namespace simdutf { +namespace haswell { +namespace { +namespace utf8_to_utf16 { +using namespace simd; + + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + + struct validating_transcoder { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + + validating_transcoder() : error(uint8_t(0)) {} + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + + + simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) { + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + simd8x64 input(reinterpret_cast(in + pos)); + if(input.is_ascii()) { + input.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + auto zero = simd8{uint8_t(0)}; + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + size_t consumed = convert_masked_utf8_to_utf16(in + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } + } + if(errors()) { return 0; } + if(pos < size) { + size_t howmany = scalar::utf8_to_utf16::convert(in + pos, size - pos, utf16_output); + if(howmany == 0) { return 0; } + utf16_output += howmany; + } + return utf16_output - start; + } + + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ +// other functions +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8.h +/* begin file src/generic/utf8.h */ + +namespace simdutf { +namespace haswell { +namespace { +namespace utf8 { + +using namespace simd; + +simdutf_really_inline size_t count_code_points(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + count += 64 - count_ones(utf8_continuation_mask); + } + return count + scalar::utf8::count_code_points(in + pos, size - pos); +} + + +simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + // We count one word for anything that is not a continuation (so + // leading bytes). + count += 64 - count_ones(utf8_continuation_mask); + int64_t utf8_4byte = input.gteq_unsigned(240); + count += count_ones(utf8_4byte); + } + return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); +} +} // utf8 namespace +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf16.h +/* begin file src/generic/utf16.h */ +#include +namespace simdutf { +namespace haswell { +namespace { +namespace utf16 { + +simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); + count += count_ones(not_pair) / 2; + } + return count + scalar::utf16::count_code_points(in + pos, size - pos); +} +simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t ascii_mask = input.lteq(0x7F); + uint64_t twobyte_mask = input.lteq(0x7FF); + uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); + + size_t ascii_count = count_ones(ascii_mask) / 2; + size_t twobyte_count = count_ones(twobyte_mask & ~ ascii_mask) / 2; + size_t threebyte_count = count_ones(not_pair_mask & ~ twobyte_mask) / 2; + size_t fourbyte_count = 32 - count_ones(not_pair_mask) / 2; + count += 2 * fourbyte_count + 3 * threebyte_count + 2 * twobyte_count + ascii_count; + } + return count + scalar::utf16::utf8_length_from_utf16(in + pos, size - pos); +} +} // utf16 +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf16.h */ + +namespace simdutf { +namespace haswell { + + +simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return haswell::utf8_validation::generic_validate_utf8(buf,len); +} + +simdutf_warn_unused bool implementation::validate_utf16(const char16_t *buf, size_t len) const noexcept { + const char16_t* tail = avx2_validate_utf16le(buf, len); + if (tail) { + return scalar::utf16::validate(tail, len - (tail - buf)); + } else { + return false; + } +} + + +simdutf_warn_unused size_t implementation::convert_utf8_to_utf16(const char* buf, size_t len, char16_t* utf16_output) const noexcept { + utf8_to_utf16::validating_transcoder converter; + return converter.convert(buf, len, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf16(const char* input, size_t size, + char16_t* utf16_output) const noexcept { + return utf8_to_utf16::convert_valid(input, size, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + std::pair ret = haswell::sse_convert_utf16_to_utf8(buf, len, utf8_output); + if (ret.first == nullptr) { return 0; } + size_t saved_bytes = ret.second - utf8_output; + if (ret.first != buf + len) { + const size_t scalar_saved_bytes = scalar::utf16_to_utf8::convert( + ret.first, len - (ret.first - buf), ret.second); + if (scalar_saved_bytes == 0) { return 0; } + saved_bytes += scalar_saved_bytes; + } + return saved_bytes; +} + +simdutf_warn_unused size_t implementation::convert_valid_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return convert_utf16_to_utf8(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::count_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::count_utf8(const char * input, size_t length) const noexcept { + return utf8::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::utf8_length_from_utf16(input, length); +} + +simdutf_warn_unused size_t implementation::utf16_length_from_utf8(const char * input, size_t length) const noexcept { + return utf8::utf16_length_from_utf8(input, length); +} + +} // namespace haswell +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/haswell/end.h +/* begin file src/simdutf/haswell/end.h */ +SIMDUTF_UNTARGET_REGION +/* end file src/simdutf/haswell/end.h */ +/* end file src/haswell/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_PPC64 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=ppc64/implementation.cpp +/* begin file src/ppc64/implementation.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/begin.h +/* begin file src/simdutf/ppc64/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "ppc64" +// #define SIMDUTF_IMPLEMENTATION ppc64 +/* end file src/simdutf/ppc64/begin.h */ +namespace simdutf { +namespace ppc64 { +namespace { +#ifndef SIMDUTF_PPC64_H +#error "ppc64.h must be included" +#endif +using namespace simd; + + +simdutf_really_inline bool is_ascii(const simd8x64& input) { + // careful: 0x80 is not ascii. + return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere(); +} + +simdutf_unused simdutf_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); +} + +simdutf_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/buf_block_reader.h +/* begin file src/generic/buf_block_reader.h */ +namespace simdutf { +namespace ppc64 { +namespace { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdutf_really_inline size_t block_index(); + simdutf_really_inline bool has_full_block() const; + simdutf_really_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdutf_really_inline size_t get_remainder(uint8_t *dst) const; + simdutf_really_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text_64(const uint8_t *text) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text(const simd8x64& in) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdutf_unused static char * format_mask(uint64_t mask) { + static char *buf = reinterpret_cast(malloc(64 + 1)); + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdutf_really_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdutf_really_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdutf_really_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdutf_really_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/buf_block_reader.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_lookup4_algorithm.h +/* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf8_validation { + +using namespace simd; + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdutf_really_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + }; + const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdutf_really_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdutf_really_inline void check_next_input(const simd8x64& input) { + if(simdutf_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; + + } + } + // do not forget to call check_eof! + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // namespace utf8_validation + +using utf8_validation::utf8_checker; + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_validator.h +/* begin file src/generic/utf8_validation/utf8_validator.h */ +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf8_validation { + +/** + * Validates that the string is actual UTF-8. + */ +template +bool generic_validate_utf8(const uint8_t * input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return !c.errors(); +} + +bool generic_validate_utf8(const char * input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input),length); +} + +} // namespace utf8_validation +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_validator.h */ +// transcoding from UTF-8 to UTF-16 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/valid_utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ + + +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf8_to_utf16 { + +using namespace simd; + + +simdutf_warn_unused size_t convert_valid(const char* input, size_t size, + char16_t* utf16_output) noexcept { + // The implementation is not specific to haswell and should be moved to the generic directory. + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + // this loop could be unrolled further. For example, we could process the mask + // far more than 64 bytes. + // + // For pure ASCII inputs, this function is not optimally fast because they are + // faster ways to just check for ASCII than to compute the continuation mask. + // However, the continuation mask is more informative. There might be a trade-off + // involved. + // + simd8x64 in(reinterpret_cast(input + pos)); + uint64_t utf8_continuation_mask = in.lt(-65 + 1); + // -65 is 0b10111111 in two-complement's, so largest possible continuation byte + if(utf8_continuation_mask != 0) { + // Slow path. We hope that the compiler will recognize that this is a slow path. + // Anything that is not a continuation mask is a 'leading byte', that is, the + // start of a new code point. + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + // The *start* of code points is not so useful, rather, we want the *end* of code points. + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times when using solely + // the slow/regular path, and at least four times if there are fast paths. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + // + // Thus we may allow convert_masked_utf8_to_utf16 to process + // more bytes at a time under a fast-path mode where 16 bytes + // are consumed at once (e.g., when encountering ASCII). + size_t consumed = convert_masked_utf8_to_utf16(input + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } else { + in.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } + } + utf16_output += scalar::utf8_to_utf16::convert_valid(input + pos, size - pos, utf16_output); + return utf16_output - start; +} + + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ + + +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf8_to_utf16 { +using namespace simd; + + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + + struct validating_transcoder { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + + validating_transcoder() : error(uint8_t(0)) {} + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + + + simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) { + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + simd8x64 input(reinterpret_cast(in + pos)); + if(input.is_ascii()) { + input.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + auto zero = simd8{uint8_t(0)}; + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + size_t consumed = convert_masked_utf8_to_utf16(in + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } + } + if(errors()) { return 0; } + if(pos < size) { + size_t howmany = scalar::utf8_to_utf16::convert(in + pos, size - pos, utf16_output); + if(howmany == 0) { return 0; } + utf16_output += howmany; + } + return utf16_output - start; + } + + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ +// other functions +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8.h +/* begin file src/generic/utf8.h */ + +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf8 { + +using namespace simd; + +simdutf_really_inline size_t count_code_points(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + count += 64 - count_ones(utf8_continuation_mask); + } + return count + scalar::utf8::count_code_points(in + pos, size - pos); +} + + +simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + // We count one word for anything that is not a continuation (so + // leading bytes). + count += 64 - count_ones(utf8_continuation_mask); + int64_t utf8_4byte = input.gteq_unsigned(240); + count += count_ones(utf8_4byte); + } + return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); +} +} // utf8 namespace +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf16.h +/* begin file src/generic/utf16.h */ +#include +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf16 { + +simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); + count += count_ones(not_pair) / 2; + } + return count + scalar::utf16::count_code_points(in + pos, size - pos); +} +simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t ascii_mask = input.lteq(0x7F); + uint64_t twobyte_mask = input.lteq(0x7FF); + uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); + + size_t ascii_count = count_ones(ascii_mask) / 2; + size_t twobyte_count = count_ones(twobyte_mask & ~ ascii_mask) / 2; + size_t threebyte_count = count_ones(not_pair_mask & ~ twobyte_mask) / 2; + size_t fourbyte_count = 32 - count_ones(not_pair_mask) / 2; + count += 2 * fourbyte_count + 3 * threebyte_count + 2 * twobyte_count + ascii_count; + } + return count + scalar::utf16::utf8_length_from_utf16(in + pos, size - pos); +} +} // utf16 +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf16.h */ + +// +// Implementation-specific overrides +// +namespace simdutf { +namespace ppc64 { + +simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return ppc64::utf8_validation::generic_validate_utf8(buf,len); +} + +simdutf_warn_unused bool implementation::validate_utf16(const char16_t *buf, size_t len) const noexcept { + return scalar::utf16::validate(buf, len); +} + +simdutf_warn_unused size_t implementation::convert_utf8_to_utf16(const char* /*buf*/, size_t /*len*/, char16_t* /*utf16_output*/) const noexcept { + return 0; // stub +} + +simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf16(const char* /*buf*/, size_t /*len*/, char16_t* /*utf16_output*/) const noexcept { + return 0; // stub +} + +simdutf_warn_unused size_t implementation::convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return scalar::utf16_to_utf8::convert(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return scalar::utf16_to_utf8::convert_valid(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::count_utf16(const char16_t * input, size_t length) const noexcept { + return scalar::utf16::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::count_utf8(const char * input, size_t length) const noexcept { + return utf8::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept { + return scalar::utf16::utf8_length_from_utf16(input, length); +} + +simdutf_warn_unused size_t implementation::utf16_length_from_utf8(const char * input, size_t length) const noexcept { + return scalar::utf8::utf16_length_from_utf8(input, length); +} + +} // namespace ppc64 +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/ppc64/end.h +/* begin file src/simdutf/ppc64/end.h */ +/* end file src/simdutf/ppc64/end.h */ +/* end file src/ppc64/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_WESTMERE +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=westmere/implementation.cpp +/* begin file src/westmere/implementation.cpp */ +#include + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/begin.h +/* begin file src/simdutf/westmere/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "westmere" +// #define SIMDUTF_IMPLEMENTATION westmere +SIMDUTF_TARGET_WESTMERE +/* end file src/simdutf/westmere/begin.h */ +namespace simdutf { +namespace westmere { +namespace { +#ifndef SIMDUTF_WESTMERE_H +#error "westmere.h must be included" +#endif +using namespace simd; + +simdutf_really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); +} + +simdutf_unused simdutf_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); +} + +simdutf_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=westmere/sse_convert_utf8_to_utf16.cpp +/* begin file src/westmere/sse_convert_utf8_to_utf16.cpp */ +// depends on "tables/utf8_to_utf16_tables.h" + + +// Convert up to 12 bytes from utf8 to utf16 using a mask indicating the +// end of the code points. Only the least significant 12 bits of the mask +// are accessed. +// It returns how many bytes were consumed (up to 12). +size_t convert_masked_utf8_to_utf16(const char *input, + uint64_t utf8_end_of_code_point_mask, + char16_t *&utf16_output) { + // we use an approach where we try to process up to 12 input bytes. + // Why 12 input bytes and not 16? Because we are concerned with the size of + // the lookup tables. Also 12 is nicely divisible by two and three. + // + // + // Optimization note: our main path below is load-latency dependent. Thus it is maybe + // beneficial to have fast paths that depend on branch prediction but have less latency. + // This results in more instructions but, potentially, also higher speeds. + // + // We first try a few fast paths. + const __m128i in = _mm_loadu_si128((__m128i *)input); + const uint16_t input_utf8_end_of_code_point_mask = + utf8_end_of_code_point_mask & 0xFFF; + if(((utf8_end_of_code_point_mask & 0xFFFF) == 0xFFFF)) { + // We process the data in chunks of 16 bytes. + _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output), _mm_cvtepu8_epi16(in)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output + 8), _mm_cvtepu8_epi16(_mm_srli_si128(in,8))); + utf16_output += 16; // We wrote 16 16-bit characters. + return 16; // We consumed 16 bytes. + } + if(((utf8_end_of_code_point_mask & 0xFFFF) == 0xaaaa)) { + // We want to take 8 2-byte UTF-8 words and turn them into 8 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. + const __m128i sh = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); + const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); + _mm_storeu_si128((__m128i *)utf16_output, composed); + utf16_output += 8; // We wrote 16 bytes, 8 code points. + return 16; + } + if(input_utf8_end_of_code_point_mask == 0x924) { + // We want to take 4 3-byte UTF-8 words and turn them into 4 2-byte UTF-16 words. + // There is probably a more efficient sequence, but the following might do. + const __m128i sh = _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = + _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits + const __m128i middlebyte = + _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + const __m128i highbyte = + _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); + const __m128i composed_repacked = _mm_packus_epi32(composed, composed); + _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); + utf16_output += 4; + return 12; + } + /// We do not have a fast path available, so we fallback. + + const uint8_t idx = + tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; + const uint8_t consumed = + tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; + if (idx < 64) { + // SIX (6) input code-words + // this is a relatively easy scenario + // we process SIX (6) input code-words. The max length in bytes of six code + // words spanning between 1 and 2 bytes each is 12 bytes. On processors + // where pdep/pext is fast, we might be able to use a small lookup table. + const __m128i sh = + _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); + const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); + _mm_storeu_si128((__m128i *)utf16_output, composed); + utf16_output += 6; // We wrote 12 bytes, 6 code points. + } else if (idx < 145) { + // FOUR (4) input code-words + const __m128i sh = + _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = + _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits + const __m128i middlebyte = + _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + const __m128i highbyte = + _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); + const __m128i composed_repacked = _mm_packus_epi32(composed, composed); + _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); + utf16_output += 4; + } else if (idx < 209) { + // TWO (2) input code-words + const __m128i sh = + _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); + const __m128i perm = _mm_shuffle_epi8(in, sh); + const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); + const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); + const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); + __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); + // correct for spurious high bit + const __m128i correct = + _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); + middlehighbyte = _mm_xor_si128(correct, middlehighbyte); + const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); + const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0x07000000)); + const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); + const __m128i composed = + _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), + _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); + const __m128i composedminus = + _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); + const __m128i lowtenbits = + _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); + const __m128i hightenbits = _mm_srli_epi32(composedminus, 10); + const __m128i lowtenbitsadd = + _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); + const __m128i hightenbitsadd = + _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); + const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); + const __m128i surrogates = + _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); + uint32_t basic_buffer[4]; + _mm_storeu_si128((__m128i *)basic_buffer, composed); + uint32_t surrogate_buffer[4]; + _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); + for (size_t i = 0; i < 3; i++) { + if (basic_buffer[i] < 65536) { + utf16_output[0] = uint16_t(basic_buffer[i]); + utf16_output++; + } else { + utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xFFFF); + utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); + utf16_output += 2; + } + } + } else { + // here we know that there is an error but we do not handle errors + } + return consumed; +} +/* end file src/westmere/sse_convert_utf8_to_utf16.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=westmere/sse_validate_utf16le.cpp +/* begin file src/westmere/sse_validate_utf16le.cpp */ +/* + In UTF-16 words in range 0xD800 to 0xDFFF have special meaning. + + In a vectorized algorithm we want to examine the most significant + nibble in order to select a fast path. If none of highest nibbles + are 0xD (13), than we are sure that UTF-16 chunk in a vector + register is valid. + + Let us analyze what we need to check if the nibble is 0xD. The + value of the preceding nibble determines what we have: + + 0xd000 .. 0xd7ff - a valid word + 0xd800 .. 0xdbff - low surrogate + 0xdc00 .. 0xdfff - high surrogate + + Other constraints we have to consider: + - there must not be two consecutive low surrogates (0xd800 .. 0xdbff) + - there must not be two consecutive high surrogates (0xdc00 .. 0xdfff) + - there must not be sole low surrogate nor high surrogate + + We're going to build three bitmasks based on the 3rd nibble: + - V = valid word, + - L = low surrogate (0xd800 .. 0xdbff) + - H = high surrogate (0xdc00 .. 0xdfff) + + 0 1 2 3 4 5 6 7 <--- word index + [ V | L | H | L | H | V | V | L ] + 1 0 0 0 0 1 1 0 - V = valid masks + 0 1 0 1 0 0 0 1 - L = low surrogate + 0 0 1 0 1 0 0 0 - H high surrogate + + + 1 0 0 0 0 1 1 0 V = valid masks + 0 1 0 1 0 0 0 0 a = L & (H >> 1) + 0 0 1 0 1 0 0 0 b = a << 1 + 1 1 1 1 1 1 1 0 c = V | a | b + ^ + the last bit can be zero, we just consume 7 words + and recheck this word in the next iteration +*/ + +/* Returns: + - pointer to the last unprocessed character (a scalar fallback should check the rest); + - nullptr if an error was detected. +*/ +const char16_t* sse_validate_utf16le(const char16_t* input, size_t size) { + const char16_t* end = input + size; + + const auto v_d8 = simd8::splat(0xd8); + const auto v_f8 = simd8::splat(0xf8); + const auto v_fc = simd8::splat(0xfc); + const auto v_dc = simd8::splat(0xdc); + + while (input + simd16::SIZE * 2 < end) { + // 0. Load data: since the validation takes into account only higher + // byte of each word, we compress the two vectors into one which + // consists only the higher bytes. + const auto in0 = simd16(input); + const auto in1 = simd16(input + simd16::SIZE / sizeof(char16_t)); + + const auto t0 = in0.shr<8>(); + const auto t1 = in1.shr<8>(); + + const auto in = simd16::pack(t0, t1); + + // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). + const auto surrogates_wordmask = (in & v_f8) == v_d8; + const uint16_t surrogates_bitmask = static_cast(surrogates_wordmask.to_bitmask()); + if (surrogates_bitmask == 0x0000) { + input += 16; + } else { + // 2. We have some surrogates that have to be distinguished: + // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) + // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) + // + // Fact: high surrogate has 11th bit set (3rd bit in the higher word) + + // V - non-surrogate words + // V = not surrogates_wordmask + const uint16_t V = static_cast(~surrogates_bitmask); + + // H - word-mask for high surrogates: the six highest bits are 0b1101'11 + const auto vH = (in & v_fc) == v_dc; + const uint16_t H = static_cast(vH.to_bitmask()); + + // L - word mask for low surrogates + // L = not H and surrogates_wordmask + const uint16_t L = static_cast(~H & surrogates_bitmask); + + const uint16_t a = static_cast(L & (H >> 1)); // A low surrogate must be followed by high one. + // (A low surrogate placed in the 7th register's word + // is an exception we handle.) + const uint16_t b = static_cast(a << 1); // Just mark that the opposite fact is hold, + // thanks to that we have only two masks for valid case. + const uint16_t c = static_cast(V | a | b); // Combine all the masks into the final one. + + if (c == 0xffff) { + // The whole input register contains valid UTF-16, i.e., + // either single words or proper surrogate pairs. + input += 16; + } else if (c == 0x7fff) { + // The 15 lower words of the input register contains valid UTF-16. + // The 15th word may be either a low or high surrogate. It the next + // iteration we 1) check if the low surrogate is followed by a high + // one, 2) reject sole high surrogate. + input += 15; + } else { + return nullptr; + } + } + } + + return input; +} +/* end file src/westmere/sse_validate_utf16le.cpp */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=westmere/sse_convert_utf16_to_utf8.cpp +/* begin file src/westmere/sse_convert_utf16_to_utf8.cpp */ +/* + The vectorized algorithm works on single SSE register i.e., it + loads eight 16-bit words. + + We consider three cases: + 1. an input register contains no surrogates and each value + is in range 0x0000 .. 0x07ff. + 2. an input register contains no surrogates and values are + is in range 0x0000 .. 0xffff. + 3. an input register contains surrogates --- i.e. codepoints + can have 16 or 32 bits. + + Ad 1. + + When values are less than 0x0800, it means that a 16-bit words + can be converted into: 1) single UTF8 byte (when it's an ASCII + char) or 2) two UTF8 bytes. + + For this case we do only some shuffle to obtain these 2-byte + codes and finally compress the whole SSE register with a single + shuffle. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + Ad 2. + + When values fit in 16-bit words, but are above 0x07ff, then + a single word may produce one, two or three UTF8 bytes. + + We prepare data for all these three cases in two registers. + The first register contains lower two UTF8 bytes (used in all + cases), while the second one contains just the third byte for + the three-UTF8-bytes case. + + Finally these two registers are interleaved forming eight-element + array of 32-bit values. The array spans two SSE registers. + The bytes from the registers are compressed using two shuffles. + + We need 256-entry lookup table to get a compression pattern + and the number of output bytes in the compressed vector register. + Each entry occupies 17 bytes. + + + To summarize: + - We need two 256-entry tables that have 8704 bytes in total. +*/ + +/* + Returns a pair: the first unprocessed byte from buf and utf8_output + A scalar routing should carry on the conversion of the tail. +*/ +std::pair sse_convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) { + + const char16_t* end = buf + len; + + const __m128i v_0000 = _mm_setzero_si128(); + const __m128i v_f800 = _mm_set1_epi16((int16_t)0xf800); + const __m128i v_d800 = _mm_set1_epi16((int16_t)0xd800); + const __m128i v_c080 = _mm_set1_epi16((int16_t)0xc080); + while (buf + 16 <= end) { + __m128i in = _mm_loadu_si128((__m128i*)buf); + // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes + const __m128i v_ff80 = _mm_set1_epi16((int16_t)0xff80); + if(_mm_testz_si128(in, v_ff80)) { // ASCII fast path!!!! + __m128i nextin = _mm_loadu_si128((__m128i*)buf+1); + if(!_mm_testz_si128(nextin, v_ff80)) { + // 1. pack the bytes + // obviously suboptimal. + const __m128i utf8_packed = _mm_packus_epi16(in,in); + // 2. store (16 bytes) + _mm_storeu_si128((__m128i*)utf8_output, utf8_packed); + // 3. adjust pointers + buf += 8; + utf8_output += 8; + in = nextin; + } else { + // 1. pack the bytes + // obviously suboptimal. + const __m128i utf8_packed = _mm_packus_epi16(in,nextin); + // 2. store (16 bytes) + _mm_storeu_si128((__m128i*)utf8_output, utf8_packed); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } + } + + // no bits set above 7th bit + const __m128i one_byte_bytemask = _mm_cmpeq_epi16(_mm_and_si128(in, v_ff80), v_0000); + const uint16_t one_byte_bitmask = static_cast(_mm_movemask_epi8(one_byte_bytemask)); + + // no bits set above 11th bit + const __m128i one_or_two_bytes_bytemask = _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_0000); + const uint16_t one_or_two_bytes_bitmask = static_cast(_mm_movemask_epi8(one_or_two_bytes_bytemask)); + + if (one_or_two_bytes_bitmask == 0xffff) { + // 1. prepare 2-byte values + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 + // expected output : [110a|aaaa|10bb|bbbb] x 8 + const __m128i v_1f00 = _mm_set1_epi16((int16_t)0x1f00); + const __m128i v_003f = _mm_set1_epi16((int16_t)0x003f); + + // t0 = [000a|aaaa|bbbb|bb00] + const __m128i t0 = _mm_slli_epi16(in, 2); + // t1 = [000a|aaaa|0000|0000] + const __m128i t1 = _mm_and_si128(t0, v_1f00); + // t2 = [0000|0000|00bb|bbbb] + const __m128i t2 = _mm_and_si128(in, v_003f); + // t3 = [000a|aaaa|00bb|bbbb] + const __m128i t3 = _mm_or_si128(t1, t2); + // t4 = [110a|aaaa|10bb|bbbb] + const __m128i t4 = _mm_or_si128(t3, v_c080); + + // 2. merge ASCII and 2-byte codewords + const __m128i utf8_unpacked = _mm_blendv_epi8(t4, in, one_byte_bytemask); + + // 3. prepare bitmask for 8-bit lookup + // one_byte_bitmask = hhggffeeddccbbaa -- the bits are doubled (h - MSB, a - LSB) + const uint16_t m0 = one_byte_bitmask & 0x5555; // m0 = 0h0g0f0e0d0c0b0a + const uint16_t m1 = static_cast(m0 >> 7); // m1 = 00000000h0g0f0e0 + const uint8_t m2 = static_cast((m0 | m1) & 0xff); // m2 = hdgcfbea + // 4. pack the bytes + const uint8_t* row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0]; + const __m128i shuffle = _mm_loadu_si128((__m128i*)(row + 1)); + const __m128i utf8_packed = _mm_shuffle_epi8(utf8_unpacked, shuffle); + + // 5. store bytes + _mm_storeu_si128((__m128i*)utf8_output, utf8_packed); + + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; + continue; + + } + + // 1. Check if there are any surrogate word in the input chunk. + // We have also deal with situation when there is a suggogate word + // at the end of a chunk. + const __m128i surrogates_bytemask = _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_d800); + + // bitmask = 0x0000 if there are no surrogates + // = 0xc000 if the last word is a surrogate + const uint16_t surrogates_bitmask = static_cast(_mm_movemask_epi8(surrogates_bytemask)); + // It might seem like checking for surrogates_bitmask == 0xc000 could help. However, + // it is likely an uncommon occurrence. + if (surrogates_bitmask == 0x0000) { + // case: words from register produce either 1, 2 or 3 UTF-8 bytes + const __m128i dup_even = _mm_setr_epi16(0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); + + /* In this branch we handle three cases: + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three UTF-8 bytes + + We expand the input word (16-bit) into two words (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. + + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. + + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. + + Finally from these two words we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ + /** + * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: + * t2 => [0ccc|cccc] [10cc|cccc] + * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) + */ +#define vec(x) _mm_set1_epi16(static_cast(x)) + // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] + const __m128i t0 = _mm_shuffle_epi8(in, dup_even); + // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] + const __m128i t1 = _mm_and_si128(t0, vec(0b0011111101111111)); + // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] + const __m128i t2 = _mm_or_si128 (t1, vec(0b1000000000000000)); + + // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] + const __m128i s0 = _mm_srli_epi16(in, 4); + // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] + const __m128i s1 = _mm_and_si128(s0, vec(0b0000111111111100)); + // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] + const __m128i s2 = _mm_maddubs_epi16(s1, vec(0x0140)); + // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] + const __m128i s3 = _mm_or_si128(s2, vec(0b1100000011100000)); + const __m128i m0 = _mm_andnot_si128(one_or_two_bytes_bytemask, vec(0b0100000000000000)); + const __m128i s4 = _mm_xor_si128(s3, m0); +#undef vec + + // 4. expand words 16-bit => 32-bit + const __m128i out0 = _mm_unpacklo_epi16(t2, s4); + const __m128i out1 = _mm_unpackhi_epi16(t2, s4); + + // 5. compress 32-bit words into 1, 2 or 3 bytes -- 2 x shuffle + const uint16_t mask = (one_byte_bitmask & 0x5555) | + (one_or_two_bytes_bitmask & 0xaaaa); + if(mask == 0) { + // We only have three-byte words. Use fast path. + const __m128i shuffle = _mm_setr_epi8(2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1); + const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle); + const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle); + _mm_storeu_si128((__m128i*)utf8_output, utf8_0); + utf8_output += 12; + _mm_storeu_si128((__m128i*)utf8_output, utf8_1); + utf8_output += 12; + buf += 8; + continue; + } + const uint8_t mask0 = uint8_t(mask); + + const uint8_t* row0 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + const __m128i shuffle0 = _mm_loadu_si128((__m128i*)(row0 + 1)); + const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle0); + + const uint8_t mask1 = static_cast(mask >> 8); + + const uint8_t* row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + const __m128i shuffle1 = _mm_loadu_si128((__m128i*)(row1 + 1)); + const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle1); + + _mm_storeu_si128((__m128i*)utf8_output, utf8_0); + utf8_output += row0[0]; + _mm_storeu_si128((__m128i*)utf8_output, utf8_1); + utf8_output += row1[0]; + + buf += 8; + // surrogate pair(s) in a register + } else { + // Let us do a scalar fallback. + // It may seem wasteful to use scalar code, but being efficient with SIMD + // in the presence of surrogate pairs may require non-trivial tables. + size_t forward = 15; + size_t k = 0; + if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} + for(; k < forward; k++) { + uint16_t word = buf[k]; + if((word & 0xFF80)==0) { + *utf8_output++ = char(word); + } else if((word & 0xF800)==0) { + *utf8_output++ = char((word>>6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else if((word &0xF800 ) != 0xD800) { + *utf8_output++ = char((word>>12) | 0b11100000); + *utf8_output++ = char(((word>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = buf[k+1]; + k++; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if((diff | diff2) > 0x3FF) { return std::make_pair(nullptr, utf8_output); } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf8_output++ = char((value>>18) | 0b11110000); + *utf8_output++ = char(((value>>12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value>>6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + } + } + buf += k; + } + } // while + + return std::make_pair(buf, utf8_output); +} +/* end file src/westmere/sse_convert_utf16_to_utf8.cpp */ + +// UTF-16 => UTF-8 conversion + +} // unnamed namespace +} // namespace westmere +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/buf_block_reader.h +/* begin file src/generic/buf_block_reader.h */ +namespace simdutf { +namespace westmere { +namespace { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdutf_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdutf_really_inline size_t block_index(); + simdutf_really_inline bool has_full_block() const; + simdutf_really_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdutf_really_inline size_t get_remainder(uint8_t *dst) const; + simdutf_really_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text_64(const uint8_t *text) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdutf_unused static char * format_input_text(const simd8x64& in) { + static char *buf = reinterpret_cast(malloc(sizeof(simd8x64) + 1)); + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdutf_unused static char * format_mask(uint64_t mask) { + static char *buf = reinterpret_cast(malloc(64 + 1)); + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdutf_really_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdutf_really_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdutf_really_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdutf_really_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/buf_block_reader.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_lookup4_algorithm.h +/* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +namespace simdutf { +namespace westmere { +namespace { +namespace utf8_validation { + +using namespace simd; + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdutf_really_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + }; + const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdutf_really_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdutf_really_inline void check_next_input(const simd8x64& input) { + if(simdutf_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; + + } + } + // do not forget to call check_eof! + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // namespace utf8_validation + +using utf8_validation::utf8_checker; + +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_validation/utf8_validator.h +/* begin file src/generic/utf8_validation/utf8_validator.h */ +namespace simdutf { +namespace westmere { +namespace { +namespace utf8_validation { + +/** + * Validates that the string is actual UTF-8. + */ +template +bool generic_validate_utf8(const uint8_t * input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return !c.errors(); +} + +bool generic_validate_utf8(const char * input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input),length); +} + +} // namespace utf8_validation +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf8_validation/utf8_validator.h */ +// transcoding from UTF-8 to UTF-16 +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/valid_utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ + + +namespace simdutf { +namespace westmere { +namespace { +namespace utf8_to_utf16 { + +using namespace simd; + + +simdutf_warn_unused size_t convert_valid(const char* input, size_t size, + char16_t* utf16_output) noexcept { + // The implementation is not specific to haswell and should be moved to the generic directory. + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + // this loop could be unrolled further. For example, we could process the mask + // far more than 64 bytes. + // + // For pure ASCII inputs, this function is not optimally fast because they are + // faster ways to just check for ASCII than to compute the continuation mask. + // However, the continuation mask is more informative. There might be a trade-off + // involved. + // + simd8x64 in(reinterpret_cast(input + pos)); + uint64_t utf8_continuation_mask = in.lt(-65 + 1); + // -65 is 0b10111111 in two-complement's, so largest possible continuation byte + if(utf8_continuation_mask != 0) { + // Slow path. We hope that the compiler will recognize that this is a slow path. + // Anything that is not a continuation mask is a 'leading byte', that is, the + // start of a new code point. + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + // The *start* of code points is not so useful, rather, we want the *end* of code points. + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times when using solely + // the slow/regular path, and at least four times if there are fast paths. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + // + // Thus we may allow convert_masked_utf8_to_utf16 to process + // more bytes at a time under a fast-path mode where 16 bytes + // are consumed at once (e.g., when encountering ASCII). + size_t consumed = convert_masked_utf8_to_utf16(input + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } else { + in.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } + } + utf16_output += scalar::utf8_to_utf16::convert_valid(input + pos, size - pos, utf16_output); + return utf16_output - start; +} + + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8_to_utf16/utf8_to_utf16.h +/* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ + + +namespace simdutf { +namespace westmere { +namespace { +namespace utf8_to_utf16 { +using namespace simd; + + + simdutf_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { +// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) +// Bit 1 = Too Long (ASCII followed by continuation) +// Bit 2 = Overlong 3-byte +// Bit 4 = Surrogate +// Bit 5 = Overlong 2-byte +// Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdutf_really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + + struct validating_transcoder { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + + validating_transcoder() : error(uint8_t(0)) {} + // + // Check whether the current bytes are valid UTF-8. + // + simdutf_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + + + simdutf_really_inline size_t convert(const char* in, size_t size, char16_t* utf16_output) { + size_t pos = 0; + char16_t* start{utf16_output}; + const size_t safety_margin = 16; // to avoid overruns! + while(pos + 64 + safety_margin <= size) { + simd8x64 input(reinterpret_cast(in + pos)); + if(input.is_ascii()) { + input.store_ascii_as_utf16(utf16_output); + utf16_output += 64; + pos += 64; + } else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + auto zero = simd8{uint8_t(0)}; + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], zero); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + uint64_t utf8_leading_mask = ~utf8_continuation_mask; + uint64_t utf8_end_of_code_point_mask = utf8_leading_mask>>1; + // We process in blocks of up to 12 bytes except possibly + // for fast paths which may process up to 16 bytes. For the + // slow path to work, we should have at least 12 input bytes left. + size_t max_starting_point = (pos + 64) - 12; + // Next loop is going to run at least five times. + while(pos < max_starting_point) { + // Performance note: our ability to compute 'consumed' and + // then shift and recompute is critical. If there is a + // latency of, say, 4 cycles on getting 'consumed', then + // the inner loop might have a total latency of about 6 cycles. + // Yet we process between 6 to 12 inputs bytes, thus we get + // a speed limit between 1 cycle/byte and 0.5 cycle/byte + // for this section of the code. Hence, there is a limit + // to how much we can further increase this latency before + // it seriously harms performance. + size_t consumed = convert_masked_utf8_to_utf16(in + pos, + utf8_end_of_code_point_mask, utf16_output); + pos += consumed; + utf8_end_of_code_point_mask >>= consumed; + } + // At this point there may remain between 0 and 12 bytes in the + // 64-byte block.These bytes will be processed again. So we have an + // 80% efficiency (in the worst case). In practice we expect an + // 85% to 90% efficiency. + } + } + if(errors()) { return 0; } + if(pos < size) { + size_t howmany = scalar::utf8_to_utf16::convert(in + pos, size - pos, utf16_output); + if(howmany == 0) { return 0; } + utf16_output += howmany; + } + return utf16_output - start; + } + + simdutf_really_inline bool errors() const { + return this->error.any_bits_set_anywhere(); + } + + }; // struct utf8_checker +} // utf8_to_utf16 namespace +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ +// other functions +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf8.h +/* begin file src/generic/utf8.h */ + +namespace simdutf { +namespace westmere { +namespace { +namespace utf8 { + +using namespace simd; + +simdutf_really_inline size_t count_code_points(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + count += 64 - count_ones(utf8_continuation_mask); + } + return count + scalar::utf8::count_code_points(in + pos, size - pos); +} + + +simdutf_really_inline size_t utf16_length_from_utf8(const char* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 64 <= size; pos += 64) { + simd8x64 input(reinterpret_cast(in + pos)); + uint64_t utf8_continuation_mask = input.lt(-65 + 1); + // We count one word for anything that is not a continuation (so + // leading bytes). + count += 64 - count_ones(utf8_continuation_mask); + int64_t utf8_4byte = input.gteq_unsigned(240); + count += count_ones(utf8_4byte); + } + return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); +} +} // utf8 namespace +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf8.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=generic/utf16.h +/* begin file src/generic/utf16.h */ +#include +namespace simdutf { +namespace westmere { +namespace { +namespace utf16 { + +simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); + count += count_ones(not_pair) / 2; + } + return count + scalar::utf16::count_code_points(in + pos, size - pos); +} +simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t size) { + size_t pos = 0; + size_t count = 0; + // This algorithm could no doubt be improved! + for(;pos + 32 <= size; pos += 32) { + simd16x32 input(reinterpret_cast(in + pos)); + uint64_t ascii_mask = input.lteq(0x7F); + uint64_t twobyte_mask = input.lteq(0x7FF); + uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); + + size_t ascii_count = count_ones(ascii_mask) / 2; + size_t twobyte_count = count_ones(twobyte_mask & ~ ascii_mask) / 2; + size_t threebyte_count = count_ones(not_pair_mask & ~ twobyte_mask) / 2; + size_t fourbyte_count = 32 - count_ones(not_pair_mask) / 2; + count += 2 * fourbyte_count + 3 * threebyte_count + 2 * twobyte_count + ascii_count; + } + return count + scalar::utf16::utf8_length_from_utf16(in + pos, size - pos); +} +} // utf16 +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf16.h */ +// +// Implementation-specific overrides +// + +namespace simdutf { +namespace westmere { + +simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return westmere::utf8_validation::generic_validate_utf8(buf, len); +} + +simdutf_warn_unused bool implementation::validate_utf16(const char16_t *buf, size_t len) const noexcept { + const char16_t* tail = sse_validate_utf16le(buf, len); + if (tail) { + return scalar::utf16::validate(tail, len - (tail - buf)); + } else { + return false; + } +} + +simdutf_warn_unused size_t implementation::convert_utf8_to_utf16(const char* buf, size_t len, char16_t* utf16_output) const noexcept { + utf8_to_utf16::validating_transcoder converter; + return converter.convert(buf, len, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf16(const char* input, size_t size, + char16_t* utf16_output) const noexcept { + return utf8_to_utf16::convert_valid(input, size, utf16_output); +} + +simdutf_warn_unused size_t implementation::convert_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + std::pair ret = sse_convert_utf16_to_utf8(buf, len, utf8_output); + if (ret.first == nullptr) { return 0; } + size_t saved_bytes = ret.second - utf8_output; + if (ret.first != buf + len) { + const size_t scalar_saved_bytes = scalar::utf16_to_utf8::convert( + ret.first, len - (ret.first - buf), ret.second); + if (scalar_saved_bytes == 0) { return 0; } + saved_bytes += scalar_saved_bytes; + } + return saved_bytes; +} + +simdutf_warn_unused size_t implementation::convert_valid_utf16_to_utf8(const char16_t* buf, size_t len, char* utf8_output) const noexcept { + return convert_utf16_to_utf8(buf, len, utf8_output); +} + +simdutf_warn_unused size_t implementation::count_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::count_utf8(const char * input, size_t length) const noexcept { + return utf8::count_code_points(input, length); +} + +simdutf_warn_unused size_t implementation::utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept { + return utf16::utf8_length_from_utf16(input, length); +} + +simdutf_warn_unused size_t implementation::utf16_length_from_utf8(const char * input, size_t length) const noexcept { + return utf8::utf16_length_from_utf8(input, length); +} + +} // namespace westmere +} // namespace simdutf + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/src, filename=simdutf/westmere/end.h +/* begin file src/simdutf/westmere/end.h */ +SIMDUTF_UNTARGET_REGION +/* end file src/simdutf/westmere/end.h */ +/* end file src/westmere/implementation.cpp */ +#endif + +SIMDUTF_POP_DISABLE_WARNINGS +/* end file src/simdutf.cpp */ diff --git a/simdutf/simdutf.h b/simdutf/simdutf.h new file mode 100644 --- /dev/null +++ b/simdutf/simdutf.h @@ -0,0 +1,1084 @@ +/* auto-generated on 2021-07-29 10:43:28 -0400. Do not edit! */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf.h +/* begin file include/simdutf.h */ +#ifndef SIMDUTF_H +#define SIMDUTF_H +#include +#include +#include +#include + +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/compiler_check.h +/* begin file include/simdutf/compiler_check.h */ +#ifndef SIMDUTF_COMPILER_CHECK_H +#define SIMDUTF_COMPILER_CHECK_H + +#ifndef __cplusplus +#error simdutf requires a C++ compiler +#endif + +#ifndef SIMDUTF_CPLUSPLUS +#if defined(_MSVC_LANG) && !defined(__clang__) +#define SIMDUTF_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) +#else +#define SIMDUTF_CPLUSPLUS __cplusplus +#endif +#endif + +// C++ 17 +#if !defined(SIMDUTF_CPLUSPLUS17) && (SIMDUTF_CPLUSPLUS >= 201703L) +#define SIMDUTF_CPLUSPLUS17 1 +#endif + +// C++ 14 +#if !defined(SIMDUTF_CPLUSPLUS14) && (SIMDUTF_CPLUSPLUS >= 201402L) +#define SIMDUTF_CPLUSPLUS14 1 +#endif + +// C++ 11 +#if !defined(SIMDUTF_CPLUSPLUS11) && (SIMDUTF_CPLUSPLUS >= 201103L) +#define SIMDUTF_CPLUSPLUS11 1 +#endif + +#ifndef SIMDUTF_CPLUSPLUS11 +#error simdutf requires a compiler compliant with the C++11 standard +#endif + +#endif // SIMDUTF_COMPILER_CHECK_H +/* end file include/simdutf/compiler_check.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/common_defs.h +/* begin file include/simdutf/common_defs.h */ +#ifndef SIMDUTF_COMMON_DEFS_H +#define SIMDUTF_COMMON_DEFS_H + +#include +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/portability.h +/* begin file include/simdutf/portability.h */ +#ifndef SIMDUTF_PORTABILITY_H +#define SIMDUTF_PORTABILITY_H + +#include +#include +#include +#include +#include +#ifndef _WIN32 +// strcasecmp, strncasecmp +#include +#endif + +#ifdef _MSC_VER +#define SIMDUTF_VISUAL_STUDIO 1 +/** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + * + * Under clang for Windows, we enable: + * * target pragmas so that part and only part of the + * code gets compiled for advanced instructions. + * + */ +#ifdef __clang__ +// clang under visual studio +#define SIMDUTF_CLANG_VISUAL_STUDIO 1 +#else +// just regular visual studio (best guess) +#define SIMDUTF_REGULAR_VISUAL_STUDIO 1 +#endif // __clang__ +#endif // _MSC_VER + +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO +// https://en.wikipedia.org/wiki/C_alternative_tokens +// This header should have no effect, except maybe +// under Visual Studio. +#include +#endif + +#if defined(__x86_64__) || defined(_M_AMD64) +#define SIMDUTF_IS_X86_64 1 +#elif defined(__aarch64__) || defined(_M_ARM64) +#define SIMDUTF_IS_ARM64 1 +#elif defined(__PPC64__) || defined(_M_PPC64) +//#define SIMDUTF_IS_PPC64 1 +#pragma message("The simdutf library does yet support SIMD acceleration under\ +POWER processors. Please see https://github.com/lemire/simdutf/issues/51") +#else +// The simdutf library is designed +// for 64-bit processors and it seems that you are not +// compiling for a known 64-bit platform. Please +// use a 64-bit target such as x64 or 64-bit ARM for best performance. +#define SIMDUTF_IS_32BITS 1 + +// We do not support 32-bit platforms, but it can be +// handy to identify them. +#if defined(_M_IX86) || defined(__i386__) +#define SIMDUTF_IS_X86_32BITS 1 +#elif defined(__arm__) || defined(_M_ARM) +#define SIMDUTF_IS_ARM_32BITS 1 +#elif defined(__PPC__) || defined(_M_PPC) +#define SIMDUTF_IS_PPC_32BITS 1 +#endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) + +#ifdef SIMDUTF_IS_32BITS +#ifndef SIMDUTF_NO_PORTABILITY_WARNING +#pragma message("The simdutf library is designed \ +for 64-bit processors and it seems that you are not \ +compiling for a known 64-bit platform. All fast kernels \ +will be disabled and performance may be poor. Please \ +use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.") +#endif // SIMDUTF_NO_PORTABILITY_WARNING +#endif // SIMDUTF_IS_32BITS + +// this is almost standard? +#undef STRINGIFY_IMPLEMENTATION_ +#undef STRINGIFY +#define STRINGIFY_IMPLEMENTATION_(a) #a +#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The simdutf users should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select SIMDUTF_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#ifdef SIMDUTF_IS_X86_64 +#ifdef __clang__ +// clang does not have GCC push pop +// warning: clang attribute push can't be used within a namespace in clang up +// til 8.0 so SIMDUTF_TARGET_REGION and SIMDUTF_UNTARGET_REGION must be *outside* of a +// namespace. +#define SIMDUTF_TARGET_REGION(T) \ + _Pragma(STRINGIFY( \ + clang attribute push(__attribute__((target(T))), apply_to = function))) +#define SIMDUTF_UNTARGET_REGION _Pragma("clang attribute pop") +#elif defined(__GNUC__) +// GCC is easier +#define SIMDUTF_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) +#define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") +#endif // clang then gcc + +#endif // x86 + +// Default target region macros don't do anything. +#ifndef SIMDUTF_TARGET_REGION +#define SIMDUTF_TARGET_REGION(T) +#define SIMDUTF_UNTARGET_REGION +#endif + +// Is threading enabled? +#if defined(_REENTRANT) || defined(_MT) +#ifndef SIMDUTF_THREADS_ENABLED +#define SIMDUTF_THREADS_ENABLED +#endif +#endif + +// workaround for large stack sizes under -O0. +// https://github.com/simdutf/simdutf/issues/691 +#ifdef __APPLE__ +#ifndef __OPTIMIZE__ +// Apple systems have small stack sizes in secondary threads. +// Lack of compiler optimization may generate high stack usage. +// Users may want to disable threads for safety, but only when +// in debug mode which we detect by the fact that the __OPTIMIZE__ +// macro is not defined. +#undef SIMDUTF_THREADS_ENABLED +#endif +#endif + + +#if defined(__clang__) +#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) +#elif defined(__GNUC__) +#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) +#else +#define NO_SANITIZE_UNDEFINED +#endif + +#ifdef SIMDUTF_VISUAL_STUDIO +// This is one case where we do not distinguish between +// regular visual studio and clang under visual studio. +// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has) +#define simdutf_strcasecmp _stricmp +#define simdutf_strncasecmp _strnicmp +#else +// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8). +// So they are only useful for ASCII in our context. +// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings +#define simdutf_strcasecmp strcasecmp +#define simdutf_strncasecmp strncasecmp +#endif + +#ifdef NDEBUG + +#ifdef SIMDUTF_VISUAL_STUDIO +#define SIMDUTF_UNREACHABLE() __assume(0) +#define SIMDUTF_ASSUME(COND) __assume(COND) +#else +#define SIMDUTF_UNREACHABLE() __builtin_unreachable(); +#define SIMDUTF_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) +#endif + +#else // NDEBUG + +#define SIMDUTF_UNREACHABLE() assert(0); +#define SIMDUTF_ASSUME(COND) assert(COND) + +#endif + +#endif // SIMDUTF_PORTABILITY_H +/* end file include/simdutf/portability.h */ + + +#if defined(__GNUC__) + // Marks a block with a name so that MCA analysis can see it. + #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); + #define SIMDUTF_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); + #define SIMDUTF_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); +#else + #define SIMDUTF_BEGIN_DEBUG_BLOCK(name) + #define SIMDUTF_END_DEBUG_BLOCK(name) + #define SIMDUTF_DEBUG_BLOCK(name, block) +#endif + +// Align to N-byte boundary +#define SIMDUTF_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) +#define SIMDUTF_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) + +#define SIMDUTF_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) + +#if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) + + #define simdutf_really_inline __forceinline + #define simdutf_never_inline __declspec(noinline) + + #define simdutf_unused + #define simdutf_warn_unused + + #ifndef simdutf_likely + #define simdutf_likely(x) x + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) x + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS __pragma(warning( push )) + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 )) + #define SIMDUTF_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER )) + // Get rid of Intellisense-only warnings (Code Analysis) + // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910). + #ifdef __has_include + #if __has_include() + #include + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) + #endif + #endif + + #ifndef SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_VS_WARNING(4996) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING + #define SIMDUTF_POP_DISABLE_WARNINGS __pragma(warning( pop )) + +#else // SIMDUTF_REGULAR_VISUAL_STUDIO + + #define simdutf_really_inline inline __attribute__((always_inline)) + #define simdutf_never_inline inline __attribute__((noinline)) + + #define simdutf_unused __attribute__((unused)) + #define simdutf_warn_unused __attribute__((warn_unused_result)) + + #ifndef simdutf_likely + #define simdutf_likely(x) __builtin_expect(!!(x), 1) + #endif + #ifndef simdutf_unlikely + #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) + #endif + + #define SIMDUTF_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") + // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary + #define SIMDUTF_PUSH_DISABLE_ALL_WARNINGS SIMDUTF_PUSH_DISABLE_WARNINGS \ + SIMDUTF_DISABLE_GCC_WARNING(-Weffc++) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wall) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wconversion) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wextra) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wattributes) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wreturn-type) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wshadow) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-parameter) \ + SIMDUTF_DISABLE_GCC_WARNING(-Wunused-variable) + #define SIMDUTF_PRAGMA(P) _Pragma(#P) + #define SIMDUTF_DISABLE_GCC_WARNING(WARNING) SIMDUTF_PRAGMA(GCC diagnostic ignored #WARNING) + #if defined(SIMDUTF_CLANG_VISUAL_STUDIO) + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS SIMDUTF_DISABLE_GCC_WARNING(-Wmicrosoft-include) + #else + #define SIMDUTF_DISABLE_UNDESIRED_WARNINGS + #endif + #define SIMDUTF_DISABLE_DEPRECATED_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wdeprecated-declarations) + #define SIMDUTF_DISABLE_STRICT_OVERFLOW_WARNING SIMDUTF_DISABLE_GCC_WARNING(-Wstrict-overflow) + #define SIMDUTF_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") + + + +#endif // MSC_VER + +#if defined(SIMDUTF_VISUAL_STUDIO) + /** + * It does not matter here whether you are using + * the regular visual studio or clang under visual + * studio. + */ + #if SIMDUTF_USING_LIBRARY + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllimport) + #else + #define SIMDUTF_DLLIMPORTEXPORT __declspec(dllexport) + #endif +#else + #define SIMDUTF_DLLIMPORTEXPORT +#endif + +/// If EXPR is an error, returns it. +#define SIMDUTF_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } } + + +#endif // SIMDUTF_COMMON_DEFS_H +/* end file include/simdutf/common_defs.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/encoding_types.h +/* begin file include/simdutf/encoding_types.h */ +#include + +namespace simdutf { + +enum encoding_type { + UTF16_LE, // BOM 0xff 0xfe + UTF16_BE, // BOM 0xfe 0xff + UTF32_LE, // BOM 0xff 0xfe 0x00 0x00 + UTF32_BE, // BOM 0x00 0x00 0xfe 0xff + UTF8, // BOM 0xef 0xbb 0xbf + unspecified +}; + +std::string to_string(encoding_type bom); + +// Note that BOM for UTF8 is discouraged. +namespace BOM { + +/** + * Checks for a BOM. If not, returns unspecified + * @param input the string to process + * @param length the length of the string in words + * @return the corresponding encoding + */ + +encoding_type check_bom(const uint8_t* byte, size_t length); +encoding_type check_bom(const char* byte, size_t length); +/** + * Returns the size, in bytes, of the BOM for a given encoding type. + * Note that UTF8 BOM are discouraged. + * @param bom the encoding type + * @return the size in bytes of the corresponding BOM + */ +size_t bom_byte_size(encoding_type bom); + +} // BOM namespace +} // simdutf namespace +/* end file include/simdutf/encoding_types.h */ + +SIMDUTF_PUSH_DISABLE_WARNINGS +SIMDUTF_DISABLE_UNDESIRED_WARNINGS + +// Public API +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/simdutf_version.h +/* begin file include/simdutf/simdutf_version.h */ +// /include/simdutf/simdutf_version.h automatically generated by release.py, +// do not change by hand +#ifndef SIMDUTF_SIMDUTF_VERSION_H +#define SIMDUTF_SIMDUTF_VERSION_H + +/** The version of simdutf being used (major.minor.revision) */ +#define SIMDUTF_VERSION 0.1.0 + +namespace simdutf { +enum { + /** + * The major version (MAJOR.minor.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MAJOR = 0, + /** + * The minor version (major.MINOR.revision) of simdutf being used. + */ + SIMDUTF_VERSION_MINOR = 1, + /** + * The revision (major.minor.REVISION) of simdutf being used. + */ + SIMDUTF_VERSION_REVISION = 0 +}; +} // namespace simdutf + +#endif // SIMDUTF_SIMDUTF_VERSION_H +/* end file include/simdutf/simdutf_version.h */ +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/implementation.h +/* begin file include/simdutf/implementation.h */ +#ifndef SIMDUTF_IMPLEMENTATION_H +#define SIMDUTF_IMPLEMENTATION_H +#include +#include +#include +// dofile: invoked with prepath=/Users/lemire/CVS/github/simdutf/include, filename=simdutf/internal/isadetection.h +/* begin file include/simdutf/internal/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDutf_INTERNAL_ISADETECTION_H +#define SIMDutf_INTERNAL_ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) +#include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include +#endif + +namespace simdutf { +namespace internal { + + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40, + ALTIVEC = 0x80 +}; + +#if defined(__PPC64__) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::ALTIVEC; +} + +#elif defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 + +#if defined(__ARM_NEON) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#else // ARM without NEON + +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif + +#elif defined(__x86_64__) || defined(_M_AMD64) // x64 + + +namespace { +// Can be found on Intel ISA Reference for CPUID +constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 +constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 +} + + + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { +#if defined(_MSC_VER) + int cpu_info[4]; + __cpuid(cpu_info, *eax); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax, ebx, ecx, edx; + uint32_t host_isa = 0x0; + + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_avx2_bit) { + host_isa |= instruction_set::AVX2; + } + if (ebx & cpuid_bmi1_bit) { + host_isa |= instruction_set::BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + host_isa |= instruction_set::BMI2; + } + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_pclmulqdq_bit) { + host_isa |= instruction_set::PCLMULQDQ; + } + + return host_isa; +} +#else // fallback + + +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + + +#endif // end SIMD extension detection code + +} // namespace internal +} // namespace simdutf + +#endif // SIMDutf_INTERNAL_ISADETECTION_H +/* end file include/simdutf/internal/isadetection.h */ + + +namespace simdutf { + +/** + * Autodetect the encoding of the input. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const uint8_t * input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast(input), length); +} + + +/** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + +/** + * Validate the UTF-16LE string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte words (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept; + +/** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) noexcept; + +/** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept; + +/** + * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE + */ +simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept; + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written words; 0 if input is not a valid UTF-16LE string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; + +/** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold the conversion result + * @return number of written words; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format. + * + * This function does not validate the input. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept; + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte words (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept; + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ +simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept; + +/** + * An implementation of simdutf for a particular CPU architecture. + * + * Also used to maintain the currently active implementation. The active implementation is + * automatically initialized on first use to the most advanced implementation supported by the host. + */ +class implementation { +public: + + /** + * The name of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual const std::string &name() const { return _name; } + + /** + * The description of this implementation. + * + * const implementation *impl = simdutf::active_implementation; + * cout << "simdutf is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; + * + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + */ + virtual const std::string &description() const { return _description; } + + /** + * The instruction sets this implementation is compiled against + * and the current CPU match. This function may poll the current CPU/system + * and should therefore not be called too often if performance is a concern. + * + * + * @return true if the implementation can be safely used on the current system (determined at runtime) + */ + bool supported_by_runtime_system() const; + + /** + * This function will try to detect the encoding + * @param input the string to identify + * @param length the length of the string in bytes. + * @return the encoding type detected + */ + virtual encoding_type autodetect_encoding(const char * input, size_t length) const noexcept; + + /** + * @private For internal implementation use + * + * The instruction sets this implementation is compiled against. + * + * @return a mask of all required `internal::instruction_set::` values + */ + virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; + + + /** + * Validate the UTF-8 string. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ + simdutf_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; + + /** + * Validate the UTF-16LE string. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16LE string to validate. + * @param len the length of the string in number of 2-byte words (char16_t). + * @return true if and only if the string is valid UTF-16LE. + */ + simdutf_warn_unused virtual bool validate_utf16(const char16_t *buf, size_t len) const noexcept = 0; + + /** + * Convert possibly broken UTF-8 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 string + */ + simdutf_warn_unused virtual size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf8_output) const noexcept = 0; + + /** + * Convert valid UTF-8 string into UTF-16LE string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ + simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; + + /** + * Compute the number of 2-byte words that this UTF-8 string would require in UTF-16LE format. + * + * This function does not validate the input. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return the number of char16_t words required to encode the UTF-8 string as UTF-16LE + */ + simdutf_warn_unused virtual size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept = 0; + + /** + * Convert possibly broken UTF-16LE string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written words; 0 if input is not a valid UTF-16LE string + */ + simdutf_warn_unused virtual size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0; + + /** + * Convert valid UTF-16LE string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold the conversion result + * @return number of written words; 0 if conversion is not possible + */ + simdutf_warn_unused virtual size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0; + + /** + * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ + simdutf_warn_unused virtual size_t utf8_length_from_utf16(const char16_t * input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to process + * @param length the length of the string in 2-byte words (char16_t) + * @return number of code points + */ + simdutf_warn_unused virtual size_t count_utf16(const char16_t * input, size_t length) const noexcept = 0; + + /** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to process + * @param length the length of the string in bytes + * @return number of code points + */ + simdutf_warn_unused virtual size_t count_utf8(const char * input, size_t length) const noexcept = 0; + + + +protected: + /** @private Construct an implementation with the given name and description. For subclasses. */ + simdutf_really_inline implementation( + std::string name, + std::string description, + uint32_t required_instruction_sets + ) : + _name(name), + _description(description), + _required_instruction_sets(required_instruction_sets) + { + } + virtual ~implementation()=default; + +private: + /** + * The name of this implementation. + */ + const std::string _name; + + /** + * The description of this implementation. + */ + const std::string _description; + + /** + * Instruction sets required for this implementation. + */ + const uint32_t _required_instruction_sets; +}; + +/** @private */ +namespace internal { + +/** + * The list of available implementations compiled into simdutf. + */ +class available_implementation_list { +public: + /** Get the list of available implementations compiled into simdutf */ + simdutf_really_inline available_implementation_list() {} + /** Number of implementations */ + size_t size() const noexcept; + /** STL const begin() iterator */ + const implementation * const *begin() const noexcept; + /** STL const end() iterator */ + const implementation * const *end() const noexcept; + + /** + * Get the implementation with the given name. + * + * Case sensitive. + * + * const implementation *impl = simdutf::available_implementations["westmere"]; + * if (!impl) { exit(1); } + * if (!imp->supported_by_runtime_system()) { exit(1); } + * simdutf::active_implementation = impl; + * + * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" + * @return the implementation, or nullptr if the parse failed. + */ + const implementation * operator[](const std::string &name) const noexcept { + for (const implementation * impl : *this) { + if (impl->name() == name) { return impl; } + } + return nullptr; + } + + /** + * Detect the most advanced implementation supported by the current host. + * + * This is used to initialize the implementation on startup. + * + * const implementation *impl = simdutf::available_implementation::detect_best_supported(); + * simdutf::active_implementation = impl; + * + * @return the most advanced supported implementation for the current host, or an + * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported + * implementation. Will never return nullptr. + */ + const implementation *detect_best_supported() const noexcept; +}; + +template +class atomic_ptr { +public: + atomic_ptr(T *_ptr) : ptr{_ptr} {} + + operator const T*() const { return ptr.load(); } + const T& operator*() const { return *ptr; } + const T* operator->() const { return ptr.load(); } + + operator T*() { return ptr.load(); } + T& operator*() { return *ptr; } + T* operator->() { return ptr.load(); } + atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; } + +private: + std::atomic ptr; +}; + +} // namespace internal + +/** + * The list of available implementations compiled into simdutf. + */ +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations; + +/** + * The active implementation. + * + * Automatically initialized on first use to the most advanced implementation supported by this hardware. + */ +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr active_implementation; + +} // namespace simdutf + +#endif // SIMDUTF_IMPLEMENTATION_H +/* end file include/simdutf/implementation.h */ + + +// Implementation-internal files (must be included before the implementations themselves, to keep +// amalgamation working--otherwise, the first time a file is included, it might be put inside the +// #ifdef SIMDUTF_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other implementations can't +// compile unless that implementation is turned on). + + +SIMDUTF_POP_DISABLE_WARNINGS + +#endif // SIMDUTF_H +/* end file include/simdutf.h */ diff --git a/src/Data/Text.hs b/src/Data/Text.hs --- a/src/Data/Text.hs +++ b/src/Data/Text.hs @@ -1,6 +1,8 @@ {-# LANGUAGE BangPatterns, CPP, MagicHash, Rank2Types, UnboxedTuples, TypeFamilies #-} {-# LANGUAGE TemplateHaskellQuotes #-} {-# LANGUAGE Trustworthy #-} +{-# LANGUAGE UnliftedFFITypes #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# OPTIONS_GHC -fno-warn-orphans #-} @@ -9,6 +11,7 @@ -- Copyright : (c) 2009, 2010, 2011, 2012 Bryan O'Sullivan, -- (c) 2009 Duncan Coutts, -- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko -- -- License : BSD-style -- Maintainer : bos@serpentine.com @@ -195,54 +198,61 @@ -- * Low level operations , copy , unpackCString# + , unpackCStringAscii# + + , measureOff ) where import Prelude (Char, Bool(..), Int, Maybe(..), String, - Eq(..), Ord(..), Ordering(..), (++), + Eq, (==), (/=), Ord(..), Ordering(..), (++), Read(..), (&&), (||), (+), (-), (.), ($), ($!), (>>), - not, return, otherwise, quot) + not, return, otherwise, quot, IO) import Control.DeepSeq (NFData(rnf)) #if defined(ASSERTS) import Control.Exception (assert) -import GHC.Stack (HasCallStack) #endif -import Data.Char (isSpace) +import Data.Bits ((.&.)) +import Data.Char (isSpace, isAscii, ord) import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) import Control.Monad (foldM) -import Control.Monad.ST (ST) +import Control.Monad.ST (ST, runST) +import Control.Monad.ST.Unsafe (unsafeIOToST) import qualified Data.Text.Array as A import qualified Data.List as L import Data.Binary (Binary(get, put)) +import Data.Int (Int8) import Data.Monoid (Monoid(..)) import Data.Semigroup (Semigroup(..)) import Data.String (IsString(..)) +import Data.Text.Internal.Encoding.Utf8 (utf8Length, utf8LengthByLeader, chr2, chr3, chr4, ord2, ord3, ord4) import qualified Data.Text.Internal.Fusion as S import qualified Data.Text.Internal.Fusion.Common as S import Data.Text.Encoding (decodeUtf8', encodeUtf8) import Data.Text.Internal.Fusion (stream, reverseStream, unstream) import Data.Text.Internal.Private (span_) -import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text) -import Data.Text.Show (singleton, unpack, unpackCString#) +import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text, append) +import Data.Text.Internal.Unsafe.Char (unsafeWrite, unsafeChr8) +import Data.Text.Show (singleton, unpack, unpackCString#, unpackCStringAscii#) import qualified Prelude as P -import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter, - reverseIter_, unsafeHead, unsafeTail) -import Data.Text.Internal.Unsafe.Char (unsafeChr) -import qualified Data.Text.Internal.Functions as F -import qualified Data.Text.Internal.Encoding.Utf16 as U16 +import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord8, reverseIter, + reverseIter_, unsafeHead, unsafeTail, unsafeDupablePerformIO, iterArray, reverseIterArray) import Data.Text.Internal.Search (indices) -import Data.Text.Internal.Unsafe.Shift (UnsafeShift(..)) #if defined(__HADDOCK__) import Data.ByteString (ByteString) import qualified Data.Text.Lazy as L import Data.Int (Int64) #endif -import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt) +import Data.Word (Word8) +import Foreign.C.Types +import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt, ByteArray#) import qualified GHC.Exts as Exts +import GHC.Stack (HasCallStack) import qualified Language.Haskell.TH.Lib as TH import qualified Language.Haskell.TH.Syntax as TH import Text.Printf (PrintfArg, formatArg, formatString) +import System.Posix.Types (CSsize(..)) -- $setup -- >>> import Data.Text @@ -292,7 +302,8 @@ -- points -- () -- as 'Char' values, including code points from this invalid range. --- This means that there are some 'Char' values that are not valid +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid -- Unicode scalar values, and the functions in this module must handle -- those cases. -- @@ -301,40 +312,18 @@ -- that are not valid Unicode scalar values with the replacement -- character \"�\" (U+FFFD). Functions that perform this -- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". --- --- (One reason for this policy of replacement is that internally, a --- 'Text' value is represented as packed UTF-16 data. Values in the --- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate --- code points, and so cannot be represented. The functions replace +-- \"Performs replacement on invalid scalar values\". The functions replace -- invalid scalar values, instead of dropping them, as a security -- measure. For details, see -- .) -- $fusion -- --- Most of the functions in this module are subject to /fusion/, --- meaning that a pipeline of such functions will usually allocate at --- most one 'Text' value. --- --- As an example, consider the following pipeline: --- --- > import Data.Text as T --- > import Data.Text.Encoding as E --- > import Data.ByteString (ByteString) --- > --- > countChars :: ByteString -> Int --- > countChars = T.length . T.toUpper . E.decodeUtf8 --- --- From the type signatures involved, this looks like it should --- allocate one 'Data.ByteString.ByteString' value, and two 'Text' --- values. However, when a module is compiled with optimisation --- enabled under GHC, the two intermediate 'Text' values will be --- optimised away, and the function will be compiled down to a single --- loop over the source 'Data.ByteString.ByteString'. --- --- Functions that can be fused by the compiler are documented with the --- phrase \"Subject to fusion\". +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transormations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". instance Eq Text where Text arrA offA lenA == Text arrB offB lenB @@ -424,23 +413,17 @@ -- | /O(n)/ Compare two 'Text' values lexicographically. compareText :: Text -> Text -> Ordering -compareText ta@(Text _arrA _offA lenA) tb@(Text _arrB _offB lenB) - | lenA == 0 && lenB == 0 = EQ - | otherwise = go 0 0 - where - go !i !j - | i >= lenA || j >= lenB = compare lenA lenB - | a < b = LT - | a > b = GT - | otherwise = go (i+di) (j+dj) - where Iter a di = iter ta i - Iter b dj = iter tb j +compareText (Text arrA offA lenA) (Text arrB offB lenB) = + A.compare arrA offA arrB offB (min lenA lenB) <> compare lenA lenB +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. -- ----------------------------------------------------------------------------- -- * Conversion to/from 'Text' --- | /O(n)/ Convert a 'String' into a 'Text'. Subject to --- fusion. Performs replacement on invalid scalar values. +-- | /O(n)/ Convert a 'String' into a 'Text'. +-- Performs replacement on invalid scalar values. pack :: String -> Text pack = unstream . S.map safe . S.streamList {-# INLINE [1] pack #-} @@ -450,54 +433,29 @@ -- | /O(n)/ Adds a character to the front of a 'Text'. This function -- is more costly than its 'List' counterpart because it requires --- copying a new array. Subject to fusion. Performs replacement on +-- copying a new array. Performs replacement on -- invalid scalar values. cons :: Char -> Text -> Text -cons c t = unstream (S.cons (safe c) (stream t)) -{-# INLINE cons #-} +cons c = unstream . S.cons (safe c) . stream +{-# INLINE [1] cons #-} infixr 5 `cons` -- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process, unless fused. Subject to fusion. +-- entire array in the process. -- Performs replacement on invalid scalar values. snoc :: Text -> Char -> Text snoc t c = unstream (S.snoc (stream t) (safe c)) {-# INLINE snoc #-} --- | /O(n)/ Appends one 'Text' to the other by copying both of them --- into a new 'Text'. Subject to fusion. -append :: Text -> Text -> Text -append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) - | len1 == 0 = b - | len2 == 0 = a - | len > 0 = Text (A.run x) 0 len - | otherwise = overflowError "append" - where - len = len1+len2 - x :: ST s (A.MArray s) - x = do - arr <- A.new len - A.copyI arr 0 arr1 off1 len1 - A.copyI arr len1 arr2 off2 len - return arr -{-# NOINLINE append #-} - -{-# RULES -"TEXT append -> fused" [~1] forall t1 t2. - append t1 t2 = unstream (S.append (stream t1) (stream t2)) -"TEXT append -> unfused" [1] forall t1 t2. - unstream (S.append (stream t1) (stream t2)) = append t1 t2 - #-} - -- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. Subject to fusion. -head :: Text -> Char +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char head t = S.head (stream t) {-# INLINE head #-} -- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. Subject to fusion. +-- 'Nothing' if empty. uncons :: Text -> Maybe (Char, Text) uncons t@(Text arr off len) | len <= 0 = Nothing @@ -505,76 +463,44 @@ in (c, text arr (off+d) (len-d)) {-# INLINE [1] uncons #-} --- | Lifted from Control.Arrow and specialized. -second :: (b -> c) -> (a,b) -> (a,c) -second f (a, b) = (a, f b) - -- | /O(1)/ Returns the last character of a 'Text', which must be --- non-empty. Subject to fusion. -last :: Text -> Char -last (Text arr off len) - | len <= 0 = emptyError "last" - | n < 0xDC00 || n > 0xDFFF = unsafeChr n - | otherwise = U16.chr2 n0 n - where n = A.unsafeIndex arr (off+len-1) - n0 = A.unsafeIndex arr (off+len-2) +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char +last t@(Text _ _ len) + | len <= 0 = emptyError "last" + | otherwise = let Iter c _ = reverseIter t (len - 1) in c {-# INLINE [1] last #-} -{-# RULES -"TEXT last -> fused" [~1] forall t. - last t = S.last (stream t) -"TEXT last -> unfused" [1] forall t. - S.last (stream t) = last t - #-} - -- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. Subject to fusion. -tail :: Text -> Text +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text tail t@(Text arr off len) | len <= 0 = emptyError "tail" | otherwise = text arr (off+d) (len-d) where d = iter_ t 0 {-# INLINE [1] tail #-} -{-# RULES -"TEXT tail -> fused" [~1] forall t. - tail t = unstream (S.tail (stream t)) -"TEXT tail -> unfused" [1] forall t. - unstream (S.tail (stream t)) = tail t - #-} - -- | /O(1)/ Returns all but the last character of a 'Text', which must --- be non-empty. Subject to fusion. -init :: Text -> Text -init (Text arr off len) | len <= 0 = emptyError "init" - | n >= 0xDC00 && n <= 0xDFFF = text arr off (len-2) - | otherwise = text arr off (len-1) - where - n = A.unsafeIndex arr (off+len-1) +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text +init t@(Text arr off len) + | len <= 0 = emptyError "init" + | otherwise = text arr off (len + reverseIter_ t (len - 1)) {-# INLINE [1] init #-} -{-# RULES -"TEXT init -> fused" [~1] forall t. - init t = unstream (S.init (stream t)) -"TEXT init -> unfused" [1] forall t. - unstream (S.init (stream t)) = init t - #-} - -- | /O(1)/ Returns all but the last character and the last character of a -- 'Text', or 'Nothing' if empty. -- -- @since 1.2.3.0 unsnoc :: Text -> Maybe (Text, Char) -unsnoc (Text arr off len) - | len <= 0 = Nothing - | n < 0xDC00 || n > 0xDFFF = Just (text arr off (len-1), unsafeChr n) - | otherwise = Just (text arr off (len-2), U16.chr2 n0 n) - where n = A.unsafeIndex arr (off+len-1) - n0 = A.unsafeIndex arr (off+len-2) +unsnoc t@(Text arr off len) + | len <= 0 = Nothing + | otherwise = Just (text arr off (len + d), c) + where + Iter c d = reverseIter t (len - 1) {-# INLINE [1] unsnoc #-} --- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to --- fusion. +-- | /O(1)/ Tests whether a 'Text' is empty or not. null :: Text -> Bool null (Text _arr _off len) = #if defined(ASSERTS) @@ -583,33 +509,44 @@ len <= 0 {-# INLINE [1] null #-} -{-# RULES -"TEXT null -> fused" [~1] forall t. - null t = S.null (stream t) -"TEXT null -> unfused" [1] forall t. - S.null (stream t) = null t - #-} - -- | /O(1)/ Tests whether a 'Text' contains exactly one character. --- Subject to fusion. isSingleton :: Text -> Bool isSingleton = S.isSingleton . stream {-# INLINE isSingleton #-} -- | /O(n)/ Returns the number of characters in a 'Text'. --- Subject to fusion. length :: #if defined(ASSERTS) HasCallStack => #endif Text -> Int -length t = S.length (stream t) +length = P.negate . measureOff P.maxBound {-# INLINE [1] length #-} -- length needs to be phased after the compareN/length rules otherwise -- it may inline before the rules have an opportunity to fire. +{-# RULES +"TEXT length/filter -> S.length/S.filter" forall p t. + length (filter p t) = S.length (S.filter p (stream t)) +"TEXT length/unstream -> S.length" forall t. + length (unstream t) = S.length t +"TEXT length/pack -> P.length" forall t. + length (pack t) = P.length t +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = mul (max 0 n) (length t) +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (mul 2 (length t) - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} + -- | /O(n)/ Compare the count of characters in a 'Text' to a number. --- Subject to fusion. -- -- This function gives the same answer as comparing against the result -- of 'length', but can short circuit if the count of characters is @@ -664,11 +601,37 @@ -- >>> T.map (\c -> if c == '.' then '!' else c) message -- "I am not angry! Not at all!" -- --- Subject to fusion. Performs replacement on invalid scalar values. +-- Performs replacement on invalid scalar values. map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map (safe . f) (stream t)) +map f = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + | otherwise = do + let !(Iter c d) = iterArray src srcOff + d' <- unsafeWrite dst dstOff (safe (f c)) + inner (srcOff + d) (dstOff + d') {-# INLINE [1] map #-} +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of -- 'Text's and concatenates the list after interspersing the first -- argument between each element of the list. @@ -678,8 +641,8 @@ -- >>> T.intercalate "NI!" ["We", "seek", "the", "Holy", "Grail"] -- "WeNI!seekNI!theNI!HolyNI!Grail" intercalate :: Text -> [Text] -> Text -intercalate t = concat . (F.intersperse t) -{-# INLINE intercalate #-} +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} -- | /O(n)/ The 'intersperse' function takes a character and places it -- between the characters of a 'Text'. @@ -689,27 +652,87 @@ -- >>> T.intersperse '.' "SHIELD" -- "S.H.I.E.L.D" -- --- Subject to fusion. Performs replacement on invalid scalar values. -intersperse :: Char -> Text -> Text -intersperse c t = unstream (S.intersperse (safe c) (stream t)) -{-# INLINE intersperse #-} +-- Performs replacement on invalid scalar values. +intersperse :: Char -> Text -> Text +intersperse c t@(Text src o l) = if l == 0 then mempty else runST $ do + let !cLen = utf8Length c + dstLen = l + length t P.* cLen + dst <- A.new dstLen + + let writeSep = case cLen of + 1 -> \dstOff -> + A.unsafeWrite dst dstOff (ord8 c) + 2 -> let (c0, c1) = ord2 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + 3 -> let (c0, c1, c2) = ord3 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + _ -> let (c0, c1, c2, c3) = ord4 c in \dstOff -> do + A.unsafeWrite dst dstOff c0 + A.unsafeWrite dst (dstOff + 1) c1 + A.unsafeWrite dst (dstOff + 2) c2 + A.unsafeWrite dst (dstOff + 3) c3 + let go !srcOff !dstOff = if srcOff >= o + l then return () else do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + A.unsafeWrite dst dstOff m0 + writeSep (dstOff + 1) + go (srcOff + 1) (dstOff + 1 + cLen) + 2 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + writeSep (dstOff + 2) + go (srcOff + 2) (dstOff + 2 + cLen) + 3 -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + writeSep (dstOff + 3) + go (srcOff + 3) (dstOff + 3 + cLen) + _ -> do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + writeSep (dstOff + 4) + go (srcOff + 4) (dstOff + 4 + cLen) + + go o 0 + arr <- A.unsafeFreeze dst + return (Text arr 0 (dstLen - cLen)) +{-# INLINE [1] intersperse #-} + -- | /O(n)/ Reverse the characters of a string. -- -- Example: -- -- >>> T.reverse "desrever" -- "reversed" --- --- Subject to fusion (fuses with its argument). reverse :: #if defined(ASSERTS) HasCallStack => #endif Text -> Text -reverse t = S.reverse (stream t) +reverse (Text (A.ByteArray ba) off len) = runST $ do + marr@(A.MutableByteArray mba) <- A.new len + unsafeIOToST $ c_reverse mba ba (intToCSize off) (intToCSize len) + brr <- A.unsafeFreeze marr + return $ Text brr 0 len {-# INLINE reverse #-} +-- | The input buffer (src :: ByteArray#, off :: CSize, len :: CSize) +-- must specify a valid UTF-8 sequence, this condition is not checked. +foreign import ccall unsafe "_hs_text_reverse" c_reverse + :: Exts.MutableByteArray# s -> ByteArray# -> CSize -> CSize -> IO () + -- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in -- @haystack@ with @replacement@. -- @@ -735,7 +758,8 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -replace :: Text +replace :: HasCallStack + => Text -- ^ @needle@ to search for. If this string is empty, an -- error will occur. -> Text @@ -759,10 +783,10 @@ let loop (i:is) o d = do let d0 = d + i - o d1 = d0 + repLen - A.copyI marr d hayArr (hayOff+o) d0 - A.copyI marr d0 repArr repOff d1 + A.copyI (i - o) marr d hayArr (hayOff+o) + A.copyI repLen marr d0 repArr repOff loop is (i + neeLen) d1 - loop [] o d = A.copyI marr d hayArr (hayOff+o) len + loop [] o d = A.copyI (len - d) marr d hayArr (hayOff+o) loop ixs 0 0 return marr @@ -786,7 +810,7 @@ -- sensitivity should use appropriate versions of the -- . --- | /O(n)/ Convert a string to folded case. Subject to fusion. +-- | /O(n)/ Convert a string to folded case. -- -- This function is mainly useful for performing caseless (also known -- as case insensitive) string comparisons. @@ -807,7 +831,7 @@ {-# INLINE toCaseFold #-} -- | /O(n)/ Convert a string to lower case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The result string may be longer than the input string. For -- instance, \"İ\" (Latin capital letter I with dot above, @@ -818,7 +842,7 @@ {-# INLINE toLower #-} -- | /O(n)/ Convert a string to upper case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The result string may be longer than the input string. For -- instance, the German \"ß\" (eszett, U+00DF) maps to the @@ -828,7 +852,7 @@ {-# INLINE toUpper #-} -- | /O(n)/ Convert a string to title case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The first letter of the input is converted to title case, as is -- every subsequent letter that immediately follows a non-letter. @@ -852,7 +876,7 @@ {-# INLINE toTitle #-} -- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. +-- specified fill character on the right. -- Performs replacement on invalid scalar values. -- -- Examples: @@ -869,13 +893,6 @@ where len = length t {-# INLINE [1] justifyLeft #-} -{-# RULES -"TEXT justifyLeft -> fused" [~1] forall k c t. - justifyLeft k c t = unstream (S.justifyLeftI k c (stream t)) -"TEXT justifyLeft -> unfused" [1] forall k c t. - unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t - #-} - -- | /O(n)/ Right-justify a string to the given length, using the -- specified fill character on the left. Performs replacement on -- invalid scalar values. @@ -933,39 +950,36 @@ -- | /O(n)/ 'foldl', applied to a binary operator, a starting value -- (typically the left-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from left to right. --- Subject to fusion. foldl :: (a -> Char -> a) -> a -> Text -> a foldl f z t = S.foldl f z (stream t) {-# INLINE foldl #-} --- | /O(n)/ A strict version of 'foldl'. Subject to fusion. +-- | /O(n)/ A strict version of 'foldl'. foldl' :: (a -> Char -> a) -> a -> Text -> a foldl' f z t = S.foldl' f z (stream t) {-# INLINE foldl' #-} -- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to fusion. -foldl1 :: (Char -> Char -> Char) -> Text -> Char +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldl1 f t = S.foldl1 f (stream t) {-# INLINE foldl1 #-} --- | /O(n)/ A strict version of 'foldl1'. Subject to fusion. -foldl1' :: (Char -> Char -> Char) -> Text -> Char +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldl1' f t = S.foldl1' f (stream t) {-# INLINE foldl1' #-} -- | /O(n)/ 'foldr', applied to a binary operator, a starting value -- (typically the right-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from right to left. --- Subject to fusion. foldr :: (Char -> a -> a) -> a -> Text -> a foldr f z t = S.foldr f z (stream t) {-# INLINE foldr #-} -- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to --- fusion. -foldr1 :: (Char -> Char -> Char) -> Text -> Char +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldr1 f t = S.foldr1 f (stream t) {-# INLINE foldr1 #-} @@ -980,12 +994,11 @@ _ -> Text (A.run go) 0 len where ts' = L.filter (not . null) ts - len = sumP "concat" $ L.map lengthWord16 ts' + len = sumP "concat" $ L.map lengthWord8 ts' go :: ST s (A.MArray s) go = do arr <- A.new len - let step i (Text a o l) = - let !j = i + l in A.copyI arr i a o j >> return j + let step i (Text a o l) = A.copyI l arr i a o >> return (i + l) foldM step 0 ts' >> return arr -- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and @@ -995,41 +1008,42 @@ {-# INLINE concatMap #-} -- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. Subject to fusion. +-- 'Text' @t@ satisfies the predicate @p@. any :: (Char -> Bool) -> Text -> Bool any p t = S.any p (stream t) {-# INLINE any #-} -- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. Subject to fusion. +-- 'Text' @t@ satisfy the predicate @p@. all :: (Char -> Bool) -> Text -> Bool all p t = S.all p (stream t) {-# INLINE all #-} -- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. Subject to fusion. -maximum :: Text -> Char +-- must be non-empty. +maximum :: HasCallStack => Text -> Char maximum t = S.maximum (stream t) {-# INLINE maximum #-} -- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. Subject to fusion. -minimum :: Text -> Char +-- must be non-empty. +minimum :: HasCallStack => Text -> Char minimum t = S.minimum (stream t) {-# INLINE minimum #-} -- ----------------------------------------------------------------------------- -- * Building 'Text's - -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. Subject to fusion. +-- successive reduced values from the left. -- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] -- --- Note that +-- __Properties__ -- --- > last (scanl f z xs) == foldl f z xs. +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ scanl :: (Char -> Char -> Char) -> Char -> Text -> Text scanl f z t = unstream (S.scanl g z (stream t)) where g a b = safe (f a b) @@ -1064,9 +1078,30 @@ -- function to each element of a 'Text', passing an accumulating -- parameter from left to right, and returns a final 'Text'. Performs -- replacement on invalid scalar values. -mapAccumL :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumL f z0 = S.mapAccumL g z0 . stream - where g a b = second safe (f a b) +mapAccumL :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumL f z0 = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + 4) o 0 z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> a -> ST s (a, Text) + outer !dst !dstLen = inner + where + inner !srcOff !dstOff !z + | srcOff >= l + o = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (z, Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + let !dstLen' = dstLen + (l + o) - srcOff + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff z + | otherwise = do + let !(Iter c d) = iterArray src srcOff + (z', c') = f z c + d' <- unsafeWrite dst dstOff (safe c') + inner (srcOff + d) (dstOff + d') z' {-# INLINE mapAccumL #-} -- | The 'mapAccumR' function behaves like a combination of 'map' and @@ -1075,9 +1110,35 @@ -- returning a final value of this accumulator together with the new -- 'Text'. -- Performs replacement on invalid scalar values. -mapAccumR :: (a -> Char -> (a,Char)) -> a -> Text -> (a, Text) -mapAccumR f z0 = second reverse . S.mapAccumL g z0 . reverseStream - where g a b = second safe (f a b) +mapAccumR :: forall a. (a -> Char -> (a, Char)) -> a -> Text -> (a, Text) +mapAccumR f z0 = go + where + go (Text src o l) = runST $ do + marr <- A.new (l + 4) + outer marr (l + o - 1) (l + 4 - 1) z0 + where + outer :: forall s. A.MArray s -> Int -> Int -> a -> ST s (a, Text) + outer !dst = inner + where + inner !srcOff !dstOff !z + | srcOff < o = do + dstLen <- A.getSizeofMArray dst + arr <- A.unsafeFreeze dst + return (z, Text arr (dstOff + 1) (dstLen - dstOff - 1)) + | dstOff < 3 = do + dstLen <- A.getSizeofMArray dst + let !dstLen' = dstLen + (srcOff - o) + 4 + dst' <- A.new dstLen' + A.copyM dst' (dstLen' - dstLen) dst 0 dstLen + outer dst' srcOff (dstOff + dstLen' - dstLen) z + | otherwise = do + let !(Iter c d) = reverseIterArray src (srcOff) + (z', c') = f z c + c'' = safe c' + !d' = utf8Length c'' + dstOff' = dstOff - d' + _ <- unsafeWrite dst (dstOff' + 1) c'' + inner (srcOff + d) dstOff' z' {-# INLINE mapAccumR #-} -- ----------------------------------------------------------------------------- @@ -1090,30 +1151,39 @@ | n <= 0 || l <= 0 = empty | n == 1 = t | isSingleton t = replicateChar n (unsafeHead t) - | otherwise = Text (A.run x) 0 len - where - len = l `mul` n -- TODO: detect overflows - x :: ST s (A.MArray s) - x = do - arr <- A.new len - A.copyI arr 0 a o l - let loop !l1 = - let rest = len - l1 in - if rest <= l1 then A.copyM arr l1 arr 0 rest >> return arr - else A.copyM arr l1 arr 0 l1 >> loop (l1 `shiftL` 1) - loop l + | otherwise = runST $ do + let totalLen = n `mul` l + marr <- A.new totalLen + A.copyI l marr 0 a o + A.tile marr l + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen {-# INLINE [1] replicate #-} - {-# RULES "TEXT replicate/singleton -> replicateChar" [~1] forall n c. replicate n (singleton c) = replicateChar n c #-} -- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the --- value of every element. Subject to fusion. +-- value of every element. replicateChar :: Int -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n (safe c)) +replicateChar !len !c' + | len <= 0 = empty + | isAscii c = runST $ do + marr <- A.newFilled len (ord c) + arr <- A.unsafeFreeze marr + return $ Text arr 0 len + | otherwise = runST $ do + let cLen = utf8Length c + totalLen = cLen P.* len + marr <- A.new totalLen + _ <- unsafeWrite marr 0 c + A.tile marr cLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 totalLen + where + c = safe c' {-# INLINE replicateChar #-} -- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' @@ -1121,8 +1191,8 @@ -- 'Text' from a seed value. The function takes the element and -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the --- string, and @b@ is the seed value for further production. Subject --- to fusion. Performs replacement on invalid scalar values. +-- string, and @b@ is the seed value for further production. +-- Performs replacement on invalid scalar values. unfoldr :: (a -> Maybe (Char,a)) -> a -> Text unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) {-# INLINE unfoldr #-} @@ -1131,8 +1201,8 @@ -- value. However, the length of the result should be limited by the -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and --- correct, otherwise its performance is similar to 'unfoldr'. Subject --- to fusion. Performs replacement on invalid scalar values. +-- correct, otherwise its performance is similar to 'unfoldr'. +-- Performs replacement on invalid scalar values. unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Text unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) {-# INLINE unfoldrN #-} @@ -1142,27 +1212,32 @@ -- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the -- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. Subject to fusion. +-- the length of the Text. take :: Int -> Text -> Text take n t@(Text arr off len) | n <= 0 = empty | n >= len = t - | otherwise = text arr off (iterN n t) + | otherwise = let m = measureOff n t in if m >= 0 then text arr off m else t {-# INLINE [1] take #-} -iterN :: Int -> Text -> Int -iterN n t@(Text _arr _off len) = loop 0 0 - where loop !i !cnt - | i >= len || cnt >= n = i - | otherwise = loop (i+d) (cnt+1) - where d = iter_ t i +-- | /O(n)/ If @t@ is long enough to contain @n@ characters, 'measureOff' @n@ @t@ +-- returns a non-negative number, measuring their size in 'Word8'. Otherwise, +-- if @t@ is shorter, return a non-positive number, which is a negated total count +-- of 'Char' available in @t@. If @t@ is empty or @n = 0@, return 0. +-- +-- This function is used to implement 'take', 'drop', 'splitAt' and 'length' +-- and is useful on its own in streaming and parsing libraries. +-- +-- @since 2.0 +measureOff :: Int -> Text -> Int +measureOff !n (Text (A.ByteArray arr) off len) = if len == 0 then 0 else + cSsizeToInt $ unsafeDupablePerformIO $ + c_measure_off arr (intToCSize off) (intToCSize len) (intToCSize n) -{-# RULES -"TEXT take -> fused" [~1] forall n t. - take n t = unstream (S.take n (stream t)) -"TEXT take -> unfused" [1] forall n t. - unstream (S.take n (stream t)) = take n t - #-} +-- | The input buffer (arr :: ByteArray#, off :: CSize, len :: CSize) +-- must specify a valid UTF-8 sequence, this condition is not checked. +foreign import ccall unsafe "_hs_text_measure_off" c_measure_off + :: ByteArray# -> CSize -> CSize -> CSize -> IO CSsize -- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after -- taking @n@ characters from the end of @t@. @@ -1190,24 +1265,15 @@ -- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the -- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. Subject to fusion. +-- is greater than the length of the 'Text'. drop :: Int -> Text -> Text drop n t@(Text arr off len) | n <= 0 = t | n >= len = empty - | otherwise = text arr (off+i) (len-i) - where i = iterN n t + | otherwise = if m >= 0 then text arr (off+m) (len-m) else mempty + where m = measureOff n t {-# INLINE [1] drop #-} -{-# RULES -"TEXT drop -> fused" [~1] forall n t. - drop n t = unstream (S.drop n (stream t)) -"TEXT drop -> unfused" [1] forall n t. - unstream (S.drop n (stream t)) = drop n t -"TEXT take . drop -> unfused" [1] forall len off t. - unstream (S.take len (S.drop off (stream t))) = take len (drop off t) - #-} - -- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after -- dropping @n@ characters from the end of @t@. -- @@ -1225,7 +1291,7 @@ -- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', -- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. Subject to fusion. +-- satisfy @p@. takeWhile :: (Char -> Bool) -> Text -> Text takeWhile p t@(Text arr off len) = loop 0 where loop !i | i >= len = t @@ -1234,13 +1300,6 @@ where Iter c d = iter t i {-# INLINE [1] takeWhile #-} -{-# RULES -"TEXT takeWhile -> fused" [~1] forall p t. - takeWhile p t = unstream (S.takeWhile p (stream t)) -"TEXT takeWhile -> unfused" [1] forall p t. - unstream (S.takeWhile p (stream t)) = takeWhile p t - #-} - -- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', -- returns the longest suffix (possibly empty) of elements that -- satisfy @p@. @@ -1255,11 +1314,11 @@ where loop !i !l | l <= 0 = t | p c = loop (i+d) (l+d) | otherwise = text arr (off+l) (len-l) - where (c,d) = reverseIter t i + where Iter c d = reverseIter t i {-# INLINE [1] takeWhileEnd #-} -- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. Subject to fusion. +-- 'takeWhile' @p@ @t@. dropWhile :: (Char -> Bool) -> Text -> Text dropWhile p t@(Text arr off len) = loop 0 0 where loop !i !l | l >= len = empty @@ -1268,13 +1327,6 @@ where Iter c d = iter t i {-# INLINE [1] dropWhile #-} -{-# RULES -"TEXT dropWhile -> fused" [~1] forall p t. - dropWhile p t = unstream (S.dropWhile p (stream t)) -"TEXT dropWhile -> unfused" [1] forall p t. - unstream (S.dropWhile p (stream t)) = dropWhile p t - #-} - -- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after -- dropping characters that satisfy the predicate @p@ from the end of -- @t@. @@ -1288,12 +1340,12 @@ where loop !i !l | l <= 0 = empty | p c = loop (i+d) (l+d) | otherwise = Text arr off l - where (c,d) = reverseIter t i + where Iter c d = reverseIter t i {-# INLINE [1] dropWhileEnd #-} -- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after -- dropping characters that satisfy the predicate @p@ from both the --- beginning and end of @t@. Subject to fusion. +-- beginning and end of @t@. dropAround :: (Char -> Bool) -> Text -> Text dropAround p = dropWhile p . dropWhileEnd p {-# INLINE [1] dropAround #-} @@ -1327,8 +1379,8 @@ splitAt n t@(Text arr off len) | n <= 0 = (empty, t) | n >= len = (t, empty) - | otherwise = let k = iterN n t - in (text arr off k, text arr (off+k) (len-k)) + | otherwise = let m = measureOff n t in + if m >= 0 then (text arr off m, text arr (off+m) (len-m)) else (t, mempty) -- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns -- a pair whose first element is the longest prefix (possibly empty) @@ -1361,7 +1413,7 @@ where Iter c d = iter t 0 n = d + findAIndexOrEnd (not . p c) (Text arr (off+d) (len-d)) --- | Returns the /array/ index (in units of 'Word16') at which a +-- | Returns the /array/ index (in units of 'Word8') at which a -- character may be found. This is /not/ the same as the logical -- index returned by e.g. 'findIndex'. findAIndexOrEnd :: (Char -> Bool) -> Text -> Int @@ -1417,7 +1469,8 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -splitOn :: Text +splitOn :: HasCallStack + => Text -- ^ String to split on. If this string is empty, an error -- will occur. -> Text @@ -1487,7 +1540,7 @@ -- | /O(n)/ The 'find' function takes a predicate and a 'Text', and -- returns the first element matching the predicate, or 'Nothing' if --- there is no such element. Subject to fusion. +-- there is no such element. find :: (Char -> Bool) -> Text -> Maybe Char find p t = S.findBy p (stream t) {-# INLINE find #-} @@ -1505,9 +1558,74 @@ -- returns a 'Text' containing those characters that satisfy the -- predicate. filter :: (Char -> Bool) -> Text -> Text -filter p t = unstream (S.filter p (stream t)) -{-# INLINE filter #-} +filter p = go + where + go (Text src o l) = runST $ do + -- It's tempting to allocate l elements at once and avoid resizing. + -- However, this can be unacceptable in scenarios where a huge array + -- is filtered with a rare predicate, resulting in a much shorter buffer. + let !dstLen = min l 64 + dst <- A.new dstLen + outer dst dstLen o 0 + where + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s Text + outer !dst !dstLen = inner + where + inner !srcOff !dstOff + | srcOff >= o + l = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff) + | dstOff + 4 > dstLen = do + -- Double size of the buffer, unless it becomes longer than + -- source string. Ensure to extend it by least 4 bytes. + let !dstLen' = dstLen + max 4 (min (l + o - srcOff) dstLen) + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + -- In case of success, filter writes exactly the same character + -- it just read (this is not a case for map, for example). + -- We leverage this fact below: no need to decode Char back into UTF8, + -- just copy bytes from input. + | otherwise = do + let m0 = A.unsafeIndex src srcOff + m1 = A.unsafeIndex src (srcOff + 1) + m2 = A.unsafeIndex src (srcOff + 2) + m3 = A.unsafeIndex src (srcOff + 3) + !d = utf8LengthByLeader m0 + case d of + 1 -> do + let !c = unsafeChr8 m0 + if not (p c) then inner (srcOff + 1) dstOff else do + A.unsafeWrite dst dstOff m0 + inner (srcOff + 1) (dstOff + 1) + 2 -> do + let !c = chr2 m0 m1 + if not (p c) then inner (srcOff + 2) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + inner (srcOff + 2) (dstOff + 2) + 3 -> do + let !c = chr3 m0 m1 m2 + if not (p c) then inner (srcOff + 3) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + inner (srcOff + 3) (dstOff + 3) + _ -> do + let !c = chr4 m0 m1 m2 m3 + if not (p c) then inner (srcOff + 4) dstOff else do + A.unsafeWrite dst dstOff m0 + A.unsafeWrite dst (dstOff + 1) m1 + A.unsafeWrite dst (dstOff + 2) m2 + A.unsafeWrite dst (dstOff + 3) m3 + inner (srcOff + 4) (dstOff + 4) +{-# INLINE [1] filter #-} +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> p c && q c) t +#-} + -- | /O(n+m)/ Find the first instance of @needle@ (which must be -- non-'null') in @haystack@. The first element of the returned tuple -- is the prefix of @haystack@ before @needle@ is matched. The second @@ -1532,7 +1650,7 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -breakOn :: Text -> Text -> (Text, Text) +breakOn :: HasCallStack => Text -> Text -> (Text, Text) breakOn pat src@(Text arr off len) | null pat = emptyError "breakOn" | otherwise = case indices pat src of @@ -1549,7 +1667,7 @@ -- -- >>> breakOnEnd "::" "a::b::c" -- ("a::b::","c") -breakOnEnd :: Text -> Text -> (Text, Text) +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) breakOnEnd pat src = (reverse b, reverse a) where (a,b) = breakOn (reverse pat) (reverse src) {-# INLINE breakOnEnd #-} @@ -1573,7 +1691,8 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty. -breakOnAll :: Text -- ^ @needle@ to search for +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for -> Text -- ^ @haystack@ in which to search -> [(Text, Text)] breakOnAll pat src@(Text arr off slen) @@ -1603,14 +1722,14 @@ -- searching for the index of @\"::\"@ and taking the substrings -- before and after that index, you would instead use @breakOnAll \"::\"@. --- | /O(n)/ 'Text' index (subscript) operator, starting from 0. Subject to fusion. -index :: Text -> Int -> Char +-- | /O(n)/ 'Text' index (subscript) operator, starting from 0. +index :: HasCallStack => Text -> Int -> Char index t n = S.index (stream t) n {-# INLINE index #-} -- | /O(n)/ The 'findIndex' function takes a predicate and a 'Text' -- and returns the index of the first element in the 'Text' satisfying --- the predicate. Subject to fusion. +-- the predicate. findIndex :: (Char -> Bool) -> Text -> Maybe Int findIndex p t = S.findIndex p (stream t) {-# INLINE findIndex #-} @@ -1621,11 +1740,11 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -count :: Text -> Text -> Int -count pat src +count :: HasCallStack => Text -> Text -> Int +count pat | null pat = emptyError "count" - | isSingleton pat = countChar (unsafeHead pat) src - | otherwise = L.length (indices pat src) + | isSingleton pat = countChar (unsafeHead pat) + | otherwise = L.length . indices pat {-# INLINE [1] count #-} {-# RULES @@ -1634,7 +1753,7 @@ #-} -- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. Subject to fusion. +-- query element appears in the given 'Text'. countChar :: Char -> Text -> Int countChar c t = S.countChar c (stream t) {-# INLINE countChar #-} @@ -1656,61 +1775,71 @@ zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2)) where g a b = safe (f a b) -{-# INLINE zipWith #-} +{-# INLINE [1] zipWith #-} -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text] -words t@(Text arr off len) = loop 0 0 +words (Text arr off len) = loop 0 0 where loop !start !n | n >= len = if start == n then [] - else [Text arr (start+off) (n-start)] - | isSpace c = + else [Text arr (start + off) (n - start)] + -- Spaces in UTF-8 take either 1 byte for 0x09..0x0D + 0x20 + | isAsciiSpace w0 = if start == n - then loop (start+1) (start+1) - else Text arr (start+off) (n-start) : loop (n+d) (n+d) - | otherwise = loop start (n+d) - where Iter c d = iter t n + then loop (n + 1) (n + 1) + else Text arr (start + off) (n - start) : loop (n + 1) (n + 1) + | w0 < 0x80 = loop start (n + 1) + -- or 2 bytes for 0xA0 + | w0 == 0xC2, w1 == 0xA0 = + if start == n + then loop (n + 2) (n + 2) + else Text arr (start + off) (n - start) : loop (n + 2) (n + 2) + | w0 < 0xE0 = loop start (n + 2) + -- or 3 bytes for 0x1680 + 0x2000..0x200A + 0x2028..0x2029 + 0x202F + 0x205F + 0x3000 + | w0 == 0xE1 && w1 == 0x9A && w2 == 0x80 + || w0 == 0xE2 && (w1 == 0x80 && isSpace (chr3 w0 w1 w2) || w1 == 0x81 && w2 == 0x9F) + || w0 == 0xE3 && w1 == 0x80 && w2 == 0x80 = + if start == n + then loop (n + 3) (n + 3) + else Text arr (start + off) (n - start) : loop (n + 3) (n + 3) + | otherwise = loop start (n + utf8LengthByLeader w0) + where + w0 = A.unsafeIndex arr (off + n) + w1 = A.unsafeIndex arr (off + n + 1) + w2 = A.unsafeIndex arr (off + n + 2) {-# INLINE words #-} --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at --- newline 'Char's. The resulting strings do not contain newlines. +-- Adapted from Data.ByteString.Internal.isSpaceWord8 +isAsciiSpace :: Word8 -> Bool +isAsciiSpace w = w .&. 0x50 == 0 && w < 0x80 && (w == 0x20 || w - 0x09 < 5) +{-# INLINE isAsciiSpace #-} + +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. lines :: Text -> [Text] -lines ps | null ps = [] - | otherwise = h : if null t - then [] - else lines (unsafeTail t) - where (# h,t #) = span_ (/= '\n') ps +lines (Text arr@(A.ByteArray arr#) off len) = go off + where + go !n + | n >= len + off = [] + | delta < 0 = [Text arr n (len + off - n)] + | otherwise = Text arr n delta : go (n + delta + 1) + where + delta = cSsizeToInt $ unsafeDupablePerformIO $ + memchr arr# (intToCSize n) (intToCSize (len + off - n)) 0x0A {-# INLINE lines #-} -{- --- | /O(n)/ Portably breaks a 'Text' up into a list of 'Text's at line --- boundaries. --- --- A line boundary is considered to be either a line feed, a carriage --- return immediately followed by a line feed, or a carriage return. --- This accounts for both Unix and Windows line ending conventions, --- and for the old convention used on Mac OS 9 and earlier. -lines' :: Text -> [Text] -lines' ps | null ps = [] - | otherwise = h : case uncons t of - Nothing -> [] - Just (c,t') - | c == '\n' -> lines t' - | c == '\r' -> case uncons t' of - Just ('\n',t'') -> lines t'' - _ -> lines t' - where (h,t) = span notEOL ps - notEOL c = c /= '\n' && c /= '\r' -{-# INLINE lines' #-} --} +foreign import ccall unsafe "_hs_text_memchr" memchr + :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize -- | /O(n)/ Joins lines, after appending a terminating newline to -- each. unlines :: [Text] -> Text -unlines = concat . L.map (`snoc` '\n') +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] {-# INLINE unlines #-} -- | /O(n)/ Joins words using single space characters. @@ -1719,17 +1848,12 @@ {-# INLINE unwords #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. Subject to fusion. +-- 'True' iff the first is a prefix of the second. isPrefixOf :: Text -> Text -> Bool isPrefixOf a@(Text _ _ alen) b@(Text _ _ blen) = alen <= blen && S.isPrefixOf (stream a) (stream b) {-# INLINE [1] isPrefixOf #-} -{-# RULES -"TEXT isPrefixOf -> fused" [~1] forall s t. - isPrefixOf s t = S.isPrefixOf (stream s) (stream t) - #-} - -- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns -- 'True' iff the first is a suffix of the second. isSuffixOf :: Text -> Text -> Bool @@ -1757,11 +1881,6 @@ | otherwise = not . L.null . indices needle $ haystack {-# INLINE [1] isInfixOf #-} -{-# RULES -"TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. - isInfixOf (singleton n) h = S.elem n (S.stream h) - #-} - ------------------------------------------------------------------------------- -- * View patterns @@ -1810,16 +1929,26 @@ -- -- >>> commonPrefixes "" "baz" -- Nothing -commonPrefixes :: Text -> Text -> Maybe (Text,Text,Text) -commonPrefixes t0@(Text arr0 off0 len0) t1@(Text arr1 off1 len1) = go 0 0 +commonPrefixes :: Text -> Text -> Maybe (Text, Text, Text) +commonPrefixes !t0@(Text arr0 off0 len0) !t1@(Text arr1 off1 len1) + | len0 == 0 = Nothing + | len1 == 0 = Nothing + | otherwise = go 0 0 where - go !i !j | i < len0 && j < len1 && a == b = go (i+d0) (j+d1) - | i > 0 = Just (Text arr0 off0 i, - text arr0 (off0+i) (len0-i), - text arr1 (off1+j) (len1-j)) - | otherwise = Nothing - where Iter a d0 = iter t0 i - Iter b d1 = iter t1 j + go !i !j + | i == len0 = Just (t0, empty, text arr1 (off1 + i) (len1 - i)) + | i == len1 = Just (t1, text arr0 (off0 + i) (len0 - i), empty) + | a == b = go (i + 1) k + | k > 0 = Just (Text arr0 off0 k, + Text arr0 (off0 + k) (len0 - k), + Text arr1 (off1 + k) (len1 - k)) + | otherwise = Nothing + where + a = A.unsafeIndex arr0 (off0 + i) + b = A.unsafeIndex arr1 (off1 + i) + isLeader = word8ToInt8 a >= -64 + k = if isLeader then i else j +{-# INLINE commonPrefixes #-} -- | /O(n)/ Return the prefix of the second string if its suffix -- matches the entire first string. @@ -1858,10 +1987,10 @@ where ax = a + x go a _ = a -emptyError :: String -> a +emptyError :: HasCallStack => String -> a emptyError fun = P.error $ "Data.Text." ++ fun ++ ": empty input" -overflowError :: String -> a +overflowError :: HasCallStack => String -> a overflowError fun = P.error $ "Data.Text." ++ fun ++ ": size overflow" -- | /O(n)/ Make a distinct copy of the given string, sharing no @@ -1878,9 +2007,20 @@ go :: ST s (A.MArray s) go = do marr <- A.new len - A.copyI marr 0 arr off len + A.copyI len marr 0 arr off return marr +ord8 :: Char -> Word8 +ord8 = P.fromIntegral . ord + +intToCSize :: Int -> CSize +intToCSize = P.fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = P.fromIntegral + +word8ToInt8 :: Word8 -> Int8 +word8ToInt8 = P.fromIntegral ------------------------------------------------- -- NOTE: the named chunk below used by doctest; diff --git a/src/Data/Text/Array.hs b/src/Data/Text/Array.hs --- a/src/Data/Text/Array.hs +++ b/src/Data/Text/Array.hs @@ -27,108 +27,147 @@ Array(..) , MArray(..) -- * Functions + , resizeM + , shrinkM , copyM , copyI + , copyFromPointer + , copyToPointer , empty , equal + , compare , run , run2 , toList , unsafeFreeze , unsafeIndex , new + , newPinned + , newFilled , unsafeWrite + , tile + , getSizeofMArray ) where #if defined(ASSERTS) -import Control.Exception (assert) -import GHC.Base (sizeofByteArray#, sizeofMutableByteArray#) import GHC.Stack (HasCallStack) #endif -import Control.Monad.ST.Unsafe (unsafeIOToST) -import Data.Bits ((.&.), xor) +#if !MIN_VERSION_base(4,11,0) import Data.Text.Internal.Unsafe (inlinePerformIO) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) -import Foreign.C.Types (CInt(CInt), CSize(CSize)) -import GHC.Base (ByteArray#, MutableByteArray#, Int(..), - indexWord16Array#, newByteArray#, - unsafeFreezeByteArray#, writeWord16Array#) +import Foreign.C.Types (CInt(..)) +#endif +import GHC.Exts hiding (toList) import GHC.ST (ST(..), runST) -import GHC.Word (Word16(..)) -import Prelude hiding (length, read) +import GHC.Word (Word8(..)) +import qualified Prelude +import Prelude hiding (length, read, compare) -- | Immutable array type. --- --- The 'Array' constructor is exposed since @text-1.1.1.3@ -data Array = Array { aBA :: ByteArray# } +data Array = ByteArray ByteArray# -- | Mutable array type, for use in the ST monad. --- --- The 'MArray' constructor is exposed since @text-1.1.1.3@ -data MArray s = MArray { maBA :: MutableByteArray# s } +data MArray s = MutableByteArray (MutableByteArray# s) -- | Create an uninitialized mutable array. new :: forall s. Int -> ST s (MArray s) -new n - | n < 0 || n .&. highBit /= 0 = array_size_error +new (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.new: size overflow" +#endif | otherwise = ST $ \s1# -> - case newByteArray# len# s1# of - (# s2#, marr# #) -> (# s2#, MArray marr# #) - where !(I# len#) = bytesInArray n - highBit = maxBound `xor` (maxBound `shiftR` 1) + case newByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) {-# INLINE new #-} -array_size_error :: a -array_size_error = error "Data.Text.Array.new: size overflow" +-- | Create an uninitialized mutable pinned array. +-- +-- @since 2.0 +newPinned :: forall s. Int -> ST s (MArray s) +newPinned (I# len#) +#if defined(ASSERTS) + | I# len# < 0 = error "Data.Text.Array.newPinned: size overflow" +#endif + | otherwise = ST $ \s1# -> + case newPinnedByteArray# len# s1# of + (# s2#, marr# #) -> (# s2#, MutableByteArray marr# #) +{-# INLINE newPinned #-} +-- | @since 2.0 +newFilled :: Int -> Int -> ST s (MArray s) +newFilled (I# len#) (I# c#) = ST $ \s1# -> + case newByteArray# len# s1# of + (# s2#, marr# #) -> case setByteArray# marr# 0# len# c# s2# of + s3# -> (# s3#, MutableByteArray marr# #) +{-# INLINE newFilled #-} + +-- | @since 2.0 +tile :: MArray s -> Int -> ST s () +tile marr tileLen = do + totalLen <- getSizeofMArray marr + let go l + | 2 * l > totalLen = copyM marr l marr 0 (totalLen - l) + | otherwise = copyM marr l marr 0 l >> go (2 * l) + go tileLen +{-# INLINE tile #-} + -- | Freeze a mutable array. Do not mutate the 'MArray' afterwards! unsafeFreeze :: MArray s -> ST s Array -unsafeFreeze MArray{..} = ST $ \s1# -> - case unsafeFreezeByteArray# maBA s1# of - (# s2#, ba# #) -> (# s2#, Array ba# #) +unsafeFreeze (MutableByteArray marr) = ST $ \s1# -> + case unsafeFreezeByteArray# marr s1# of + (# s2#, ba# #) -> (# s2#, ByteArray ba# #) {-# INLINE unsafeFreeze #-} --- | Indicate how many bytes would be used for an array of the given --- size. -bytesInArray :: Int -> Int -bytesInArray n = n `shiftL` 1 -{-# INLINE bytesInArray #-} - -- | Unchecked read of an immutable array. May return garbage or -- crash on an out-of-bounds access. unsafeIndex :: #if defined(ASSERTS) HasCallStack => #endif - Array -> Int -> Word16 -unsafeIndex a@Array{..} i@(I# i#) = + Array -> Int -> Word8 +unsafeIndex (ByteArray arr) i@(I# i#) = #if defined(ASSERTS) - let word16len = I# (sizeofByteArray# aBA) `quot` 2 in - if i < 0 || i >= word16len - then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word16len) + let word8len = I# (sizeofByteArray# arr) in + if i < 0 || i >= word8len + then error ("Data.Text.Array.unsafeIndex: bounds error, offset " ++ show i ++ ", length " ++ show word8len) else #endif - case indexWord16Array# aBA i# of r# -> (W16# r#) + case indexWord8Array# arr i# of r# -> (W8# r#) {-# INLINE unsafeIndex #-} +-- | @since 2.0 +getSizeofMArray :: MArray s -> ST s Int +getSizeofMArray (MutableByteArray marr) = ST $ \s0# -> + -- Cannot simply use (deprecated) 'sizeofMutableByteArray#', because it is + -- unsafe in the presence of 'shrinkMutableByteArray#' and 'resizeMutableByteArray#'. + case getSizeofMutableByteArray# marr s0# of + (# s1#, word8len# #) -> (# s1#, I# word8len# #) + +#if defined(ASSERTS) +checkBoundsM :: HasCallStack => MArray s -> Int -> Int -> ST s () +checkBoundsM ma i elSize = do + len <- getSizeofMArray ma + if i < 0 || i + elSize > len + then error ("bounds error, offset " ++ show i ++ ", length " ++ show len) + else return () +#endif + -- | Unchecked write of a mutable array. May return garbage or crash -- on an out-of-bounds access. unsafeWrite :: #if defined(ASSERTS) HasCallStack => #endif - MArray s -> Int -> Word16 -> ST s () -unsafeWrite ma@MArray{..} i@(I# i#) (W16# e#) = ST $ \s1# -> + MArray s -> Int -> Word8 -> ST s () +unsafeWrite ma@(MutableByteArray marr) i@(I# i#) (W8# e#) = #if defined(ASSERTS) - let word16len = I# (sizeofMutableByteArray# maBA) `quot` 2 in - if i < 0 || i >= word16len then error ("Data.Text.Array.unsafeWrite: bounds error, offset " ++ show i ++ ", length " ++ show word16len) else + checkBoundsM ma i 1 >> #endif - case writeWord16Array# maBA i# e# s1# of - s2# -> (# s2#, () #) + (ST $ \s1# -> case writeWord8Array# marr i# e# s1# of + s2# -> (# s2#, () #)) {-# INLINE unsafeWrite #-} -- | Convert an immutable array to a list. -toList :: Array -> Int -> Int -> [Word16] +toList :: Array -> Int -> Int -> [Word8] toList ary off len = loop 0 where loop i | i < len = unsafeIndex ary (off+i) : loop (i+1) | otherwise = [] @@ -151,6 +190,31 @@ return (arr,b)) {-# INLINE run2 #-} +-- | @since 2.0 +resizeM :: MArray s -> Int -> ST s (MArray s) +resizeM (MutableByteArray ma) i@(I# i#) = ST $ \s1# -> + case resizeMutableByteArray# ma i# s1# of + (# s2#, newArr #) -> (# s2#, MutableByteArray newArr #) +{-# INLINE resizeM #-} + +-- | @since 2.0 +shrinkM :: +#if defined(ASSERTS) + HasCallStack => +#endif + MArray s -> Int -> ST s () +shrinkM (MutableByteArray marr) i@(I# newSize) = do +#if defined(ASSERTS) + oldSize <- getSizeofMArray (MutableByteArray marr) + if I# newSize > oldSize + then error $ "shrinkM: shrink cannot grow " ++ show oldSize ++ " to " ++ show (I# newSize) + else return () +#endif + ST $ \s1# -> + case shrinkMutableByteArray# marr newSize s1# of + s2# -> (# s2#, () #) +{-# INLINE shrinkM #-} + -- | Copy some elements of a mutable array. copyM :: MArray s -- ^ Destination -> Int -- ^ Destination offset @@ -158,57 +222,109 @@ -> Int -- ^ Source offset -> Int -- ^ Count -> ST s () -copyM dest didx src sidx count - | count <= 0 = return () - | otherwise = +copyM dst@(MutableByteArray dst#) dstOff@(I# dstOff#) src@(MutableByteArray src#) srcOff@(I# srcOff#) count@(I# count#) #if defined(ASSERTS) - assert (sidx + count <= I# (sizeofMutableByteArray# (maBA src)) `quot` 2) . - assert (didx + count <= I# (sizeofMutableByteArray# (maBA dest)) `quot` 2) . + | count < 0 = error $ + "copyM: count must be >= 0, but got " ++ show count #endif - unsafeIOToST $ memcpyM (maBA dest) (intToCSize didx) - (maBA src) (intToCSize sidx) - (intToCSize count) + | otherwise = do +#if defined(ASSERTS) + srcLen <- getSizeofMArray src + dstLen <- getSizeofMArray dst + if srcOff + count > srcLen + then error "copyM: source is too short" + else return () + if dstOff + count > dstLen + then error "copyM: destination is too short" + else return () +#endif + ST $ \s1# -> case copyMutableByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) {-# INLINE copyM #-} -- | Copy some elements of an immutable array. -copyI :: MArray s -- ^ Destination +copyI :: Int -- ^ Count + -> MArray s -- ^ Destination -> Int -- ^ Destination offset -> Array -- ^ Source -> Int -- ^ Source offset - -> Int -- ^ First offset in destination /not/ to - -- copy (i.e. /not/ length) -> ST s () -copyI dest i0 src j0 top - | i0 >= top = return () - | otherwise = unsafeIOToST $ - memcpyI (maBA dest) (intToCSize i0) - (aBA src) (intToCSize j0) - (intToCSize (top-i0)) +copyI count@(I# count#) (MutableByteArray dst#) dstOff@(I# dstOff#) (ByteArray src#) (I# srcOff#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyI: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArray# src# srcOff# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) {-# INLINE copyI #-} +-- | Copy from pointer. +-- +-- @since 2.0 +copyFromPointer + :: MArray s -- ^ Destination + -> Int -- ^ Destination offset + -> Ptr Word8 -- ^ Source + -> Int -- ^ Count + -> ST s () +copyFromPointer (MutableByteArray dst#) dstOff@(I# dstOff#) (Ptr src#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyFromPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyAddrToByteArray# src# dst# dstOff# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyFromPointer #-} + +-- | Copy to pointer. +-- +-- @since 2.0 +copyToPointer + :: Array -- ^ Source + -> Int -- ^ Source offset + -> Ptr Word8 -- ^ Destination + -> Int -- ^ Count + -> ST s () +copyToPointer (ByteArray src#) srcOff@(I# srcOff#) (Ptr dst#) count@(I# count#) +#if defined(ASSERTS) + | count < 0 = error $ + "copyToPointer: count must be >= 0, but got " ++ show count +#endif + | otherwise = ST $ \s1# -> + case copyByteArrayToAddr# src# srcOff# dst# count# s1# of + s2# -> (# s2#, () #) +{-# INLINE copyToPointer #-} + -- | Compare portions of two arrays for equality. No bounds checking -- is performed. -equal :: Array -- ^ First +equal :: Array -> Int -> Array -> Int -> Int -> Bool +equal src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count == 0 +{-# INLINE equal #-} + +-- | Compare portions of two arrays. No bounds checking is performed. +-- +-- @since 2.0 +compare :: Array -> Int -> Array -> Int -> Int -> Ordering +compare src1 off1 src2 off2 count = compareInternal src1 off1 src2 off2 count `Prelude.compare` 0 +{-# INLINE compare #-} + +compareInternal + :: Array -- ^ First -> Int -- ^ Offset into first -> Array -- ^ Second -> Int -- ^ Offset into second -> Int -- ^ Count - -> Bool -equal arrA offA arrB offB count = inlinePerformIO $ do - i <- memcmp (aBA arrA) (intToCSize offA) - (aBA arrB) (intToCSize offB) (intToCSize count) - return $! i == 0 -{-# INLINE equal #-} - -intToCSize :: Int -> CSize -intToCSize = fromIntegral - -foreign import ccall unsafe "_hs_text_memcpy" memcpyI - :: MutableByteArray# s -> CSize -> ByteArray# -> CSize -> CSize -> IO () - -foreign import ccall unsafe "_hs_text_memcmp" memcmp - :: ByteArray# -> CSize -> ByteArray# -> CSize -> CSize -> IO CInt + -> Int +compareInternal (ByteArray src1#) (I# off1#) (ByteArray src2#) (I# off2#) (I# count#) = i + where +#if MIN_VERSION_base(4,11,0) + i = I# (compareByteArrays# src1# off1# src2# off2# count#) +#else + i = fromIntegral (inlinePerformIO (memcmp src1# off1# src2# off2# count#)) -foreign import ccall unsafe "_hs_text_memcpy" memcpyM - :: MutableByteArray# s -> CSize -> MutableByteArray# s -> CSize -> CSize - -> IO () +foreign import ccall unsafe "_hs_text_memcmp2" memcmp + :: ByteArray# -> Int# -> ByteArray# -> Int# -> Int# -> IO CInt +#endif +{-# INLINE compareInternal #-} diff --git a/src/Data/Text/Encoding.hs b/src/Data/Text/Encoding.hs --- a/src/Data/Text/Encoding.hs +++ b/src/Data/Text/Encoding.hs @@ -2,11 +2,14 @@ UnliftedFFITypes #-} {-# LANGUAGE Trustworthy #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE ViewPatterns #-} -- | -- Module : Data.Text.Encoding -- Copyright : (c) 2009, 2010, 2011 Bryan O'Sullivan, -- (c) 2009 Duncan Coutts, -- (c) 2008, 2009 Tom Harper +-- (c) 2021 Andrew Lelechenko -- -- License : BSD-style -- Maintainer : bos@serpentine.com @@ -35,6 +38,7 @@ -- ** Controllable error handling , decodeUtf8With + , decodeUtf8Lenient , decodeUtf16LEWith , decodeUtf16BEWith , decodeUtf32LEWith @@ -60,40 +64,44 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO) -import Control.Exception (evaluate, try, throwIO, ErrorCall(ErrorCall)) -import Control.Monad.ST (runST) -import Data.Bits ((.&.)) -import Data.ByteString as B +import Control.Exception (evaluate, try) +import Control.Monad.ST (runST, ST) +import Data.Bits (shiftR, (.&.)) +import Data.ByteString (ByteString) +import qualified Data.ByteString as B import qualified Data.ByteString.Internal as B -import Data.Foldable (traverse_) -import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode) -import Data.Text.Internal (Text(..), safe, text) -import Data.Text.Internal.Functions -import Data.Text.Internal.Private (runText) -import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftR) -import Data.Text.Show () +import qualified Data.ByteString.Short.Internal as SBS +import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode, lenientDecode) +import Data.Text.Internal (Text(..), safe, empty, append) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Internal.Unsafe.Char (unsafeWrite) +import Data.Text.Show as T (singleton) import Data.Text.Unsafe (unsafeDupablePerformIO) -import Data.Word (Word8, Word16, Word32) -import Foreign.C.Types (CSize(CSize)) -import Foreign.Marshal.Utils (with) -import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr) -import Foreign.Storable (Storable, peek, poke) -import GHC.Base (ByteArray#, MutableByteArray#) +import Data.Word (Word8) +import Foreign.C.Types (CSize(..)) +import Foreign.Ptr (Ptr, minusPtr, plusPtr) +import Foreign.Storable (poke, peekByteOff) +import GHC.Exts (byteArrayContents#, unsafeCoerce#) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(PlainPtr)) import qualified Data.ByteString.Builder as B import qualified Data.ByteString.Builder.Internal as B hiding (empty, append) import qualified Data.ByteString.Builder.Prim as BP import qualified Data.ByteString.Builder.Prim.Internal as BP +import Data.Text.Internal.Encoding.Utf8 (utf8DecodeStart, utf8DecodeContinue, DecoderResult(..)) import qualified Data.Text.Array as A import qualified Data.Text.Internal.Encoding.Fusion as E -import qualified Data.Text.Internal.Encoding.Utf16 as U16 import qualified Data.Text.Internal.Fusion as F import Data.Text.Internal.ByteStringCompat #if defined(ASSERTS) import GHC.Stack (HasCallStack) #endif -#include "text_cbits.h" +#ifdef SIMDUTF +import Foreign.C.Types (CInt(..)) +#else +import qualified Data.ByteString.Unsafe as B +import Data.Text.Internal.Encoding.Utf8 (CodePoint(..)) +#endif -- $strict -- @@ -108,106 +116,187 @@ -- 'decodeUtf8With' allows the programmer to determine what to do on a -- decoding error. --- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII +-- | Decode a 'ByteString' containing 7-bit ASCII -- encoded text. +-- +-- This is a partial function: it checks that input does not contain +-- anything except ASCII and copies buffer or throws an error otherwise. +-- decodeASCII :: ByteString -> Text -decodeASCII = decodeUtf8 -{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-} +decodeASCII bs = withBS bs $ \fp len -> if len == 0 then empty else runST $ do + asciiPrefixLen <- fmap cSizeToInt $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii src (src `plusPtr` len) + if asciiPrefixLen == len + then let !(SBS.SBS arr) = SBS.toShort bs in + return (Text (A.ByteArray arr) 0 len) + else error $ "decodeASCII: detected non-ASCII codepoint at " ++ show asciiPrefixLen -- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. -- -- 'decodeLatin1' is semantically equivalent to -- @Data.Text.pack . Data.ByteString.Char8.unpack@ +-- +-- This is a total function. However, bear in mind that decoding Latin-1 (non-ASCII) +-- characters to UTf-8 requires actual work and is not just buffer copying. +-- decodeLatin1 :: #if defined(ASSERTS) HasCallStack => #endif ByteString -> Text -decodeLatin1 bs = withBS bs aux where - aux fp len = text a 0 len - where - a = A.run (A.new len >>= unsafeIOToST . go) - go dest = unsafeWithForeignPtr fp $ \ptr -> do - c_decode_latin1 (A.maBA dest) ptr (ptr `plusPtr` len) - return dest +decodeLatin1 bs = withBS bs $ \fp len -> runST $ do + dst <- A.new (2 * len) + let inner srcOff dstOff = if srcOff >= len then return dstOff else do + asciiPrefixLen <- fmap cSizeToInt $ unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + c_is_ascii (src `plusPtr` srcOff) (src `plusPtr` len) + if asciiPrefixLen == 0 + then do + byte <- unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> peekByteOff src srcOff + A.unsafeWrite dst dstOff (0xC0 + (byte `shiftR` 6)) + A.unsafeWrite dst (dstOff + 1) (0x80 + (byte .&. 0x3F)) + inner (srcOff + 1) (dstOff + 2) + else do + unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff (src `plusPtr` srcOff) asciiPrefixLen + inner (srcOff + asciiPrefixLen) (dstOff + asciiPrefixLen) + actualLen <- inner 0 0 + dst' <- A.resizeM dst actualLen + arr <- A.unsafeFreeze dst' + return $ Text arr 0 actualLen + +foreign import ccall unsafe "_hs_text_is_ascii" c_is_ascii + :: Ptr Word8 -> Ptr Word8 -> IO CSize + +isValidBS :: ByteString -> Bool +#ifdef SIMDUTF +isValidBS bs = withBS bs $ \fp len -> unsafeDupablePerformIO $ + unsafeWithForeignPtr fp $ \ptr -> (/= 0) <$> c_is_valid_utf8 ptr (fromIntegral len) +#else +#if MIN_VERSION_bytestring(0,11,2) +isValidBS = B.isValidUtf8 +#else +isValidBS bs = start 0 + where + start ix + | ix >= B.length bs = True + | otherwise = case utf8DecodeStart (B.unsafeIndex bs ix) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st _ -> step (ix + 1) st + step ix st + | ix >= B.length bs = False + -- We do not use decoded code point, so passing a dummy value to save an argument. + | otherwise = case utf8DecodeContinue (B.unsafeIndex bs ix) st (CodePoint 0) of + Accept{} -> start (ix + 1) + Reject{} -> False + Incomplete st' _ -> step (ix + 1) st' +#endif +#endif + -- | Decode a 'ByteString' containing UTF-8 encoded text. -- --- __NOTE__: The replacement character returned by 'OnDecodeError' --- MUST be within the BMP plane; surrogate code points will --- automatically be remapped to the replacement char @U+FFFD@ --- (/since 0.11.3.0/), whereas code points beyond the BMP will throw an --- 'error' (/since 1.2.3.1/); For earlier versions of @text@ using --- those unsupported code points would result in undefined behavior. +-- Surrogate code points in replacement character returned by 'OnDecodeError' +-- will be automatically remapped to the replacement char @U+FFFD@. decodeUtf8With :: #if defined(ASSERTS) HasCallStack => #endif OnDecodeError -> ByteString -> Text -decodeUtf8With onErr bs = withBS bs aux - where - aux fp len = runText $ \done -> do - let go dest = unsafeWithForeignPtr fp $ \ptr -> - with (0::CSize) $ \destOffPtr -> do - let end = ptr `plusPtr` len - loop curPtr = do - curPtr' <- c_decode_utf8 (A.maBA dest) destOffPtr curPtr end - if curPtr' == end - then do - n <- peek destOffPtr - unsafeSTToIO (done dest (cSizeToInt n)) - else do - x <- peek curPtr' - case onErr desc (Just x) of - Nothing -> loop $ curPtr' `plusPtr` 1 - Just c - | c > '\xFFFF' -> throwUnsupportedReplChar - | otherwise -> do - destOff <- peek destOffPtr - w <- unsafeSTToIO $ - unsafeWrite dest (cSizeToInt destOff) - (safe c) - poke destOffPtr (destOff + intToCSize w) - loop $ curPtr' `plusPtr` 1 - loop ptr - (unsafeIOToST . go) =<< A.new len - where - desc = "Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream" +decodeUtf8With onErr bs + | isValidBS bs = + let !(SBS.SBS arr) = SBS.toShort bs in + (Text (A.ByteArray arr) 0 (B.length bs)) + | B.null undecoded = txt + | otherwise = txt `append` (case onErr desc (Just (B.head undecoded)) of + Nothing -> txt' + Just c -> T.singleton c `append` txt') + where + (txt, undecoded) = decodeUtf8With2 onErr mempty bs + txt' = decodeUtf8With onErr (B.tail undecoded) + desc = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" - throwUnsupportedReplChar = throwIO $ - ErrorCall "decodeUtf8With: non-BMP replacement characters not supported" - -- TODO: The code currently assumes that the transcoded UTF-16 - -- stream is at most twice as long (in bytes) as the input UTF-8 - -- stream. To justify this assumption one has to assume that the - -- error handler replacement character also satisfies this - -- invariant, by emitting at most one UTF16 code unit. - -- - -- One easy way to support the full range of code-points for - -- replacement characters in the error handler is to simply change - -- the (over-)allocation to `A.new (2*len)` and then shrink back the - -- `ByteArray#` to the real size (recent GHCs have a cheap - -- `ByteArray#` resize-primop for that which allow the GC to reclaim - -- the overallocation). However, this would require 4 times as much - -- (temporary) storage as the original UTF-8 required. - -- - -- Another strategy would be to optimistically assume that - -- replacement characters are within the BMP, and if the case of a - -- non-BMP replacement occurs reallocate the target buffer (or throw - -- an exception, and fallback to a pessimistic codepath, like e.g. - -- `decodeUtf8With onErr bs = F.unstream (E.streamUtf8 onErr bs)`) - -- - -- Alternatively, `OnDecodeError` could become a datastructure which - -- statically encodes the replacement-character range, - -- e.g. something isomorphic to - -- - -- Either (... -> Maybe Word16) (... -> Maybe Char) - -- - -- And allow to statically switch between the BMP/non-BMP - -- replacement-character codepaths. There's multiple ways to address - -- this with different tradeoffs; but ideally we should optimise for - -- the optimistic/error-free case. -{- INLINE[0] decodeUtf8With #-} +-- | Decode two consecutive bytestrings, returning Text and undecoded remainder. +decodeUtf8With2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + OnDecodeError -> ByteString -> ByteString -> (Text, ByteString) +decodeUtf8With2 onErr bs1@(B.length -> len1) bs2@(B.length -> len2) = runST $ do + marr <- A.new len' + outer marr len' 0 0 + where + len = len1 + len2 + len' = len + 4 + index i + | i < len1 = B.index bs1 i + | otherwise = B.index bs2 (i - len1) + + -- We need Data.ByteString.findIndexEnd, but it is unavailable before bytestring-0.10.12.0 + guessUtf8Boundary :: Int + guessUtf8Boundary + | len2 >= 1 && w0 < 0x80 = len2 -- last char is ASCII + | len2 >= 1 && w0 >= 0xC0 = len2 - 1 -- last char starts a code point + | len2 >= 2 && w1 >= 0xC0 = len2 - 2 -- pre-last char starts a code point + | len2 >= 3 && w2 >= 0xC0 = len2 - 3 + | len2 >= 4 && w3 >= 0xC0 = len2 - 4 + | otherwise = 0 + where + w0 = B.index bs2 (len2 - 1) + w1 = B.index bs2 (len2 - 2) + w2 = B.index bs2 (len2 - 3) + w3 = B.index bs2 (len2 - 4) + + decodeFrom :: Int -> DecoderResult + decodeFrom off = step (off + 1) (utf8DecodeStart (index off)) + where + step i (Incomplete a b) + | i < len = step (i + 1) (utf8DecodeContinue (index i) a b) + step _ st = st + + outer :: forall s. A.MArray s -> Int -> Int -> Int -> ST s (Text, ByteString) + outer dst dstLen = inner + where + inner srcOff dstOff + | srcOff >= len = do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + return (Text arr 0 dstOff, mempty) + + | srcOff >= len1 + , srcOff < len1 + guessUtf8Boundary + , dstOff + (len1 + guessUtf8Boundary - srcOff) <= dstLen + , bs <- B.drop (srcOff - len1) (B.take guessUtf8Boundary bs2) + , isValidBS bs = do + withBS bs $ \fp _ -> unsafeIOToST $ unsafeWithForeignPtr fp $ \src -> + unsafeSTToIO $ A.copyFromPointer dst dstOff src (len1 + guessUtf8Boundary - srcOff) + inner (len1 + guessUtf8Boundary) (dstOff + (len1 + guessUtf8Boundary - srcOff)) + + | dstOff + 4 > dstLen = do + let dstLen' = dstLen + 4 + dst' <- A.resizeM dst dstLen' + outer dst' dstLen' srcOff dstOff + + | otherwise = case decodeFrom srcOff of + Accept c -> do + d <- unsafeWrite dst dstOff c + inner (srcOff + d) (dstOff + d) + Reject -> case onErr desc (Just (index srcOff)) of + Nothing -> inner (srcOff + 1) dstOff + Just c -> do + d <- unsafeWrite dst dstOff (safe c) + inner (srcOff + 1) (dstOff + d) + Incomplete{} -> do + A.shrinkM dst dstOff + arr <- A.unsafeFreeze dst + let bs = if srcOff >= len1 + then B.drop (srcOff - len1) bs2 + else B.drop srcOff (bs1 `B.append` bs2) + return (Text arr 0 dstOff, bs) + + desc = "Data.Text.Internal.Encoding: Invalid UTF-8 stream" + -- $stream -- -- The 'streamDecodeUtf8' and 'streamDecodeUtf8With' functions accept @@ -264,7 +353,7 @@ -- | A stream oriented decoding result. -- -- @since 1.0.0.0 -data Decoding = Some Text ByteString (ByteString -> Decoding) +data Decoding = Some !Text !ByteString (ByteString -> Decoding) instance Show Decoding where showsPrec d (Some t bs _) = showParen (d > prec) $ @@ -273,9 +362,6 @@ showString " _" where prec = 10; prec' = prec + 1 -newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable) -newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable) - -- | Decode, in a stream oriented way, a 'ByteString' containing UTF-8 -- encoded text that is known to be valid. -- @@ -301,69 +387,11 @@ HasCallStack => #endif OnDecodeError -> ByteString -> Decoding -streamDecodeUtf8With onErr = decodeChunk B.empty 0 0 - where - -- We create a slightly larger than necessary buffer to accommodate a - -- potential surrogate pair started in the last buffer (@undecoded0@), or - -- replacement characters for each byte in @undecoded0@ if the - -- sequence turns out to be invalid. There can be up to three bytes there, - -- hence we allocate @len+3@ 16-bit words. - decodeChunk :: ByteString -> CodePoint -> DecoderState -> ByteString - -> Decoding - decodeChunk undecoded0 codepoint0 state0 bs = withBS bs aux where - aux fp len = runST $ (unsafeIOToST . decodeChunkToBuffer) =<< A.new (len+3) - where - decodeChunkToBuffer :: A.MArray s -> IO Decoding - decodeChunkToBuffer dest = unsafeWithForeignPtr fp $ \ptr -> - with (0::CSize) $ \destOffPtr -> - with codepoint0 $ \codepointPtr -> - with state0 $ \statePtr -> - with nullPtr $ \curPtrPtr -> - let end = ptr `plusPtr` len - loop curPtr = do - prevState <- peek statePtr - poke curPtrPtr curPtr - lastPtr <- c_decode_utf8_with_state (A.maBA dest) destOffPtr - curPtrPtr end codepointPtr statePtr - state <- peek statePtr - case state of - UTF8_REJECT -> do - -- We encountered an encoding error - poke statePtr 0 - let skipByte x = case onErr desc (Just x) of - Nothing -> return () - Just c -> do - destOff <- peek destOffPtr - w <- unsafeSTToIO $ - unsafeWrite dest (cSizeToInt destOff) (safe c) - poke destOffPtr (destOff + intToCSize w) - if ptr == lastPtr && prevState /= UTF8_ACCEPT then do - -- If we can't complete the sequence @undecoded0@ from - -- the previous chunk, we invalidate the bytes from - -- @undecoded0@ and retry decoding the current chunk from - -- the initial state. - traverse_ skipByte (B.unpack undecoded0 ) - loop lastPtr - else do - peek lastPtr >>= skipByte - loop (lastPtr `plusPtr` 1) - - _ -> do - -- We encountered the end of the buffer while decoding - n <- peek destOffPtr - codepoint <- peek codepointPtr - chunkText <- unsafeSTToIO $ do - arr <- A.unsafeFreeze dest - return $! text arr 0 (cSizeToInt n) - let left = lastPtr `minusPtr` ptr - !undecoded = case state of - UTF8_ACCEPT -> B.empty - _ | left == 0 && prevState /= UTF8_ACCEPT -> B.append undecoded0 bs - | otherwise -> B.drop left bs - return $ Some chunkText undecoded - (decodeChunk undecoded codepoint state) - in loop ptr - desc = "Data.Text.Internal.Encoding.streamDecodeUtf8With: Invalid UTF-8 stream" +streamDecodeUtf8With onErr = go mempty + where + go bs1 bs2 = Some txt undecoded (go undecoded) + where + (txt, undecoded) = decodeUtf8With2 onErr bs1 bs2 -- | Decode a 'ByteString' containing UTF-8 encoded text that is known -- to be valid. @@ -372,11 +400,13 @@ -- thrown that cannot be caught in pure code. For more control over -- the handling of invalid data, use 'decodeUtf8'' or -- 'decodeUtf8With'. +-- +-- This is a partial function: it checks that input is a well-formed +-- UTF-8 sequence and copies buffer or throws an error otherwise. +-- decodeUtf8 :: ByteString -> Text decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-} -{-# RULES "STREAM stream/decodeUtf8 fusion" [1] - forall bs. F.stream (decodeUtf8 bs) = E.streamUtf8 strictDecode bs #-} -- | Decode a 'ByteString' containing UTF-8 encoded text. -- @@ -390,12 +420,49 @@ decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode {-# INLINE decodeUtf8' #-} +-- | Decode a 'ByteString' containing UTF-8 encoded text. +-- +-- Any invalid input bytes will be replaced with the Unicode replacement +-- character U+FFFD. +decodeUtf8Lenient :: ByteString -> Text +decodeUtf8Lenient = decodeUtf8With lenientDecode + -- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. -- -- @since 1.1.0.0 encodeUtf8Builder :: Text -> B.Builder -encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8) +encodeUtf8Builder = + -- manual eta-expansion to ensure inlining works as expected + \txt -> B.builder (step txt) + where + step txt@(Text arr off len) !k br@(B.BufferRange op ope) + -- Ensure that the common case is not recursive and therefore yields + -- better code. + | op' <= ope = do + unsafeSTToIO $ A.copyToPointer arr off op len + k (B.BufferRange op' ope) + | otherwise = textCopyStep txt k br + where + op' = op `plusPtr` len +{-# INLINE encodeUtf8Builder #-} +textCopyStep :: Text -> B.BuildStep a -> B.BuildStep a +textCopyStep (Text arr off len) k = + go off (off + len) + where + go !ip !ipe (B.BufferRange op ope) + | inpRemaining <= outRemaining = do + unsafeSTToIO $ A.copyToPointer arr ip op inpRemaining + let !br = B.BufferRange (op `plusPtr` inpRemaining) ope + k br + | otherwise = do + unsafeSTToIO $ A.copyToPointer arr ip op outRemaining + let !ip' = ip + outRemaining + return $ B.bufferFull 1 ope (go ip' ipe) + where + outRemaining = ope `minusPtr` op + inpRemaining = ipe - ip + -- | Encode text using UTF-8 encoding and escape the ASCII characters using -- a 'BP.BoundedPrim'. -- @@ -425,56 +492,31 @@ -- is smaller than 8, as this will save on divisions. | otherwise = return $ B.bufferFull bound op0 (outerLoop i0) where - outRemaining = (ope `minusPtr` op0) `div` bound + outRemaining = (ope `minusPtr` op0) `quot` bound inpRemaining = iend - i0 goPartial !iendTmp = go i0 op0 where go !i !op - | i < iendTmp = case A.unsafeIndex arr i of - w | w <= 0x7F -> do - BP.runB be (word16ToWord8 w) op >>= go (i + 1) - | w <= 0x7FF -> do - poke8 @Word16 0 $ (w `shiftR` 6) + 0xC0 - poke8 @Word16 1 $ (w .&. 0x3f) + 0x80 - go (i + 1) (op `plusPtr` 2) - | 0xD800 <= w && w <= 0xDBFF -> do - let c = ord $ U16.chr2 w (A.unsafeIndex arr (i+1)) - poke8 @Int 0 $ (c `shiftR` 18) + 0xF0 - poke8 @Int 1 $ ((c `shiftR` 12) .&. 0x3F) + 0x80 - poke8 @Int 2 $ ((c `shiftR` 6) .&. 0x3F) + 0x80 - poke8 @Int 3 $ (c .&. 0x3F) + 0x80 - go (i + 2) (op `plusPtr` 4) - | otherwise -> do - poke8 @Word16 0 $ (w `shiftR` 12) + 0xE0 - poke8 @Word16 1 $ ((w `shiftR` 6) .&. 0x3F) + 0x80 - poke8 @Word16 2 $ (w .&. 0x3F) + 0x80 - go (i + 1) (op `plusPtr` 3) - | otherwise = - outerLoop i (B.BufferRange op ope) - where - -- Take care, a is either Word16 or Int above - poke8 :: Integral a => Int -> a -> IO () - poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8) + | i < iendTmp = do + let w = A.unsafeIndex arr i + if w < 0x80 + then BP.runB be w op >>= go (i + 1) + else poke op w >> go (i + 1) (op `plusPtr` 1) + | otherwise = outerLoop i (B.BufferRange op ope) -- | Encode text using UTF-8 encoding. encodeUtf8 :: Text -> ByteString encodeUtf8 (Text arr off len) | len == 0 = B.empty + -- It would be easier to use Data.ByteString.Short.fromShort and slice later, + -- but this is undesirable when len is significantly smaller than length arr. | otherwise = unsafeDupablePerformIO $ do - fp <- B.mallocByteString (len*3) -- see https://github.com/haskell/text/issues/194 for why len*3 is enough - unsafeWithForeignPtr fp $ \ptr -> - with ptr $ \destPtr -> do - c_encode_utf8 destPtr (A.aBA arr) (intToCSize off) (intToCSize len) - newDest <- peek destPtr - let utf8len = newDest `minusPtr` ptr - if utf8len >= len `shiftR` 1 - then return (mkBS fp utf8len) - else do - fp' <- B.mallocByteString utf8len - unsafeWithForeignPtr fp' $ \ptr' -> do - B.memcpy ptr' ptr utf8len - return (mkBS fp' utf8len) + marr@(A.MutableByteArray mba) <- unsafeSTToIO $ A.newPinned len + unsafeSTToIO $ A.copyI len marr 0 arr off + let fp = ForeignPtr (byteArrayContents# (unsafeCoerce# mba)) + (PlainPtr mba) + pure $ B.fromForeignPtr fp 0 len -- | Decode text from little endian UTF-16 encoding. decodeUtf16LEWith :: OnDecodeError -> ByteString -> Text @@ -555,23 +597,7 @@ cSizeToInt :: CSize -> Int cSizeToInt = fromIntegral -intToCSize :: Int -> CSize -intToCSize = fromIntegral - -word16ToWord8 :: Word16 -> Word8 -word16ToWord8 = fromIntegral - -foreign import ccall unsafe "_hs_text_decode_utf8" c_decode_utf8 - :: MutableByteArray# s -> Ptr CSize - -> Ptr Word8 -> Ptr Word8 -> IO (Ptr Word8) - -foreign import ccall unsafe "_hs_text_decode_utf8_state" c_decode_utf8_with_state - :: MutableByteArray# s -> Ptr CSize - -> Ptr (Ptr Word8) -> Ptr Word8 - -> Ptr CodePoint -> Ptr DecoderState -> IO (Ptr Word8) - -foreign import ccall unsafe "_hs_text_decode_latin1" c_decode_latin1 - :: MutableByteArray# s -> Ptr Word8 -> Ptr Word8 -> IO () - -foreign import ccall unsafe "_hs_text_encode_utf8" c_encode_utf8 - :: Ptr (Ptr Word8) -> ByteArray# -> CSize -> CSize -> IO () +#ifdef SIMDUTF +foreign import ccall unsafe "_hs_text_is_valid_utf8" c_is_valid_utf8 + :: Ptr Word8 -> CSize -> IO CInt +#endif diff --git a/src/Data/Text/Foreign.hs b/src/Data/Text/Foreign.hs --- a/src/Data/Text/Foreign.hs +++ b/src/Data/Text/Foreign.hs @@ -14,7 +14,7 @@ ( -- * Interoperability with native code -- $interop - I16 + I8 -- * Safe conversion functions , fromPtr , useAsPtr @@ -23,29 +23,25 @@ , peekCStringLen , withCStringLen -- * Unsafe conversion code - , lengthWord16 + , lengthWord8 , unsafeCopyToPtr -- * Low-level manipulation -- $lowlevel - , dropWord16 - , takeWord16 + , dropWord8 + , takeWord8 ) where -#if defined(ASSERTS) -import Control.Exception (assert) -#endif -import Control.Monad.ST.Unsafe (unsafeIOToST) +import Control.Monad.ST.Unsafe (unsafeSTToIO) import Data.ByteString.Unsafe (unsafePackCStringLen, unsafeUseAsCStringLen) import Data.Text.Encoding (decodeUtf8, encodeUtf8) import Data.Text.Internal (Text(..), empty) -import Data.Text.Internal.Functions (unsafeWithForeignPtr) -import Data.Text.Unsafe (lengthWord16) -import Data.Word (Word16) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) +import Data.Text.Unsafe (lengthWord8) +import Data.Word (Word8) import Foreign.C.String (CStringLen) import Foreign.ForeignPtr (ForeignPtr, mallocForeignPtrArray) import Foreign.Marshal.Alloc (allocaBytes) -import Foreign.Ptr (Ptr, castPtr, plusPtr) -import Foreign.Storable (peek, poke) +import Foreign.Ptr (Ptr, castPtr) import qualified Data.Text.Array as A -- $interop @@ -54,101 +50,95 @@ -- to have a fixed address in the Haskell heap. All communication with -- native code must thus occur by copying data back and forth. -- --- The 'Text' type's internal representation is UTF-16, using the --- platform's native endianness. This makes copied data suitable for --- use with native libraries that use a similar representation, such --- as ICU. To interoperate with native libraries that use different --- internal representations, such as UTF-8 or UTF-32, consider using +-- The 'Text' type's internal representation is UTF-8. +-- To interoperate with native libraries that use different +-- internal representations, such as UTF-16 or UTF-32, consider using -- the functions in the 'Data.Text.Encoding' module. --- | A type representing a number of UTF-16 code units. -newtype I16 = I16 Int +-- | A type representing a number of UTF-8 code units. +-- +-- @since 2.0 +newtype I8 = I8 Int deriving (Bounded, Enum, Eq, Integral, Num, Ord, Read, Real, Show) --- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word16' by copying the +-- | /O(n)/ Create a new 'Text' from a 'Ptr' 'Word8' by copying the -- contents of the array. -fromPtr :: Ptr Word16 -- ^ source array - -> I16 -- ^ length of source array (in 'Word16' units) +fromPtr :: Ptr Word8 -- ^ source array + -> I8 -- ^ length of source array (in 'Word8' units) -> IO Text -fromPtr _ (I16 0) = return empty -fromPtr ptr (I16 len) = -#if defined(ASSERTS) - assert (len > 0) $ -#endif - return $! Text arr 0 len - where - arr = A.run (A.new len >>= copy) - copy marr = loop ptr 0 - where - loop !p !i | i == len = return marr - | otherwise = do - A.unsafeWrite marr i =<< unsafeIOToST (peek p) - loop (p `plusPtr` 2) (i + 1) +fromPtr ptr (I8 len) = unsafeSTToIO $ do + dst <- A.new len + A.copyFromPointer dst 0 ptr len + arr <- A.unsafeFreeze dst + return $! Text arr 0 len -- $lowlevel -- --- Foreign functions that use UTF-16 internally may return indices in --- units of 'Word16' instead of characters. These functions may +-- Foreign functions that use UTF-8 internally may return indices in +-- units of 'Word8' instead of characters. These functions may -- safely be used with such indices, as they will adjust offsets if -- necessary to preserve the validity of a Unicode string. --- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word16' units in +-- | /O(1)/ Return the prefix of the 'Text' of @n@ 'Word8' units in -- length. -- --- If @n@ would cause the 'Text' to end inside a surrogate pair, the --- end of the prefix will be advanced by one additional 'Word16' unit +-- If @n@ would cause the 'Text' to end inside a code point, the +-- end of the prefix will be advanced by several additional 'Word8' units -- to maintain its validity. -takeWord16 :: I16 -> Text -> Text -takeWord16 (I16 n) t@(Text arr off len) - | n <= 0 = empty - | n >= len || m >= len = t - | otherwise = Text arr off m - where - m | w < 0xD800 || w > 0xDBFF = n - | otherwise = n+1 - w = A.unsafeIndex arr (off+n-1) +-- +-- @since 2.0 +takeWord8 :: I8 -> Text -> Text +takeWord8 = (fst .) . splitAtWord8 --- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word16' units +-- | /O(1)/ Return the suffix of the 'Text', with @n@ 'Word8' units -- dropped from its beginning. -- --- If @n@ would cause the 'Text' to begin inside a surrogate pair, the --- beginning of the suffix will be advanced by one additional 'Word16' +-- If @n@ would cause the 'Text' to begin inside a code point, the +-- beginning of the suffix will be advanced by several additional 'Word8' -- unit to maintain its validity. -dropWord16 :: I16 -> Text -> Text -dropWord16 (I16 n) t@(Text arr off len) - | n <= 0 = t - | n >= len || m >= len = empty - | otherwise = Text arr (off+m) (len-m) +-- +-- @since 2.0 +dropWord8 :: I8 -> Text -> Text +dropWord8 = (snd .) . splitAtWord8 + +splitAtWord8 :: I8 -> Text -> (Text, Text) +splitAtWord8 (I8 n) t@(Text arr off len) + | n <= 0 = (empty, t) + | n >= len || m >= len = (t, empty) + | otherwise = (Text arr off m, Text arr (off+m) (len-m)) where - m | w < 0xD800 || w > 0xDBFF = n - | otherwise = n+1 - w = A.unsafeIndex arr (off+n-1) + m | w0 < 0x80 = n -- last char is ASCII + | w0 >= 0xF0 = n+3 -- last char starts 4-byte sequence + | w0 >= 0xE0 = n+2 -- last char starts 3-byte sequence + | w0 >= 0xC0 = n+1 -- last char starts 2-byte sequence + | w1 >= 0xF0 = n+2 -- pre-last char starts 4-byte sequence + | w1 >= 0xE0 = n+1 -- pre-last char starts 3-byte sequence + | w1 >= 0xC0 = n -- pre-last char starts 2-byte sequence + | w2 >= 0xF0 = n+1 -- pre-pre-last char starts 4-byte sequence + | otherwise = n -- pre-pre-last char starts 3-byte sequence + w0 = A.unsafeIndex arr (off+n-1) + w1 = A.unsafeIndex arr (off+n-2) + w2 = A.unsafeIndex arr (off+n-3) -- | /O(n)/ Copy a 'Text' to an array. The array is assumed to be big -- enough to hold the contents of the entire 'Text'. -unsafeCopyToPtr :: Text -> Ptr Word16 -> IO () -unsafeCopyToPtr (Text arr off len) ptr = loop ptr off - where - end = off + len - loop !p !i | i == end = return () - | otherwise = do - poke p (A.unsafeIndex arr i) - loop (p `plusPtr` 2) (i + 1) +unsafeCopyToPtr :: Text -> Ptr Word8 -> IO () +unsafeCopyToPtr (Text arr off len) ptr = unsafeSTToIO $ A.copyToPointer arr off ptr len -- | /O(n)/ Perform an action on a temporary, mutable copy of a -- 'Text'. The copy is freed as soon as the action returns. -useAsPtr :: Text -> (Ptr Word16 -> I16 -> IO a) -> IO a +useAsPtr :: Text -> (Ptr Word8 -> I8 -> IO a) -> IO a useAsPtr t@(Text _arr _off len) action = - allocaBytes (len * 2) $ \buf -> do + allocaBytes len $ \buf -> do unsafeCopyToPtr t buf - action (castPtr buf) (I16 len) + action (castPtr buf) (I8 len) -- | /O(n)/ Make a mutable copy of a 'Text'. -asForeignPtr :: Text -> IO (ForeignPtr Word16, I16) +asForeignPtr :: Text -> IO (ForeignPtr Word8, I8) asForeignPtr t@(Text _arr _off len) = do fp <- mallocForeignPtrArray len unsafeWithForeignPtr fp $ unsafeCopyToPtr t - return (fp, I16 len) + return (fp, I8 len) -- | /O(n)/ Decode a C string with explicit length, which is assumed -- to have been encoded as UTF-8. If decoding fails, a diff --git a/src/Data/Text/IO.hs b/src/Data/Text/IO.hs --- a/src/Data/Text/IO.hs +++ b/src/Data/Text/IO.hs @@ -10,16 +10,17 @@ -- -- Efficient locale-sensitive support for text I\/O. -- --- Skip past the synopsis for some important notes on performance and --- portability across different versions of GHC. +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding (e.g. UTF-8), and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. module Data.Text.IO ( - -- * Performance - -- $performance - - -- * Locale support - -- $locale -- * File-at-a-time operations readFile , writeFile @@ -62,25 +63,19 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell) import System.IO.Error (isEOFError) --- $performance --- #performance# --- --- The functions in this module obey the runtime system's locale, --- character set encoding, and line ending conversion settings. --- --- If you know in advance that you will be working with data that has --- a specific encoding (e.g. UTF-8), and your application is highly --- performance sensitive, you may find that it is faster to perform --- I\/O with bytestrings and to encode and decode yourself than to use --- the functions in this module. --- --- Whether this will hold depends on the version of GHC you are using, --- the platform you are working on, the data you are working with, and --- the encodings you are using, so be sure to test for yourself. - -- | The 'readFile' function reads a file and returns the contents of -- the file as a string. The entire file is read strictly, as with -- 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Encoding.decodeUtf8' '.' 'Data.ByteString.readFile' +-- is a much faster and safer alternative. readFile :: FilePath -> IO Text readFile name = openFile name ReadMode >>= hGetContents @@ -89,7 +84,7 @@ writeFile :: FilePath -> Text -> IO () writeFile p = withFile p WriteMode . flip hPutStr --- | Write a string the end of a file. +-- | Write a string to the end of a file. appendFile :: FilePath -> Text -> IO () appendFile p = withFile p AppendMode . flip hPutStr @@ -312,37 +307,3 @@ -- | Write a string to 'stdout', followed by a newline. putStrLn :: Text -> IO () putStrLn = hPutStrLn stdout - --- $locale --- --- /Note/: The behaviour of functions in this module depends on the --- version of GHC you are using. --- --- Beginning with GHC 6.12, text I\/O is performed using the system or --- handle's current locale and line ending conventions. --- --- Under GHC 6.10 and earlier, the system I\/O libraries do not --- support locale-sensitive I\/O or line ending conversion. On these --- versions of GHC, functions in this library all use UTF-8. What --- does this mean in practice? --- --- * All data that is read will be decoded as UTF-8. --- --- * Before data is written, it is first encoded as UTF-8. --- --- * On both reading and writing, the platform's native newline --- conversion is performed. --- --- If you must use a non-UTF-8 locale on an older version of GHC, you --- will have to perform the transcoding yourself, e.g. as follows: --- --- > import qualified Data.ByteString as B --- > import Data.Text (Text) --- > import Data.Text.Encoding (encodeUtf16) --- > --- > putStr_Utf16LE :: Text -> IO () --- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t) --- --- On transcoding errors, an 'IOError' exception is thrown. You can --- use the API in "Data.Text.Encoding" if you need more control over --- error handling or transcoding. diff --git a/src/Data/Text/Internal.hs b/src/Data/Text/Internal.hs --- a/src/Data/Text/Internal.hs +++ b/src/Data/Text/Internal.hs @@ -33,6 +33,7 @@ -- * Code that must be here for accessibility , empty , empty_ + , append -- * Utilities , firstf -- * Checked multiplication @@ -47,6 +48,7 @@ import Control.Exception (assert) import GHC.Stack (HasCallStack) #endif +import Control.Monad.ST (ST) import Data.Bits import Data.Int (Int32, Int64) import Data.Text.Internal.Unsafe.Char (ord) @@ -55,9 +57,9 @@ -- | A space efficient, packed, unboxed Unicode text type. data Text = Text - {-# UNPACK #-} !A.Array -- payload (Word16 elements) - {-# UNPACK #-} !Int -- offset (units of Word16, not Char) - {-# UNPACK #-} !Int -- length (units of Word16, not Char) + {-# UNPACK #-} !A.Array -- bytearray encoded as UTF-8 + {-# UNPACK #-} !Int -- offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + {-# UNPACK #-} !Int -- length in bytes (not in Char!), pointing to an end of UTF-8 sequence deriving (Typeable) -- | Smart constructor. @@ -65,13 +67,16 @@ #if defined(ASSERTS) HasCallStack => #endif - A.Array -> Int -> Int -> Text + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text text_ arr off len = #if defined(ASSERTS) let c = A.unsafeIndex arr off in assert (len >= 0) . assert (off >= 0) . - assert (len == 0 || c < 0xDC00 || c > 0xDFFF) $ + assert (len == 0 || c < 0x80 || c >= 0xC0) $ #endif Text arr off len {-# INLINE text_ #-} @@ -86,13 +91,34 @@ empty_ = Text A.empty 0 0 {-# NOINLINE empty_ #-} +-- | /O(n)/ Appends one 'Text' to the other by copying both of them +-- into a new 'Text'. +append :: Text -> Text -> Text +append a@(Text arr1 off1 len1) b@(Text arr2 off2 len2) + | len1 == 0 = b + | len2 == 0 = a + | len > 0 = Text (A.run x) 0 len + | otherwise = error $ "Data.Text.append: size overflow" + where + len = len1+len2 + x :: ST s (A.MArray s) + x = do + arr <- A.new len + A.copyI len1 arr 0 arr1 off1 + A.copyI len2 arr len1 arr2 off2 + return arr +{-# NOINLINE append #-} + -- | Construct a 'Text' without invisibly pinning its byte array in -- memory if its length has dwindled to zero. text :: #if defined(ASSERTS) HasCallStack => #endif - A.Array -> Int -> Int -> Text + A.Array -- ^ bytearray encoded as UTF-8 + -> Int -- ^ offset in bytes (not in Char!), pointing to a start of UTF-8 sequence + -> Int -- ^ length in bytes (not in Char!), pointing to an end of UTF-8 sequence + -> Text text arr off len | len == 0 = empty | otherwise = text_ arr off len {-# INLINE text #-} @@ -109,7 +135,7 @@ -- | Map a 'Char' to a 'Text'-safe value. -- --- UTF-16 surrogate code points are not included in the set of Unicode +-- Unicode 'Data.Char.Surrogate' code points are not included in the set of Unicode -- scalar values, but are unfortunately admitted as valid 'Char' -- values by Haskell. They cannot be represented in a 'Text'. This -- function remaps those code points to the Unicode replacement @@ -191,19 +217,17 @@ -- $internals -- --- Internally, the 'Text' type is represented as an array of 'Word16' --- UTF-16 code units. The offset and length fields in the constructor +-- Internally, the 'Text' type is represented as an array of 'Word8' +-- UTF-8 code units. The offset and length fields in the constructor -- are in these units, /not/ units of 'Char'. -- -- Invariants that all functions must maintain: -- --- * Since the 'Text' type uses UTF-16 internally, it cannot represent +-- * Since the 'Text' type uses UTF-8 internally, it cannot represent -- characters in the reserved surrogate code point range U+D800 to -- U+DFFF. To maintain this invariant, the 'safe' function maps -- 'Char' values in this range to the replacement character (U+FFFD, -- \'�\'). -- --- * A leading (or \"high\") surrogate code unit (0xD800–0xDBFF) must --- always be followed by a trailing (or \"low\") surrogate code unit --- (0xDC00-0xDFFF). A trailing surrogate code unit must always be --- preceded by a leading surrogate code unit. +-- * Offset and length must point to a valid UTF-8 sequence of bytes. +-- Violation of this may cause memory access violation and divergence. diff --git a/src/Data/Text/Internal/Builder.hs b/src/Data/Text/Internal/Builder.hs --- a/src/Data/Text/Internal/Builder.hs +++ b/src/Data/Text/Internal/Builder.hs @@ -140,7 +140,7 @@ HasCallStack => #endif Char -> Builder -singleton c = writeAtMost 2 $ \ marr o -> unsafeWrite marr o (safe c) +singleton c = writeAtMost 4 $ \ marr o -> unsafeWrite marr o (safe c) {-# INLINE singleton #-} ------------------------------------------------------------------------ @@ -168,7 +168,7 @@ fromText :: S.Text -> Builder fromText t@(Text arr off l) | S.null t = empty - | l <= copyLimit = writeN l $ \marr o -> A.copyI marr o arr off (l+o) + | l <= copyLimit = writeN l $ \marr o -> A.copyI l marr o arr off | otherwise = flush `append` mapBuilder (t :) {-# INLINE [1] fromText #-} @@ -185,7 +185,8 @@ fromString str = Builder $ \k (Buffer p0 o0 u0 l0) -> let loop !marr !o !u !l [] = k (Buffer marr o u l) loop marr o u l s@(c:cs) - | l <= 1 = do + | l <= 3 = do + A.shrinkM marr (o + u) arr <- A.unsafeFreeze marr let !t = Text arr o u marr' <- A.new chunkSize diff --git a/src/Data/Text/Internal/Encoding/Fusion.hs b/src/Data/Text/Internal/Encoding/Fusion.hs --- a/src/Data/Text/Internal/Encoding/Fusion.hs +++ b/src/Data/Text/Internal/Encoding/Fusion.hs @@ -37,14 +37,14 @@ #if defined(ASSERTS) import Control.Exception (assert) #endif +import Data.Bits (shiftL, shiftR) import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy) import Data.Text.Internal.Fusion (Step(..), Stream(..)) import Data.Text.Internal.Fusion.Size import Data.Text.Encoding.Error import Data.Text.Internal.Encoding.Fusion.Common -import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) -import Data.Text.Internal.Functions (unsafeWithForeignPtr) +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) import Data.Word (Word8, Word16, Word32) import Foreign.ForeignPtr (ForeignPtr) import Foreign.Storable (pokeByteOff) @@ -99,7 +99,7 @@ {-# INLINE next #-} next i | i >= l = Done - | i+1 < l && U16.validate1 x1 = Yield (unsafeChr x1) (i+2) + | i+1 < l && U16.validate1 x1 = Yield (unsafeChr16 x1) (i+2) | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) | otherwise = decodeError "streamUtf16LE" "UTF-16LE" onErr Nothing (i+1) where @@ -117,7 +117,7 @@ {-# INLINE next #-} next i | i >= l = Done - | i+1 < l && U16.validate1 x1 = Yield (unsafeChr x1) (i+2) + | i+1 < l && U16.validate1 x1 = Yield (unsafeChr16 x1) (i+2) | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4) | otherwise = decodeError "streamUtf16BE" "UTF-16BE" onErr Nothing (i+1) where diff --git a/src/Data/Text/Internal/Encoding/Fusion/Common.hs b/src/Data/Text/Internal/Encoding/Fusion/Common.hs --- a/src/Data/Text/Internal/Encoding/Fusion/Common.hs +++ b/src/Data/Text/Internal/Encoding/Fusion/Common.hs @@ -29,11 +29,10 @@ , restreamUtf32BE ) where -import Data.Bits ((.&.)) +import Data.Bits ((.&.), shiftR) import Data.Text.Internal.Fusion (Step(..), Stream(..)) import Data.Text.Internal.Fusion.Types (RS(..)) import Data.Text.Internal.Unsafe.Char (ord) -import Data.Text.Internal.Unsafe.Shift (shiftR) import Data.Word (Word8) restreamUtf16BE :: Stream Char -> Stream Word8 diff --git a/src/Data/Text/Internal/Encoding/Utf8.hs b/src/Data/Text/Internal/Encoding/Utf8.hs --- a/src/Data/Text/Internal/Encoding/Utf8.hs +++ b/src/Data/Text/Internal/Encoding/Utf8.hs @@ -5,6 +5,7 @@ -- Copyright : (c) 2008, 2009 Tom Harper, -- (c) 2009, 2010 Bryan O'Sullivan, -- (c) 2009 Duncan Coutts +-- (c) 2021 Andrew Lelechenko -- -- License : BSD-style -- Maintainer : bos@serpentine.com @@ -17,9 +18,10 @@ -- -- Basic UTF-8 validation and character manipulation. module Data.Text.Internal.Encoding.Utf8 - ( + ( utf8Length + , utf8LengthByLeader -- Decomposition - ord2 + , ord2 , ord3 , ord4 -- Construction @@ -31,11 +33,20 @@ , validate2 , validate3 , validate4 + -- * Naive decoding + , DecoderResult(..) + , DecoderState(..) + , CodePoint(..) + , utf8DecodeStart + , utf8DecodeContinue ) where -import Data.Bits ((.&.)) -import Data.Text.Internal.Unsafe.Char (ord) -import Data.Text.Internal.Unsafe.Shift (shiftR) +#if defined(ASSERTS) +import Control.Exception (assert) +import GHC.Stack (HasCallStack) +#endif +import Data.Bits (Bits(..), FiniteBits(..)) +import Data.Char (ord, chr) import GHC.Exts import GHC.Word (Word8(..)) @@ -53,34 +64,79 @@ between x y z = x >= y && x <= z {-# INLINE between #-} -ord2 :: Char -> (Word8,Word8) +-- This is a branchless version of +-- utf8Length c +-- | ord c < 0x80 = 1 +-- | ord c < 0x800 = 2 +-- | ord c < 0x10000 = 3 +-- | otherwise = 4 +-- Implementation suggested by Alex Mason. + +-- | @since 2.0 +utf8Length :: Char -> Int +utf8Length (C# c) = I# ((1# +# geChar# c (chr# 0x80#)) +# (geChar# c (chr# 0x800#) +# geChar# c (chr# 0x10000#))) +{-# INLINE utf8Length #-} + +-- This is a branchless version of +-- utf8LengthByLeader w +-- | w < 0x80 = 1 +-- | w < 0xE0 = 2 +-- | w < 0xF0 = 3 +-- | otherwise = 4 +-- +-- c `xor` I# (c# <=# 0#) is a branchless equivalent of c `max` 1. +-- It is crucial to write c# <=# 0# and not c# ==# 0#, otherwise +-- GHC is tempted to "optimize" by introduction of branches. + +-- | @since 2.0 +utf8LengthByLeader :: Word8 -> Int +utf8LengthByLeader w = c `xor` I# (c# <=# 0#) + where + !c@(I# c#) = countLeadingZeros (complement w) +{-# INLINE utf8LengthByLeader #-} + +ord2 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8) ord2 c = - -- ord2 is used only in test suite to construct a deliberately invalid ByteString, - -- actually violating the assertion, so it is commented out - -- assert (n >= 0x80 && n <= 0x07ff) +#if defined(ASSERTS) + assert (n >= 0x80 && n <= 0x07ff) +#endif (x1,x2) where n = ord c x1 = intToWord8 $ (n `shiftR` 6) + 0xC0 x2 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord2 #-} -ord3 :: Char -> (Word8,Word8,Word8) +ord3 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8) ord3 c = - -- ord3 is used only in test suite to construct a deliberately invalid ByteString, - -- actually violating the assertion, so it is commented out - -- assert (n >= 0x0800 && n <= 0xffff) +#if defined(ASSERTS) + assert (n >= 0x0800 && n <= 0xffff) +#endif (x1,x2,x3) where n = ord c x1 = intToWord8 $ (n `shiftR` 12) + 0xE0 x2 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 x3 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord3 #-} -ord4 :: Char -> (Word8,Word8,Word8,Word8) +ord4 :: +#if defined(ASSERTS) + HasCallStack => +#endif + Char -> (Word8,Word8,Word8,Word8) ord4 c = - -- ord4 is used only in test suite to construct a deliberately invalid ByteString, - -- actually violating the assertion, so it is commented out - -- assert (n >= 0x10000) +#if defined(ASSERTS) + assert (n >= 0x10000) +#endif (x1,x2,x3,x4) where n = ord c @@ -88,6 +144,7 @@ x2 = intToWord8 $ ((n `shiftR` 12) .&. 0x3F) + 0x80 x3 = intToWord8 $ ((n `shiftR` 6) .&. 0x3F) + 0x80 x4 = intToWord8 $ (n .&. 0x3F) + 0x80 +{-# INLINE ord4 #-} chr2 :: Word8 -> Word8 -> Char chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#)) @@ -109,7 +166,7 @@ !z3# = y3# -# 0x80# {-# INLINE chr3 #-} -chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char +chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) = C# (chr# (z1# +# z2# +# z3# +# z4#)) where @@ -167,3 +224,69 @@ intToWord8 :: Int -> Word8 intToWord8 = fromIntegral + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +------------------------------------------------------------------------------- +-- Naive UTF8 decoder. +-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for the explanation of the state machine. + +newtype ByteClass = ByteClass Word8 + +byteToClass :: Word8 -> ByteClass +byteToClass n = ByteClass (W8# el#) + where + !(I# n#) = word8ToInt n + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\SOH\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\a\b\b\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\STX\n\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\ETX\EOT\ETX\ETX\v\ACK\ACK\ACK\ENQ\b\b\b\b\b\b\b\b\b\b\b"# + +newtype DecoderState = DecoderState Word8 + deriving (Eq) + +utf8AcceptState :: DecoderState +utf8AcceptState = DecoderState 0 + +utf8RejectState :: DecoderState +utf8RejectState = DecoderState 12 + +updateState :: ByteClass -> DecoderState -> DecoderState +updateState (ByteClass c) (DecoderState s) = DecoderState (W8# el#) + where + !(I# n#) = word8ToInt (c + s) + el# = indexWord8OffAddr# table# n# + + table# :: Addr# + table# = "\NUL\f\CAN$<`T\f\f\f0H\f\f\f\f\f\f\f\f\f\f\f\f\f\NUL\f\f\f\f\f\NUL\f\NUL\f\f\f\CAN\f\f\f\f\f\CAN\f\CAN\f\f\f\f\f\f\f\f\f\CAN\f\f\f\f\f\CAN\f\f\f\f\f\f\f\CAN\f\f\f\f\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f$\f$\f\f\f$\f\f\f\f\f\f\f\f\f\f"# + +newtype CodePoint = CodePoint Int + +-- | @since 2.0 +data DecoderResult + = Accept !Char + | Incomplete !DecoderState !CodePoint + | Reject + +-- | @since 2.0 +utf8DecodeStart :: Word8 -> DecoderResult +utf8DecodeStart !w + | st == utf8AcceptState = Accept (chr (word8ToInt w)) + | st == utf8RejectState = Reject + | otherwise = Incomplete st (CodePoint cp) + where + cl@(ByteClass cl') = byteToClass w + st = updateState cl utf8AcceptState + cp = word8ToInt $ (0xff `unsafeShiftR` word8ToInt cl') .&. w + +-- | @since 2.0 +utf8DecodeContinue :: Word8 -> DecoderState -> CodePoint -> DecoderResult +utf8DecodeContinue !w !st (CodePoint !cp) + | st' == utf8AcceptState = Accept (chr cp') + | st' == utf8RejectState = Reject + | otherwise = Incomplete st' (CodePoint cp') + where + cl = byteToClass w + st' = updateState cl st + cp' = (cp `shiftL` 6) .|. word8ToInt (w .&. 0x3f) diff --git a/src/Data/Text/Internal/Functions.hs b/src/Data/Text/Internal/Functions.hs deleted file mode 100644 --- a/src/Data/Text/Internal/Functions.hs +++ /dev/null @@ -1,47 +0,0 @@ -{-# LANGUAGE CPP #-} - --- | --- Module : Data.Text.Internal.Functions --- Copyright : 2010 Bryan O'Sullivan --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Useful functions. - -module Data.Text.Internal.Functions - ( - intersperse, - unsafeWithForeignPtr - ) where - -import Foreign.Ptr (Ptr) -import Foreign.ForeignPtr (ForeignPtr) -#if MIN_VERSION_base(4,15,0) -import qualified GHC.ForeignPtr (unsafeWithForeignPtr) -#else -import qualified Foreign.ForeignPtr (withForeignPtr) -#endif - --- | A lazier version of Data.List.intersperse. The other version --- causes space leaks! -intersperse :: a -> [a] -> [a] -intersperse _ [] = [] -intersperse sep (x:xs) = x : go xs - where - go [] = [] - go (y:ys) = sep : y: go ys -{-# INLINE intersperse #-} - -unsafeWithForeignPtr :: ForeignPtr a -> (Ptr a -> IO b) -> IO b -#if MIN_VERSION_base(4,15,0) -unsafeWithForeignPtr = GHC.ForeignPtr.unsafeWithForeignPtr -#else -unsafeWithForeignPtr = Foreign.ForeignPtr.withForeignPtr -#endif diff --git a/src/Data/Text/Internal/Fusion.hs b/src/Data/Text/Internal/Fusion.hs --- a/src/Data/Text/Internal/Fusion.hs +++ b/src/Data/Text/Internal/Fusion.hs @@ -50,68 +50,91 @@ ) where import Prelude (Bool(..), Char, Maybe(..), Monad(..), Int, - Num(..), Ord(..), ($), (&&), - fromIntegral, otherwise) -import Data.Bits ((.&.)) + Num(..), Ord(..), ($), + otherwise) +import Data.Bits (shiftL, shiftR) import Data.Text.Internal (Text(..)) import Data.Text.Internal.Private (runText) -import Data.Text.Internal.Unsafe.Char (ord, unsafeChr, unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftL, shiftR) +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeWrite) import qualified Data.Text.Array as A import qualified Data.Text.Internal.Fusion.Common as S import Data.Text.Internal.Fusion.Types import Data.Text.Internal.Fusion.Size import qualified Data.Text.Internal as I -import qualified Data.Text.Internal.Encoding.Utf16 as U16 -import Data.Word (Word16) - -#if defined(ASSERTS) +import qualified Data.Text.Internal.Encoding.Utf8 as U8 import GHC.Stack (HasCallStack) -#endif default(Int) --- | /O(n)/ Convert a 'Text' into a 'Stream Char'. +-- | /O(n)/ Convert 'Text' into a 'Stream' 'Char'. +-- +-- __Properties__ +-- +-- @'unstream' . 'stream' = 'Data.Function.id'@ +-- +-- @'stream' . 'unstream' = 'Data.Function.id' @ stream :: #if defined(ASSERTS) HasCallStack => #endif Text -> Stream Char -stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 1) len) +stream (Text arr off len) = Stream next off (betweenSize (len `shiftR` 2) len) where !end = off+len next !i - | i >= end = Done - | n >= 0xD800 && n <= 0xDBFF = Yield (U16.chr2 n n2) (i + 2) - | otherwise = Yield (unsafeChr n) (i + 1) + | i >= end = Done + | otherwise = Yield chr (i + l) where - n = A.unsafeIndex arr i - n2 = A.unsafeIndex arr (i + 1) + n0 = A.unsafeIndex arr i + n1 = A.unsafeIndex arr (i + 1) + n2 = A.unsafeIndex arr (i + 2) + n3 = A.unsafeIndex arr (i + 3) + + l = U8.utf8LengthByLeader n0 + chr = case l of + 1 -> unsafeChr8 n0 + 2 -> U8.chr2 n0 n1 + 3 -> U8.chr3 n0 n1 n2 + _ -> U8.chr4 n0 n1 n2 n3 {-# INLINE [0] stream #-} --- | /O(n)/ Convert a 'Text' into a 'Stream Char', but iterate --- backwards. +-- | /O(n)/ Converts 'Text' into a 'Stream' 'Char', but iterates +-- backwards through the text. +-- +-- __Properties__ +-- +-- @'unstream' . 'reverseStream' = 'Data.Text.reverse' @ reverseStream :: Text -> Stream Char -reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 1) len) +reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `shiftR` 2) len) where {-# INLINE next #-} next !i - | i < off = Done - | n >= 0xDC00 && n <= 0xDFFF = Yield (U16.chr2 n2 n) (i - 2) - | otherwise = Yield (unsafeChr n) (i - 1) + | i < off = Done + | n0 < 0x80 = Yield (unsafeChr8 n0) (i - 1) + | n1 >= 0xC0 = Yield (U8.chr2 n1 n0) (i - 2) + | n2 >= 0xC0 = Yield (U8.chr3 n2 n1 n0) (i - 3) + | otherwise = Yield (U8.chr4 n3 n2 n1 n0) (i - 4) where - n = A.unsafeIndex arr i - n2 = A.unsafeIndex arr (i - 1) + n0 = A.unsafeIndex arr i + n1 = A.unsafeIndex arr (i - 1) + n2 = A.unsafeIndex arr (i - 2) + n3 = A.unsafeIndex arr (i - 3) {-# INLINE [0] reverseStream #-} --- | /O(n)/ Convert a 'Stream Char' into a 'Text'. +-- | /O(n)/ Convert 'Stream' 'Char' into a 'Text'. +-- +-- __Properties__ +-- +-- @'unstream' . 'stream' = 'Data.Function.id'@ +-- +-- @'stream' . 'unstream' = 'Data.Function.id' @ unstream :: Stream Char -> Text unstream (Stream next0 s0 len) = runText $ \done -> do -- Before encoding each char we perform a buffer realloc check assuming - -- worst case encoding size of two 16-bit units for the char. Just add an + -- worst case encoding size of four 8-bit units for the char. Just add an -- extra space to the buffer so that we do not end up reallocating even when -- all the chars are encoded as single unit. - let mlen = upperBound 4 len + 1 + let mlen = upperBound 4 len + 3 arr0 <- A.new mlen let outer !arr !maxi = encode where @@ -122,7 +145,7 @@ Skip si' -> encode si' di Yield c si' -- simply check for the worst case - | maxi < di + 1 -> realloc si di + | maxi < di + 3 -> realloc si di | otherwise -> do n <- unsafeWrite arr di c encode si' (di + n) @@ -131,8 +154,7 @@ {-# NOINLINE realloc #-} realloc !si !di = do let newlen = (maxi + 1) * 2 - arr' <- A.new newlen - A.copyM arr' 0 arr 0 di + arr' <- A.resizeM arr newlen outer arr' (newlen - 1) si di outer arr0 (mlen - 1) s0 0 @@ -143,11 +165,20 @@ -- ---------------------------------------------------------------------------- -- * Basic stream functions +-- | /O(n)/ Returns the number of characters in a 'Stream'. +-- +-- __Properties__ +-- +-- @'length' . 'stream' = 'Data.Text.length' @ length :: Stream Char -> Int length = S.lengthI {-# INLINE[0] length #-} --- | /O(n)/ Reverse the characters of a string. +-- | /O(n)/ Reverse the characters of a 'Stream' returning 'Text'. +-- +-- __Properties__ +-- +-- @'reverse' . 'stream' = 'Data.Text.reverse' @ reverse :: #if defined(ASSERTS) HasCallStack => @@ -168,26 +199,20 @@ let newLen = len `shiftL` 1 marr' <- A.new newLen A.copyM marr' (newLen-len) marr 0 len - write s1 (len+i) newLen marr' - | otherwise -> write s1 i len marr - where n = ord x - least | n < 0x10000 = 0 - | otherwise = 1 - m = n - 0x10000 - lo = intToWord16 $ (m `shiftR` 10) + 0xD800 - hi = intToWord16 $ (m .&. 0x3FF) + 0xDC00 - write t j l mar - | n < 0x10000 = do - A.unsafeWrite mar j (intToWord16 n) - loop t (j-1) l mar - | otherwise = do - A.unsafeWrite mar (j-1) lo - A.unsafeWrite mar j hi - loop t (j-2) l mar + _ <- unsafeWrite marr' (len + i - least) x + loop s1 (len + i - least - 1) newLen marr' + | otherwise -> do + _ <- unsafeWrite marr (i - least) x + loop s1 (i - least - 1) len marr + where least = U8.utf8Length x - 1 {-# INLINE [0] reverse #-} -- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with -- the input and result reversed. +-- +-- __Properties__ +-- +-- @'reverse' . 'reverseScanr' f c . 'reverseStream' = 'Data.Text.scanr' f c @ reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low where @@ -204,6 +229,10 @@ -- value. However, the length of the result is limited by the -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the length of the result is known. +-- +-- __Properties__ +-- +-- @'unstream' ('unfoldrN' n f a) = 'Data.Text.unfoldrN' n f a @ unfoldrN :: Int -> (a -> Maybe (Char,a)) -> a -> Stream Char unfoldrN n = S.unfoldrNI n {-# INLINE [0] unfoldrN #-} @@ -212,19 +241,31 @@ -- ** Indexing streams -- | /O(n)/ stream index (subscript) operator, starting from 0. -index :: Stream Char -> Int -> Char +-- +-- __Properties__ +-- +-- @'index' ('stream' t) n = 'Data.Text.index' t n @ +index :: HasCallStack => Stream Char -> Int -> Char index = S.indexI {-# INLINE [0] index #-} -- | The 'findIndex' function takes a predicate and a stream and -- returns the index of the first element in the stream -- satisfying the predicate. +-- +-- __Properties__ +-- +-- @'findIndex' p . 'stream' = 'Data.Text.findIndex' p @ findIndex :: (Char -> Bool) -> Stream Char -> Maybe Int findIndex = S.findIndexI {-# INLINE [0] findIndex #-} -- | /O(n)/ The 'count' function returns the number of times the query -- element appears in the given stream. +-- +-- __Properties__ +-- +-- @'countChar' c . 'stream' = 'Data.Text.countChar' c @ countChar :: Char -> Stream Char -> Int countChar = S.countCharI {-# INLINE [0] countChar #-} @@ -232,6 +273,10 @@ -- | /O(n)/ Like a combination of 'map' and 'foldl''. Applies a -- function to each element of a 'Text', passing an accumulating -- parameter from left to right, and returns a final 'Text'. +-- +-- __Properties__ +-- +-- @'mapAccumL' g z0 . 'stream' = 'Data.Text.mapAccumL' g z0@ mapAccumL :: #if defined(ASSERTS) HasCallStack => @@ -250,15 +295,10 @@ Yield x s' | j >= top -> {-# SCC "mapAccumL/resize" #-} do let top' = (top + 1) `shiftL` 1 - arr' <- A.new top' - A.copyM arr' 0 arr 0 top + arr' <- A.resizeM arr top' outer arr' top' z s i | otherwise -> do d <- unsafeWrite arr i c loop z' s' (i+d) where (z',c) = f z x - j | ord c < 0x10000 = i - | otherwise = i + 1 + j = i + U8.utf8Length c - 1 {-# INLINE [0] mapAccumL #-} - -intToWord16 :: Int -> Word16 -intToWord16 = fromIntegral diff --git a/src/Data/Text/Internal/Fusion/CaseMapping.hs b/src/Data/Text/Internal/Fusion/CaseMapping.hs --- a/src/Data/Text/Internal/Fusion/CaseMapping.hs +++ b/src/Data/Text/Internal/Fusion/CaseMapping.hs @@ -1,756 +1,6420 @@ -{-# LANGUAGE Rank2Types #-} --- AUTOMATICALLY GENERATED - DO NOT EDIT --- Generated by scripts/CaseMapping.hs --- CaseFolding-13.0.0.txt --- Date: 2019-09-08, 23:30:59 GMT --- SpecialCasing-13.0.0.txt --- Date: 2019-09-08, 23:31:24 GMT - -module Data.Text.Internal.Fusion.CaseMapping where -import Data.Char -import Data.Text.Internal.Fusion.Types - -upperMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE upperMapping #-} --- LATIN SMALL LETTER SHARP S -upperMapping '\x00df' s = Yield '\x0053' (CC s '\x0053' '\x0000') --- LATIN SMALL LIGATURE FF -upperMapping '\xfb00' s = Yield '\x0046' (CC s '\x0046' '\x0000') --- LATIN SMALL LIGATURE FI -upperMapping '\xfb01' s = Yield '\x0046' (CC s '\x0049' '\x0000') --- LATIN SMALL LIGATURE FL -upperMapping '\xfb02' s = Yield '\x0046' (CC s '\x004c' '\x0000') --- LATIN SMALL LIGATURE FFI -upperMapping '\xfb03' s = Yield '\x0046' (CC s '\x0046' '\x0049') --- LATIN SMALL LIGATURE FFL -upperMapping '\xfb04' s = Yield '\x0046' (CC s '\x0046' '\x004c') --- LATIN SMALL LIGATURE LONG S T -upperMapping '\xfb05' s = Yield '\x0053' (CC s '\x0054' '\x0000') --- LATIN SMALL LIGATURE ST -upperMapping '\xfb06' s = Yield '\x0053' (CC s '\x0054' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -upperMapping '\x0587' s = Yield '\x0535' (CC s '\x0552' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -upperMapping '\xfb13' s = Yield '\x0544' (CC s '\x0546' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -upperMapping '\xfb14' s = Yield '\x0544' (CC s '\x0535' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -upperMapping '\xfb15' s = Yield '\x0544' (CC s '\x053b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -upperMapping '\xfb16' s = Yield '\x054e' (CC s '\x0546' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -upperMapping '\xfb17' s = Yield '\x0544' (CC s '\x053d' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -upperMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -upperMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -upperMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- LATIN SMALL LETTER J WITH CARON -upperMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -upperMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -upperMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -upperMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -upperMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -upperMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -upperMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -upperMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -upperMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -upperMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -upperMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -upperMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -upperMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -upperMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -upperMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -upperMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -upperMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -upperMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -upperMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -upperMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -upperMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f80' s = Yield '\x1f08' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f81' s = Yield '\x1f09' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f82' s = Yield '\x1f0a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f83' s = Yield '\x1f0b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f84' s = Yield '\x1f0c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f85' s = Yield '\x1f0d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f86' s = Yield '\x1f0e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f87' s = Yield '\x1f0f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f88' s = Yield '\x1f08' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f89' s = Yield '\x1f09' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8a' s = Yield '\x1f0a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f8b' s = Yield '\x1f0b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8c' s = Yield '\x1f0c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f8d' s = Yield '\x1f0d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8e' s = Yield '\x1f0e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f8f' s = Yield '\x1f0f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1f90' s = Yield '\x1f28' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1f91' s = Yield '\x1f29' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f92' s = Yield '\x1f2a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1f93' s = Yield '\x1f2b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f94' s = Yield '\x1f2c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1f95' s = Yield '\x1f2d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f96' s = Yield '\x1f2e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1f97' s = Yield '\x1f2f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1f98' s = Yield '\x1f28' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1f99' s = Yield '\x1f29' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9a' s = Yield '\x1f2a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1f9b' s = Yield '\x1f2b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9c' s = Yield '\x1f2c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1f9d' s = Yield '\x1f2d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9e' s = Yield '\x1f2e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1f9f' s = Yield '\x1f2f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -upperMapping '\x1fa0' s = Yield '\x1f68' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -upperMapping '\x1fa1' s = Yield '\x1f69' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa2' s = Yield '\x1f6a' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -upperMapping '\x1fa3' s = Yield '\x1f6b' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa4' s = Yield '\x1f6c' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -upperMapping '\x1fa5' s = Yield '\x1f6d' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa6' s = Yield '\x1f6e' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fa7' s = Yield '\x1f6f' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -upperMapping '\x1fa8' s = Yield '\x1f68' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -upperMapping '\x1fa9' s = Yield '\x1f69' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1faa' s = Yield '\x1f6a' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -upperMapping '\x1fab' s = Yield '\x1f6b' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fac' s = Yield '\x1f6c' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -upperMapping '\x1fad' s = Yield '\x1f6d' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1fae' s = Yield '\x1f6e' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -upperMapping '\x1faf' s = Yield '\x1f6f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -upperMapping '\x1fb3' s = Yield '\x0391' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -upperMapping '\x1fbc' s = Yield '\x0391' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -upperMapping '\x1fc3' s = Yield '\x0397' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -upperMapping '\x1fcc' s = Yield '\x0397' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -upperMapping '\x1ff3' s = Yield '\x03a9' (CC s '\x0399' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -upperMapping '\x1ffc' s = Yield '\x03a9' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -upperMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -upperMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0399' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0399') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0399') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -upperMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0399') -upperMapping c s = Yield (toUpper c) (CC s '\0' '\0') -lowerMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE lowerMapping #-} --- LATIN CAPITAL LETTER I WITH DOT ABOVE -lowerMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000') -lowerMapping c s = Yield (toLower c) (CC s '\0' '\0') -titleMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE titleMapping #-} --- LATIN SMALL LETTER SHARP S -titleMapping '\x00df' s = Yield '\x0053' (CC s '\x0073' '\x0000') --- LATIN SMALL LIGATURE FF -titleMapping '\xfb00' s = Yield '\x0046' (CC s '\x0066' '\x0000') --- LATIN SMALL LIGATURE FI -titleMapping '\xfb01' s = Yield '\x0046' (CC s '\x0069' '\x0000') --- LATIN SMALL LIGATURE FL -titleMapping '\xfb02' s = Yield '\x0046' (CC s '\x006c' '\x0000') --- LATIN SMALL LIGATURE FFI -titleMapping '\xfb03' s = Yield '\x0046' (CC s '\x0066' '\x0069') --- LATIN SMALL LIGATURE FFL -titleMapping '\xfb04' s = Yield '\x0046' (CC s '\x0066' '\x006c') --- LATIN SMALL LIGATURE LONG S T -titleMapping '\xfb05' s = Yield '\x0053' (CC s '\x0074' '\x0000') --- LATIN SMALL LIGATURE ST -titleMapping '\xfb06' s = Yield '\x0053' (CC s '\x0074' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -titleMapping '\x0587' s = Yield '\x0535' (CC s '\x0582' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -titleMapping '\xfb13' s = Yield '\x0544' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -titleMapping '\xfb14' s = Yield '\x0544' (CC s '\x0565' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -titleMapping '\xfb15' s = Yield '\x0544' (CC s '\x056b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -titleMapping '\xfb16' s = Yield '\x054e' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -titleMapping '\xfb17' s = Yield '\x0544' (CC s '\x056d' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -titleMapping '\x0149' s = Yield '\x02bc' (CC s '\x004e' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -titleMapping '\x0390' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -titleMapping '\x03b0' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- LATIN SMALL LETTER J WITH CARON -titleMapping '\x01f0' s = Yield '\x004a' (CC s '\x030c' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -titleMapping '\x1e96' s = Yield '\x0048' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -titleMapping '\x1e97' s = Yield '\x0054' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -titleMapping '\x1e98' s = Yield '\x0057' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -titleMapping '\x1e99' s = Yield '\x0059' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -titleMapping '\x1e9a' s = Yield '\x0041' (CC s '\x02be' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -titleMapping '\x1f50' s = Yield '\x03a5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -titleMapping '\x1f52' s = Yield '\x03a5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -titleMapping '\x1f54' s = Yield '\x03a5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -titleMapping '\x1f56' s = Yield '\x03a5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -titleMapping '\x1fb6' s = Yield '\x0391' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -titleMapping '\x1fc6' s = Yield '\x0397' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -titleMapping '\x1fd2' s = Yield '\x0399' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -titleMapping '\x1fd3' s = Yield '\x0399' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -titleMapping '\x1fd6' s = Yield '\x0399' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -titleMapping '\x1fd7' s = Yield '\x0399' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -titleMapping '\x1fe2' s = Yield '\x03a5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -titleMapping '\x1fe3' s = Yield '\x03a5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -titleMapping '\x1fe4' s = Yield '\x03a1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -titleMapping '\x1fe6' s = Yield '\x03a5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -titleMapping '\x1fe7' s = Yield '\x03a5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -titleMapping '\x1ff6' s = Yield '\x03a9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1fb2' s = Yield '\x1fba' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1fb4' s = Yield '\x0386' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1fc2' s = Yield '\x1fca' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1fc4' s = Yield '\x0389' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -titleMapping '\x1ff2' s = Yield '\x1ffa' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -titleMapping '\x1ff4' s = Yield '\x038f' (CC s '\x0345' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1fb7' s = Yield '\x0391' (CC s '\x0342' '\x0345') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0345') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -titleMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0345') -titleMapping c s = Yield (toTitle c) (CC s '\0' '\0') -foldMapping :: forall s. Char -> s -> Step (CC s) Char -{-# NOINLINE foldMapping #-} --- MICRO SIGN -foldMapping '\x00b5' s = Yield '\x03bc' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER SHARP S -foldMapping '\x00df' s = Yield '\x0073' (CC s '\x0073' '\x0000') --- LATIN CAPITAL LETTER I WITH DOT ABOVE -foldMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000') --- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE -foldMapping '\x0149' s = Yield '\x02bc' (CC s '\x006e' '\x0000') --- LATIN SMALL LETTER LONG S -foldMapping '\x017f' s = Yield '\x0073' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER J WITH CARON -foldMapping '\x01f0' s = Yield '\x006a' (CC s '\x030c' '\x0000') --- COMBINING GREEK YPOGEGRAMMENI -foldMapping '\x0345' s = Yield '\x03b9' (CC s '\x0000' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS -foldMapping '\x0390' s = Yield '\x03b9' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS -foldMapping '\x03b0' s = Yield '\x03c5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER FINAL SIGMA -foldMapping '\x03c2' s = Yield '\x03c3' (CC s '\x0000' '\x0000') --- GREEK BETA SYMBOL -foldMapping '\x03d0' s = Yield '\x03b2' (CC s '\x0000' '\x0000') --- GREEK THETA SYMBOL -foldMapping '\x03d1' s = Yield '\x03b8' (CC s '\x0000' '\x0000') --- GREEK PHI SYMBOL -foldMapping '\x03d5' s = Yield '\x03c6' (CC s '\x0000' '\x0000') --- GREEK PI SYMBOL -foldMapping '\x03d6' s = Yield '\x03c0' (CC s '\x0000' '\x0000') --- GREEK KAPPA SYMBOL -foldMapping '\x03f0' s = Yield '\x03ba' (CC s '\x0000' '\x0000') --- GREEK RHO SYMBOL -foldMapping '\x03f1' s = Yield '\x03c1' (CC s '\x0000' '\x0000') --- GREEK LUNATE EPSILON SYMBOL -foldMapping '\x03f5' s = Yield '\x03b5' (CC s '\x0000' '\x0000') --- ARMENIAN SMALL LIGATURE ECH YIWN -foldMapping '\x0587' s = Yield '\x0565' (CC s '\x0582' '\x0000') --- CHEROKEE SMALL LETTER YE -foldMapping '\x13f8' s = Yield '\x13f0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YI -foldMapping '\x13f9' s = Yield '\x13f1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YO -foldMapping '\x13fa' s = Yield '\x13f2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YU -foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YV -foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MV -foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER ROUNDED VE -foldMapping '\x1c80' s = Yield '\x0432' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER LONG-LEGGED DE -foldMapping '\x1c81' s = Yield '\x0434' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER NARROW O -foldMapping '\x1c82' s = Yield '\x043e' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER WIDE ES -foldMapping '\x1c83' s = Yield '\x0441' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER TALL TE -foldMapping '\x1c84' s = Yield '\x0442' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER THREE-LEGGED TE -foldMapping '\x1c85' s = Yield '\x0442' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER TALL HARD SIGN -foldMapping '\x1c86' s = Yield '\x044a' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER TALL YAT -foldMapping '\x1c87' s = Yield '\x0463' (CC s '\x0000' '\x0000') --- CYRILLIC SMALL LETTER UNBLENDED UK -foldMapping '\x1c88' s = Yield '\xa64b' (CC s '\x0000' '\x0000') --- LATIN SMALL LETTER H WITH LINE BELOW -foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000') --- LATIN SMALL LETTER T WITH DIAERESIS -foldMapping '\x1e97' s = Yield '\x0074' (CC s '\x0308' '\x0000') --- LATIN SMALL LETTER W WITH RING ABOVE -foldMapping '\x1e98' s = Yield '\x0077' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER Y WITH RING ABOVE -foldMapping '\x1e99' s = Yield '\x0079' (CC s '\x030a' '\x0000') --- LATIN SMALL LETTER A WITH RIGHT HALF RING -foldMapping '\x1e9a' s = Yield '\x0061' (CC s '\x02be' '\x0000') --- LATIN SMALL LETTER LONG S WITH DOT ABOVE -foldMapping '\x1e9b' s = Yield '\x1e61' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER SHARP S -foldMapping '\x1e9e' s = Yield '\x0073' (CC s '\x0073' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI -foldMapping '\x1f50' s = Yield '\x03c5' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA -foldMapping '\x1f52' s = Yield '\x03c5' (CC s '\x0313' '\x0300') --- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA -foldMapping '\x1f54' s = Yield '\x03c5' (CC s '\x0313' '\x0301') --- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI -foldMapping '\x1f56' s = Yield '\x03c5' (CC s '\x0313' '\x0342') --- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f80' s = Yield '\x1f00' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f81' s = Yield '\x1f01' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f82' s = Yield '\x1f02' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f83' s = Yield '\x1f03' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f84' s = Yield '\x1f04' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f85' s = Yield '\x1f05' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f86' s = Yield '\x1f06' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f87' s = Yield '\x1f07' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f88' s = Yield '\x1f00' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f89' s = Yield '\x1f01' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8a' s = Yield '\x1f02' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f8b' s = Yield '\x1f03' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8c' s = Yield '\x1f04' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f8d' s = Yield '\x1f05' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8e' s = Yield '\x1f06' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f8f' s = Yield '\x1f07' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1f90' s = Yield '\x1f20' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1f91' s = Yield '\x1f21' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f92' s = Yield '\x1f22' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1f93' s = Yield '\x1f23' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f94' s = Yield '\x1f24' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1f95' s = Yield '\x1f25' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f96' s = Yield '\x1f26' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1f97' s = Yield '\x1f27' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1f98' s = Yield '\x1f20' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1f99' s = Yield '\x1f21' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9a' s = Yield '\x1f22' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1f9b' s = Yield '\x1f23' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9c' s = Yield '\x1f24' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1f9d' s = Yield '\x1f25' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9e' s = Yield '\x1f26' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1f9f' s = Yield '\x1f27' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI -foldMapping '\x1fa0' s = Yield '\x1f60' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI -foldMapping '\x1fa1' s = Yield '\x1f61' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa2' s = Yield '\x1f62' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI -foldMapping '\x1fa3' s = Yield '\x1f63' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa4' s = Yield '\x1f64' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI -foldMapping '\x1fa5' s = Yield '\x1f65' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa6' s = Yield '\x1f66' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fa7' s = Yield '\x1f67' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI -foldMapping '\x1fa8' s = Yield '\x1f60' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI -foldMapping '\x1fa9' s = Yield '\x1f61' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1faa' s = Yield '\x1f62' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI -foldMapping '\x1fab' s = Yield '\x1f63' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fac' s = Yield '\x1f64' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI -foldMapping '\x1fad' s = Yield '\x1f65' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1fae' s = Yield '\x1f66' (CC s '\x03b9' '\x0000') --- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI -foldMapping '\x1faf' s = Yield '\x1f67' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fb2' s = Yield '\x1f70' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI -foldMapping '\x1fb3' s = Yield '\x03b1' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fb4' s = Yield '\x03ac' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI -foldMapping '\x1fb6' s = Yield '\x03b1' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fb7' s = Yield '\x03b1' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI -foldMapping '\x1fbc' s = Yield '\x03b1' (CC s '\x03b9' '\x0000') --- GREEK PROSGEGRAMMENI -foldMapping '\x1fbe' s = Yield '\x03b9' (CC s '\x0000' '\x0000') --- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1fc2' s = Yield '\x1f74' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI -foldMapping '\x1fc3' s = Yield '\x03b7' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1fc4' s = Yield '\x03ae' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI -foldMapping '\x1fc6' s = Yield '\x03b7' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1fc7' s = Yield '\x03b7' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI -foldMapping '\x1fcc' s = Yield '\x03b7' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA -foldMapping '\x1fd2' s = Yield '\x03b9' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA -foldMapping '\x1fd3' s = Yield '\x03b9' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER IOTA WITH PERISPOMENI -foldMapping '\x1fd6' s = Yield '\x03b9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fd7' s = Yield '\x03b9' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA -foldMapping '\x1fe2' s = Yield '\x03c5' (CC s '\x0308' '\x0300') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA -foldMapping '\x1fe3' s = Yield '\x03c5' (CC s '\x0308' '\x0301') --- GREEK SMALL LETTER RHO WITH PSILI -foldMapping '\x1fe4' s = Yield '\x03c1' (CC s '\x0313' '\x0000') --- GREEK SMALL LETTER UPSILON WITH PERISPOMENI -foldMapping '\x1fe6' s = Yield '\x03c5' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI -foldMapping '\x1fe7' s = Yield '\x03c5' (CC s '\x0308' '\x0342') --- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI -foldMapping '\x1ff2' s = Yield '\x1f7c' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI -foldMapping '\x1ff3' s = Yield '\x03c9' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI -foldMapping '\x1ff4' s = Yield '\x03ce' (CC s '\x03b9' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI -foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000') --- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI -foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9') --- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI -foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000') --- LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY -foldMapping '\xa7c7' s = Yield '\xa7c8' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY -foldMapping '\xa7c9' s = Yield '\xa7ca' (CC s '\x0000' '\x0000') --- LATIN CAPITAL LETTER REVERSED HALF H -foldMapping '\xa7f5' s = Yield '\xa7f6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER A -foldMapping '\xab70' s = Yield '\x13a0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER E -foldMapping '\xab71' s = Yield '\x13a1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER I -foldMapping '\xab72' s = Yield '\x13a2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER O -foldMapping '\xab73' s = Yield '\x13a3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER U -foldMapping '\xab74' s = Yield '\x13a4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER V -foldMapping '\xab75' s = Yield '\x13a5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GA -foldMapping '\xab76' s = Yield '\x13a6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER KA -foldMapping '\xab77' s = Yield '\x13a7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GE -foldMapping '\xab78' s = Yield '\x13a8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GI -foldMapping '\xab79' s = Yield '\x13a9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GO -foldMapping '\xab7a' s = Yield '\x13aa' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GU -foldMapping '\xab7b' s = Yield '\x13ab' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER GV -foldMapping '\xab7c' s = Yield '\x13ac' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HA -foldMapping '\xab7d' s = Yield '\x13ad' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HE -foldMapping '\xab7e' s = Yield '\x13ae' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HI -foldMapping '\xab7f' s = Yield '\x13af' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HO -foldMapping '\xab80' s = Yield '\x13b0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HU -foldMapping '\xab81' s = Yield '\x13b1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HV -foldMapping '\xab82' s = Yield '\x13b2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LA -foldMapping '\xab83' s = Yield '\x13b3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LE -foldMapping '\xab84' s = Yield '\x13b4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LI -foldMapping '\xab85' s = Yield '\x13b5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LO -foldMapping '\xab86' s = Yield '\x13b6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LU -foldMapping '\xab87' s = Yield '\x13b7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER LV -foldMapping '\xab88' s = Yield '\x13b8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MA -foldMapping '\xab89' s = Yield '\x13b9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER ME -foldMapping '\xab8a' s = Yield '\x13ba' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MI -foldMapping '\xab8b' s = Yield '\x13bb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MO -foldMapping '\xab8c' s = Yield '\x13bc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER MU -foldMapping '\xab8d' s = Yield '\x13bd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NA -foldMapping '\xab8e' s = Yield '\x13be' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER HNA -foldMapping '\xab8f' s = Yield '\x13bf' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NAH -foldMapping '\xab90' s = Yield '\x13c0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NE -foldMapping '\xab91' s = Yield '\x13c1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NI -foldMapping '\xab92' s = Yield '\x13c2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NO -foldMapping '\xab93' s = Yield '\x13c3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NU -foldMapping '\xab94' s = Yield '\x13c4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER NV -foldMapping '\xab95' s = Yield '\x13c5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUA -foldMapping '\xab96' s = Yield '\x13c6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUE -foldMapping '\xab97' s = Yield '\x13c7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUI -foldMapping '\xab98' s = Yield '\x13c8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUO -foldMapping '\xab99' s = Yield '\x13c9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUU -foldMapping '\xab9a' s = Yield '\x13ca' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER QUV -foldMapping '\xab9b' s = Yield '\x13cb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SA -foldMapping '\xab9c' s = Yield '\x13cc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER S -foldMapping '\xab9d' s = Yield '\x13cd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SE -foldMapping '\xab9e' s = Yield '\x13ce' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SI -foldMapping '\xab9f' s = Yield '\x13cf' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SO -foldMapping '\xaba0' s = Yield '\x13d0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SU -foldMapping '\xaba1' s = Yield '\x13d1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER SV -foldMapping '\xaba2' s = Yield '\x13d2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DA -foldMapping '\xaba3' s = Yield '\x13d3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TA -foldMapping '\xaba4' s = Yield '\x13d4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DE -foldMapping '\xaba5' s = Yield '\x13d5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TE -foldMapping '\xaba6' s = Yield '\x13d6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DI -foldMapping '\xaba7' s = Yield '\x13d7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TI -foldMapping '\xaba8' s = Yield '\x13d8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DO -foldMapping '\xaba9' s = Yield '\x13d9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DU -foldMapping '\xabaa' s = Yield '\x13da' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DV -foldMapping '\xabab' s = Yield '\x13db' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER DLA -foldMapping '\xabac' s = Yield '\x13dc' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLA -foldMapping '\xabad' s = Yield '\x13dd' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLE -foldMapping '\xabae' s = Yield '\x13de' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLI -foldMapping '\xabaf' s = Yield '\x13df' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLO -foldMapping '\xabb0' s = Yield '\x13e0' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLU -foldMapping '\xabb1' s = Yield '\x13e1' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TLV -foldMapping '\xabb2' s = Yield '\x13e2' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSA -foldMapping '\xabb3' s = Yield '\x13e3' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSE -foldMapping '\xabb4' s = Yield '\x13e4' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSI -foldMapping '\xabb5' s = Yield '\x13e5' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSO -foldMapping '\xabb6' s = Yield '\x13e6' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSU -foldMapping '\xabb7' s = Yield '\x13e7' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER TSV -foldMapping '\xabb8' s = Yield '\x13e8' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WA -foldMapping '\xabb9' s = Yield '\x13e9' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WE -foldMapping '\xabba' s = Yield '\x13ea' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WI -foldMapping '\xabbb' s = Yield '\x13eb' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WO -foldMapping '\xabbc' s = Yield '\x13ec' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WU -foldMapping '\xabbd' s = Yield '\x13ed' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER WV -foldMapping '\xabbe' s = Yield '\x13ee' (CC s '\x0000' '\x0000') --- CHEROKEE SMALL LETTER YA -foldMapping '\xabbf' s = Yield '\x13ef' (CC s '\x0000' '\x0000') --- LATIN SMALL LIGATURE FF -foldMapping '\xfb00' s = Yield '\x0066' (CC s '\x0066' '\x0000') --- LATIN SMALL LIGATURE FI -foldMapping '\xfb01' s = Yield '\x0066' (CC s '\x0069' '\x0000') --- LATIN SMALL LIGATURE FL -foldMapping '\xfb02' s = Yield '\x0066' (CC s '\x006c' '\x0000') --- LATIN SMALL LIGATURE FFI -foldMapping '\xfb03' s = Yield '\x0066' (CC s '\x0066' '\x0069') --- LATIN SMALL LIGATURE FFL -foldMapping '\xfb04' s = Yield '\x0066' (CC s '\x0066' '\x006c') --- LATIN SMALL LIGATURE LONG S T -foldMapping '\xfb05' s = Yield '\x0073' (CC s '\x0074' '\x0000') --- LATIN SMALL LIGATURE ST -foldMapping '\xfb06' s = Yield '\x0073' (CC s '\x0074' '\x0000') --- ARMENIAN SMALL LIGATURE MEN NOW -foldMapping '\xfb13' s = Yield '\x0574' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN ECH -foldMapping '\xfb14' s = Yield '\x0574' (CC s '\x0565' '\x0000') --- ARMENIAN SMALL LIGATURE MEN INI -foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000') --- ARMENIAN SMALL LIGATURE VEW NOW -foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000') --- ARMENIAN SMALL LIGATURE MEN XEH -foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000') -foldMapping c s = Yield (toLower c) (CC s '\0' '\0') +-- AUTOMATICALLY GENERATED - DO NOT EDIT +-- Generated by scripts/CaseMapping.hs +-- CaseFolding-14.0.0.txt +-- Date: 2021-03-08, 19:35:41 GMT +-- SpecialCasing-14.0.0.txt +-- Date: 2021-03-08, 19:35:55 GMT + +{-# LANGUAGE LambdaCase, MagicHash, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} +module Data.Text.Internal.Fusion.CaseMapping where +import GHC.Int +import GHC.Exts +unI64 :: Int64 -> _ {- unboxed Int64 -} +unI64 (I64# n) = n + +upperMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE upperMapping #-} +upperMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 174063699 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 146800710 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 153092166 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 159383622 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 321057542111302 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 334251681644614 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 176160851 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 176160851 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2856322357 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2831156548 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2795504964 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2808087876 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2831156558 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2812282180 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1931484936 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1931484937 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1931484938 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1931484939 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1931484940 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1931484941 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1931484942 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1931484943 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1931484936 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1931484937 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1931484938 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1931484939 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1931484940 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1931484941 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1931484942 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1931484943 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1931484968 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1931484969 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1931484970 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1931484971 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1931484972 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1931484973 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1931484974 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1931484975 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1931484968 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1931484969 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1931484970 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1931484971 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1931484972 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1931484973 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1931484974 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1931484975 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1931485032 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1931485033 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1931485034 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1931485035 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1931485036 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1931485037 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1931485038 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1931485039 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1931485032 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1931485033 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1931485034 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1931485035 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1931485036 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1931485037 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1931485038 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1931485039 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1931477905 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1931477905 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1931477911 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1931477911 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1931477929 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1931477929 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1931485114 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1931477894 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1931485130 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1931477897 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1931485178 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1931477903 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4050602585752465 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4050602585752471 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4050602585752489 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c5'# -> unI64 452 + '\x01c6'# -> unI64 452 + '\x01c8'# -> unI64 455 + '\x01c9'# -> unI64 455 + '\x01cb'# -> unI64 458 + '\x01cc'# -> unI64 458 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f2'# -> unI64 497 + '\x01f3'# -> unI64 497 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x10d0'# -> unI64 7312 + '\x10d1'# -> unI64 7313 + '\x10d2'# -> unI64 7314 + '\x10d3'# -> unI64 7315 + '\x10d4'# -> unI64 7316 + '\x10d5'# -> unI64 7317 + '\x10d6'# -> unI64 7318 + '\x10d7'# -> unI64 7319 + '\x10d8'# -> unI64 7320 + '\x10d9'# -> unI64 7321 + '\x10da'# -> unI64 7322 + '\x10db'# -> unI64 7323 + '\x10dc'# -> unI64 7324 + '\x10dd'# -> unI64 7325 + '\x10de'# -> unI64 7326 + '\x10df'# -> unI64 7327 + '\x10e0'# -> unI64 7328 + '\x10e1'# -> unI64 7329 + '\x10e2'# -> unI64 7330 + '\x10e3'# -> unI64 7331 + '\x10e4'# -> unI64 7332 + '\x10e5'# -> unI64 7333 + '\x10e6'# -> unI64 7334 + '\x10e7'# -> unI64 7335 + '\x10e8'# -> unI64 7336 + '\x10e9'# -> unI64 7337 + '\x10ea'# -> unI64 7338 + '\x10eb'# -> unI64 7339 + '\x10ec'# -> unI64 7340 + '\x10ed'# -> unI64 7341 + '\x10ee'# -> unI64 7342 + '\x10ef'# -> unI64 7343 + '\x10f0'# -> unI64 7344 + '\x10f1'# -> unI64 7345 + '\x10f2'# -> unI64 7346 + '\x10f3'# -> unI64 7347 + '\x10f4'# -> unI64 7348 + '\x10f5'# -> unI64 7349 + '\x10f6'# -> unI64 7350 + '\x10f7'# -> unI64 7351 + '\x10f8'# -> unI64 7352 + '\x10f9'# -> unI64 7353 + '\x10fa'# -> unI64 7354 + '\x10fd'# -> unI64 7357 + '\x10fe'# -> unI64 7358 + '\x10ff'# -> unI64 7359 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fbe'# -> unI64 921 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7d1'# -> unI64 42960 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +lowerMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE lowerMapping #-} +lowerMapping = \case + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1e9e'# -> unI64 223 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1f88'# -> unI64 8064 + '\x1f89'# -> unI64 8065 + '\x1f8a'# -> unI64 8066 + '\x1f8b'# -> unI64 8067 + '\x1f8c'# -> unI64 8068 + '\x1f8d'# -> unI64 8069 + '\x1f8e'# -> unI64 8070 + '\x1f8f'# -> unI64 8071 + '\x1f98'# -> unI64 8080 + '\x1f99'# -> unI64 8081 + '\x1f9a'# -> unI64 8082 + '\x1f9b'# -> unI64 8083 + '\x1f9c'# -> unI64 8084 + '\x1f9d'# -> unI64 8085 + '\x1f9e'# -> unI64 8086 + '\x1f9f'# -> unI64 8087 + '\x1fa8'# -> unI64 8096 + '\x1fa9'# -> unI64 8097 + '\x1faa'# -> unI64 8098 + '\x1fab'# -> unI64 8099 + '\x1fac'# -> unI64 8100 + '\x1fad'# -> unI64 8101 + '\x1fae'# -> unI64 8102 + '\x1faf'# -> unI64 8103 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fbc'# -> unI64 8115 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fcc'# -> unI64 8131 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x1ffc'# -> unI64 8179 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7d0'# -> unI64 42961 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 +titleMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE titleMapping #-} +titleMapping = \case + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172563 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909574 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201030 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492486 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575494 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108806 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269715 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269715 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985653 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819844 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168260 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751172 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819854 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945476 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 163578556 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429861 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778634 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373256 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390036 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584343 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584345 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200769 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459557 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987429 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498533 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720293 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025681 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025687 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918745 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429849 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025689 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651609 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918757 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429861 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459553 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025701 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651621 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025705 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1755324346 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1755317126 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1755324362 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1755317129 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1755324410 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1755317135 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 3681166678819729 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 3681166678819735 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 3681166678819753 + '\x0061'# -> unI64 65 + '\x0062'# -> unI64 66 + '\x0063'# -> unI64 67 + '\x0064'# -> unI64 68 + '\x0065'# -> unI64 69 + '\x0066'# -> unI64 70 + '\x0067'# -> unI64 71 + '\x0068'# -> unI64 72 + '\x0069'# -> unI64 73 + '\x006a'# -> unI64 74 + '\x006b'# -> unI64 75 + '\x006c'# -> unI64 76 + '\x006d'# -> unI64 77 + '\x006e'# -> unI64 78 + '\x006f'# -> unI64 79 + '\x0070'# -> unI64 80 + '\x0071'# -> unI64 81 + '\x0072'# -> unI64 82 + '\x0073'# -> unI64 83 + '\x0074'# -> unI64 84 + '\x0075'# -> unI64 85 + '\x0076'# -> unI64 86 + '\x0077'# -> unI64 87 + '\x0078'# -> unI64 88 + '\x0079'# -> unI64 89 + '\x007a'# -> unI64 90 + '\x00b5'# -> unI64 924 + '\x00e0'# -> unI64 192 + '\x00e1'# -> unI64 193 + '\x00e2'# -> unI64 194 + '\x00e3'# -> unI64 195 + '\x00e4'# -> unI64 196 + '\x00e5'# -> unI64 197 + '\x00e6'# -> unI64 198 + '\x00e7'# -> unI64 199 + '\x00e8'# -> unI64 200 + '\x00e9'# -> unI64 201 + '\x00ea'# -> unI64 202 + '\x00eb'# -> unI64 203 + '\x00ec'# -> unI64 204 + '\x00ed'# -> unI64 205 + '\x00ee'# -> unI64 206 + '\x00ef'# -> unI64 207 + '\x00f0'# -> unI64 208 + '\x00f1'# -> unI64 209 + '\x00f2'# -> unI64 210 + '\x00f3'# -> unI64 211 + '\x00f4'# -> unI64 212 + '\x00f5'# -> unI64 213 + '\x00f6'# -> unI64 214 + '\x00f8'# -> unI64 216 + '\x00f9'# -> unI64 217 + '\x00fa'# -> unI64 218 + '\x00fb'# -> unI64 219 + '\x00fc'# -> unI64 220 + '\x00fd'# -> unI64 221 + '\x00fe'# -> unI64 222 + '\x00ff'# -> unI64 376 + '\x0101'# -> unI64 256 + '\x0103'# -> unI64 258 + '\x0105'# -> unI64 260 + '\x0107'# -> unI64 262 + '\x0109'# -> unI64 264 + '\x010b'# -> unI64 266 + '\x010d'# -> unI64 268 + '\x010f'# -> unI64 270 + '\x0111'# -> unI64 272 + '\x0113'# -> unI64 274 + '\x0115'# -> unI64 276 + '\x0117'# -> unI64 278 + '\x0119'# -> unI64 280 + '\x011b'# -> unI64 282 + '\x011d'# -> unI64 284 + '\x011f'# -> unI64 286 + '\x0121'# -> unI64 288 + '\x0123'# -> unI64 290 + '\x0125'# -> unI64 292 + '\x0127'# -> unI64 294 + '\x0129'# -> unI64 296 + '\x012b'# -> unI64 298 + '\x012d'# -> unI64 300 + '\x012f'# -> unI64 302 + '\x0131'# -> unI64 73 + '\x0133'# -> unI64 306 + '\x0135'# -> unI64 308 + '\x0137'# -> unI64 310 + '\x013a'# -> unI64 313 + '\x013c'# -> unI64 315 + '\x013e'# -> unI64 317 + '\x0140'# -> unI64 319 + '\x0142'# -> unI64 321 + '\x0144'# -> unI64 323 + '\x0146'# -> unI64 325 + '\x0148'# -> unI64 327 + '\x014b'# -> unI64 330 + '\x014d'# -> unI64 332 + '\x014f'# -> unI64 334 + '\x0151'# -> unI64 336 + '\x0153'# -> unI64 338 + '\x0155'# -> unI64 340 + '\x0157'# -> unI64 342 + '\x0159'# -> unI64 344 + '\x015b'# -> unI64 346 + '\x015d'# -> unI64 348 + '\x015f'# -> unI64 350 + '\x0161'# -> unI64 352 + '\x0163'# -> unI64 354 + '\x0165'# -> unI64 356 + '\x0167'# -> unI64 358 + '\x0169'# -> unI64 360 + '\x016b'# -> unI64 362 + '\x016d'# -> unI64 364 + '\x016f'# -> unI64 366 + '\x0171'# -> unI64 368 + '\x0173'# -> unI64 370 + '\x0175'# -> unI64 372 + '\x0177'# -> unI64 374 + '\x017a'# -> unI64 377 + '\x017c'# -> unI64 379 + '\x017e'# -> unI64 381 + '\x017f'# -> unI64 83 + '\x0180'# -> unI64 579 + '\x0183'# -> unI64 386 + '\x0185'# -> unI64 388 + '\x0188'# -> unI64 391 + '\x018c'# -> unI64 395 + '\x0192'# -> unI64 401 + '\x0195'# -> unI64 502 + '\x0199'# -> unI64 408 + '\x019a'# -> unI64 573 + '\x019e'# -> unI64 544 + '\x01a1'# -> unI64 416 + '\x01a3'# -> unI64 418 + '\x01a5'# -> unI64 420 + '\x01a8'# -> unI64 423 + '\x01ad'# -> unI64 428 + '\x01b0'# -> unI64 431 + '\x01b4'# -> unI64 435 + '\x01b6'# -> unI64 437 + '\x01b9'# -> unI64 440 + '\x01bd'# -> unI64 444 + '\x01bf'# -> unI64 503 + '\x01c4'# -> unI64 453 + '\x01c6'# -> unI64 453 + '\x01c7'# -> unI64 456 + '\x01c9'# -> unI64 456 + '\x01ca'# -> unI64 459 + '\x01cc'# -> unI64 459 + '\x01ce'# -> unI64 461 + '\x01d0'# -> unI64 463 + '\x01d2'# -> unI64 465 + '\x01d4'# -> unI64 467 + '\x01d6'# -> unI64 469 + '\x01d8'# -> unI64 471 + '\x01da'# -> unI64 473 + '\x01dc'# -> unI64 475 + '\x01dd'# -> unI64 398 + '\x01df'# -> unI64 478 + '\x01e1'# -> unI64 480 + '\x01e3'# -> unI64 482 + '\x01e5'# -> unI64 484 + '\x01e7'# -> unI64 486 + '\x01e9'# -> unI64 488 + '\x01eb'# -> unI64 490 + '\x01ed'# -> unI64 492 + '\x01ef'# -> unI64 494 + '\x01f1'# -> unI64 498 + '\x01f3'# -> unI64 498 + '\x01f5'# -> unI64 500 + '\x01f9'# -> unI64 504 + '\x01fb'# -> unI64 506 + '\x01fd'# -> unI64 508 + '\x01ff'# -> unI64 510 + '\x0201'# -> unI64 512 + '\x0203'# -> unI64 514 + '\x0205'# -> unI64 516 + '\x0207'# -> unI64 518 + '\x0209'# -> unI64 520 + '\x020b'# -> unI64 522 + '\x020d'# -> unI64 524 + '\x020f'# -> unI64 526 + '\x0211'# -> unI64 528 + '\x0213'# -> unI64 530 + '\x0215'# -> unI64 532 + '\x0217'# -> unI64 534 + '\x0219'# -> unI64 536 + '\x021b'# -> unI64 538 + '\x021d'# -> unI64 540 + '\x021f'# -> unI64 542 + '\x0223'# -> unI64 546 + '\x0225'# -> unI64 548 + '\x0227'# -> unI64 550 + '\x0229'# -> unI64 552 + '\x022b'# -> unI64 554 + '\x022d'# -> unI64 556 + '\x022f'# -> unI64 558 + '\x0231'# -> unI64 560 + '\x0233'# -> unI64 562 + '\x023c'# -> unI64 571 + '\x023f'# -> unI64 11390 + '\x0240'# -> unI64 11391 + '\x0242'# -> unI64 577 + '\x0247'# -> unI64 582 + '\x0249'# -> unI64 584 + '\x024b'# -> unI64 586 + '\x024d'# -> unI64 588 + '\x024f'# -> unI64 590 + '\x0250'# -> unI64 11375 + '\x0251'# -> unI64 11373 + '\x0252'# -> unI64 11376 + '\x0253'# -> unI64 385 + '\x0254'# -> unI64 390 + '\x0256'# -> unI64 393 + '\x0257'# -> unI64 394 + '\x0259'# -> unI64 399 + '\x025b'# -> unI64 400 + '\x025c'# -> unI64 42923 + '\x0260'# -> unI64 403 + '\x0261'# -> unI64 42924 + '\x0263'# -> unI64 404 + '\x0265'# -> unI64 42893 + '\x0266'# -> unI64 42922 + '\x0268'# -> unI64 407 + '\x0269'# -> unI64 406 + '\x026a'# -> unI64 42926 + '\x026b'# -> unI64 11362 + '\x026c'# -> unI64 42925 + '\x026f'# -> unI64 412 + '\x0271'# -> unI64 11374 + '\x0272'# -> unI64 413 + '\x0275'# -> unI64 415 + '\x027d'# -> unI64 11364 + '\x0280'# -> unI64 422 + '\x0282'# -> unI64 42949 + '\x0283'# -> unI64 425 + '\x0287'# -> unI64 42929 + '\x0288'# -> unI64 430 + '\x0289'# -> unI64 580 + '\x028a'# -> unI64 433 + '\x028b'# -> unI64 434 + '\x028c'# -> unI64 581 + '\x0292'# -> unI64 439 + '\x029d'# -> unI64 42930 + '\x029e'# -> unI64 42928 + '\x0345'# -> unI64 921 + '\x0371'# -> unI64 880 + '\x0373'# -> unI64 882 + '\x0377'# -> unI64 886 + '\x037b'# -> unI64 1021 + '\x037c'# -> unI64 1022 + '\x037d'# -> unI64 1023 + '\x03ac'# -> unI64 902 + '\x03ad'# -> unI64 904 + '\x03ae'# -> unI64 905 + '\x03af'# -> unI64 906 + '\x03b1'# -> unI64 913 + '\x03b2'# -> unI64 914 + '\x03b3'# -> unI64 915 + '\x03b4'# -> unI64 916 + '\x03b5'# -> unI64 917 + '\x03b6'# -> unI64 918 + '\x03b7'# -> unI64 919 + '\x03b8'# -> unI64 920 + '\x03b9'# -> unI64 921 + '\x03ba'# -> unI64 922 + '\x03bb'# -> unI64 923 + '\x03bc'# -> unI64 924 + '\x03bd'# -> unI64 925 + '\x03be'# -> unI64 926 + '\x03bf'# -> unI64 927 + '\x03c0'# -> unI64 928 + '\x03c1'# -> unI64 929 + '\x03c2'# -> unI64 931 + '\x03c3'# -> unI64 931 + '\x03c4'# -> unI64 932 + '\x03c5'# -> unI64 933 + '\x03c6'# -> unI64 934 + '\x03c7'# -> unI64 935 + '\x03c8'# -> unI64 936 + '\x03c9'# -> unI64 937 + '\x03ca'# -> unI64 938 + '\x03cb'# -> unI64 939 + '\x03cc'# -> unI64 908 + '\x03cd'# -> unI64 910 + '\x03ce'# -> unI64 911 + '\x03d0'# -> unI64 914 + '\x03d1'# -> unI64 920 + '\x03d5'# -> unI64 934 + '\x03d6'# -> unI64 928 + '\x03d7'# -> unI64 975 + '\x03d9'# -> unI64 984 + '\x03db'# -> unI64 986 + '\x03dd'# -> unI64 988 + '\x03df'# -> unI64 990 + '\x03e1'# -> unI64 992 + '\x03e3'# -> unI64 994 + '\x03e5'# -> unI64 996 + '\x03e7'# -> unI64 998 + '\x03e9'# -> unI64 1000 + '\x03eb'# -> unI64 1002 + '\x03ed'# -> unI64 1004 + '\x03ef'# -> unI64 1006 + '\x03f0'# -> unI64 922 + '\x03f1'# -> unI64 929 + '\x03f2'# -> unI64 1017 + '\x03f3'# -> unI64 895 + '\x03f5'# -> unI64 917 + '\x03f8'# -> unI64 1015 + '\x03fb'# -> unI64 1018 + '\x0430'# -> unI64 1040 + '\x0431'# -> unI64 1041 + '\x0432'# -> unI64 1042 + '\x0433'# -> unI64 1043 + '\x0434'# -> unI64 1044 + '\x0435'# -> unI64 1045 + '\x0436'# -> unI64 1046 + '\x0437'# -> unI64 1047 + '\x0438'# -> unI64 1048 + '\x0439'# -> unI64 1049 + '\x043a'# -> unI64 1050 + '\x043b'# -> unI64 1051 + '\x043c'# -> unI64 1052 + '\x043d'# -> unI64 1053 + '\x043e'# -> unI64 1054 + '\x043f'# -> unI64 1055 + '\x0440'# -> unI64 1056 + '\x0441'# -> unI64 1057 + '\x0442'# -> unI64 1058 + '\x0443'# -> unI64 1059 + '\x0444'# -> unI64 1060 + '\x0445'# -> unI64 1061 + '\x0446'# -> unI64 1062 + '\x0447'# -> unI64 1063 + '\x0448'# -> unI64 1064 + '\x0449'# -> unI64 1065 + '\x044a'# -> unI64 1066 + '\x044b'# -> unI64 1067 + '\x044c'# -> unI64 1068 + '\x044d'# -> unI64 1069 + '\x044e'# -> unI64 1070 + '\x044f'# -> unI64 1071 + '\x0450'# -> unI64 1024 + '\x0451'# -> unI64 1025 + '\x0452'# -> unI64 1026 + '\x0453'# -> unI64 1027 + '\x0454'# -> unI64 1028 + '\x0455'# -> unI64 1029 + '\x0456'# -> unI64 1030 + '\x0457'# -> unI64 1031 + '\x0458'# -> unI64 1032 + '\x0459'# -> unI64 1033 + '\x045a'# -> unI64 1034 + '\x045b'# -> unI64 1035 + '\x045c'# -> unI64 1036 + '\x045d'# -> unI64 1037 + '\x045e'# -> unI64 1038 + '\x045f'# -> unI64 1039 + '\x0461'# -> unI64 1120 + '\x0463'# -> unI64 1122 + '\x0465'# -> unI64 1124 + '\x0467'# -> unI64 1126 + '\x0469'# -> unI64 1128 + '\x046b'# -> unI64 1130 + '\x046d'# -> unI64 1132 + '\x046f'# -> unI64 1134 + '\x0471'# -> unI64 1136 + '\x0473'# -> unI64 1138 + '\x0475'# -> unI64 1140 + '\x0477'# -> unI64 1142 + '\x0479'# -> unI64 1144 + '\x047b'# -> unI64 1146 + '\x047d'# -> unI64 1148 + '\x047f'# -> unI64 1150 + '\x0481'# -> unI64 1152 + '\x048b'# -> unI64 1162 + '\x048d'# -> unI64 1164 + '\x048f'# -> unI64 1166 + '\x0491'# -> unI64 1168 + '\x0493'# -> unI64 1170 + '\x0495'# -> unI64 1172 + '\x0497'# -> unI64 1174 + '\x0499'# -> unI64 1176 + '\x049b'# -> unI64 1178 + '\x049d'# -> unI64 1180 + '\x049f'# -> unI64 1182 + '\x04a1'# -> unI64 1184 + '\x04a3'# -> unI64 1186 + '\x04a5'# -> unI64 1188 + '\x04a7'# -> unI64 1190 + '\x04a9'# -> unI64 1192 + '\x04ab'# -> unI64 1194 + '\x04ad'# -> unI64 1196 + '\x04af'# -> unI64 1198 + '\x04b1'# -> unI64 1200 + '\x04b3'# -> unI64 1202 + '\x04b5'# -> unI64 1204 + '\x04b7'# -> unI64 1206 + '\x04b9'# -> unI64 1208 + '\x04bb'# -> unI64 1210 + '\x04bd'# -> unI64 1212 + '\x04bf'# -> unI64 1214 + '\x04c2'# -> unI64 1217 + '\x04c4'# -> unI64 1219 + '\x04c6'# -> unI64 1221 + '\x04c8'# -> unI64 1223 + '\x04ca'# -> unI64 1225 + '\x04cc'# -> unI64 1227 + '\x04ce'# -> unI64 1229 + '\x04cf'# -> unI64 1216 + '\x04d1'# -> unI64 1232 + '\x04d3'# -> unI64 1234 + '\x04d5'# -> unI64 1236 + '\x04d7'# -> unI64 1238 + '\x04d9'# -> unI64 1240 + '\x04db'# -> unI64 1242 + '\x04dd'# -> unI64 1244 + '\x04df'# -> unI64 1246 + '\x04e1'# -> unI64 1248 + '\x04e3'# -> unI64 1250 + '\x04e5'# -> unI64 1252 + '\x04e7'# -> unI64 1254 + '\x04e9'# -> unI64 1256 + '\x04eb'# -> unI64 1258 + '\x04ed'# -> unI64 1260 + '\x04ef'# -> unI64 1262 + '\x04f1'# -> unI64 1264 + '\x04f3'# -> unI64 1266 + '\x04f5'# -> unI64 1268 + '\x04f7'# -> unI64 1270 + '\x04f9'# -> unI64 1272 + '\x04fb'# -> unI64 1274 + '\x04fd'# -> unI64 1276 + '\x04ff'# -> unI64 1278 + '\x0501'# -> unI64 1280 + '\x0503'# -> unI64 1282 + '\x0505'# -> unI64 1284 + '\x0507'# -> unI64 1286 + '\x0509'# -> unI64 1288 + '\x050b'# -> unI64 1290 + '\x050d'# -> unI64 1292 + '\x050f'# -> unI64 1294 + '\x0511'# -> unI64 1296 + '\x0513'# -> unI64 1298 + '\x0515'# -> unI64 1300 + '\x0517'# -> unI64 1302 + '\x0519'# -> unI64 1304 + '\x051b'# -> unI64 1306 + '\x051d'# -> unI64 1308 + '\x051f'# -> unI64 1310 + '\x0521'# -> unI64 1312 + '\x0523'# -> unI64 1314 + '\x0525'# -> unI64 1316 + '\x0527'# -> unI64 1318 + '\x0529'# -> unI64 1320 + '\x052b'# -> unI64 1322 + '\x052d'# -> unI64 1324 + '\x052f'# -> unI64 1326 + '\x0561'# -> unI64 1329 + '\x0562'# -> unI64 1330 + '\x0563'# -> unI64 1331 + '\x0564'# -> unI64 1332 + '\x0565'# -> unI64 1333 + '\x0566'# -> unI64 1334 + '\x0567'# -> unI64 1335 + '\x0568'# -> unI64 1336 + '\x0569'# -> unI64 1337 + '\x056a'# -> unI64 1338 + '\x056b'# -> unI64 1339 + '\x056c'# -> unI64 1340 + '\x056d'# -> unI64 1341 + '\x056e'# -> unI64 1342 + '\x056f'# -> unI64 1343 + '\x0570'# -> unI64 1344 + '\x0571'# -> unI64 1345 + '\x0572'# -> unI64 1346 + '\x0573'# -> unI64 1347 + '\x0574'# -> unI64 1348 + '\x0575'# -> unI64 1349 + '\x0576'# -> unI64 1350 + '\x0577'# -> unI64 1351 + '\x0578'# -> unI64 1352 + '\x0579'# -> unI64 1353 + '\x057a'# -> unI64 1354 + '\x057b'# -> unI64 1355 + '\x057c'# -> unI64 1356 + '\x057d'# -> unI64 1357 + '\x057e'# -> unI64 1358 + '\x057f'# -> unI64 1359 + '\x0580'# -> unI64 1360 + '\x0581'# -> unI64 1361 + '\x0582'# -> unI64 1362 + '\x0583'# -> unI64 1363 + '\x0584'# -> unI64 1364 + '\x0585'# -> unI64 1365 + '\x0586'# -> unI64 1366 + '\x13f8'# -> unI64 5104 + '\x13f9'# -> unI64 5105 + '\x13fa'# -> unI64 5106 + '\x13fb'# -> unI64 5107 + '\x13fc'# -> unI64 5108 + '\x13fd'# -> unI64 5109 + '\x1c80'# -> unI64 1042 + '\x1c81'# -> unI64 1044 + '\x1c82'# -> unI64 1054 + '\x1c83'# -> unI64 1057 + '\x1c84'# -> unI64 1058 + '\x1c85'# -> unI64 1058 + '\x1c86'# -> unI64 1066 + '\x1c87'# -> unI64 1122 + '\x1c88'# -> unI64 42570 + '\x1d79'# -> unI64 42877 + '\x1d7d'# -> unI64 11363 + '\x1d8e'# -> unI64 42950 + '\x1e01'# -> unI64 7680 + '\x1e03'# -> unI64 7682 + '\x1e05'# -> unI64 7684 + '\x1e07'# -> unI64 7686 + '\x1e09'# -> unI64 7688 + '\x1e0b'# -> unI64 7690 + '\x1e0d'# -> unI64 7692 + '\x1e0f'# -> unI64 7694 + '\x1e11'# -> unI64 7696 + '\x1e13'# -> unI64 7698 + '\x1e15'# -> unI64 7700 + '\x1e17'# -> unI64 7702 + '\x1e19'# -> unI64 7704 + '\x1e1b'# -> unI64 7706 + '\x1e1d'# -> unI64 7708 + '\x1e1f'# -> unI64 7710 + '\x1e21'# -> unI64 7712 + '\x1e23'# -> unI64 7714 + '\x1e25'# -> unI64 7716 + '\x1e27'# -> unI64 7718 + '\x1e29'# -> unI64 7720 + '\x1e2b'# -> unI64 7722 + '\x1e2d'# -> unI64 7724 + '\x1e2f'# -> unI64 7726 + '\x1e31'# -> unI64 7728 + '\x1e33'# -> unI64 7730 + '\x1e35'# -> unI64 7732 + '\x1e37'# -> unI64 7734 + '\x1e39'# -> unI64 7736 + '\x1e3b'# -> unI64 7738 + '\x1e3d'# -> unI64 7740 + '\x1e3f'# -> unI64 7742 + '\x1e41'# -> unI64 7744 + '\x1e43'# -> unI64 7746 + '\x1e45'# -> unI64 7748 + '\x1e47'# -> unI64 7750 + '\x1e49'# -> unI64 7752 + '\x1e4b'# -> unI64 7754 + '\x1e4d'# -> unI64 7756 + '\x1e4f'# -> unI64 7758 + '\x1e51'# -> unI64 7760 + '\x1e53'# -> unI64 7762 + '\x1e55'# -> unI64 7764 + '\x1e57'# -> unI64 7766 + '\x1e59'# -> unI64 7768 + '\x1e5b'# -> unI64 7770 + '\x1e5d'# -> unI64 7772 + '\x1e5f'# -> unI64 7774 + '\x1e61'# -> unI64 7776 + '\x1e63'# -> unI64 7778 + '\x1e65'# -> unI64 7780 + '\x1e67'# -> unI64 7782 + '\x1e69'# -> unI64 7784 + '\x1e6b'# -> unI64 7786 + '\x1e6d'# -> unI64 7788 + '\x1e6f'# -> unI64 7790 + '\x1e71'# -> unI64 7792 + '\x1e73'# -> unI64 7794 + '\x1e75'# -> unI64 7796 + '\x1e77'# -> unI64 7798 + '\x1e79'# -> unI64 7800 + '\x1e7b'# -> unI64 7802 + '\x1e7d'# -> unI64 7804 + '\x1e7f'# -> unI64 7806 + '\x1e81'# -> unI64 7808 + '\x1e83'# -> unI64 7810 + '\x1e85'# -> unI64 7812 + '\x1e87'# -> unI64 7814 + '\x1e89'# -> unI64 7816 + '\x1e8b'# -> unI64 7818 + '\x1e8d'# -> unI64 7820 + '\x1e8f'# -> unI64 7822 + '\x1e91'# -> unI64 7824 + '\x1e93'# -> unI64 7826 + '\x1e95'# -> unI64 7828 + '\x1e9b'# -> unI64 7776 + '\x1ea1'# -> unI64 7840 + '\x1ea3'# -> unI64 7842 + '\x1ea5'# -> unI64 7844 + '\x1ea7'# -> unI64 7846 + '\x1ea9'# -> unI64 7848 + '\x1eab'# -> unI64 7850 + '\x1ead'# -> unI64 7852 + '\x1eaf'# -> unI64 7854 + '\x1eb1'# -> unI64 7856 + '\x1eb3'# -> unI64 7858 + '\x1eb5'# -> unI64 7860 + '\x1eb7'# -> unI64 7862 + '\x1eb9'# -> unI64 7864 + '\x1ebb'# -> unI64 7866 + '\x1ebd'# -> unI64 7868 + '\x1ebf'# -> unI64 7870 + '\x1ec1'# -> unI64 7872 + '\x1ec3'# -> unI64 7874 + '\x1ec5'# -> unI64 7876 + '\x1ec7'# -> unI64 7878 + '\x1ec9'# -> unI64 7880 + '\x1ecb'# -> unI64 7882 + '\x1ecd'# -> unI64 7884 + '\x1ecf'# -> unI64 7886 + '\x1ed1'# -> unI64 7888 + '\x1ed3'# -> unI64 7890 + '\x1ed5'# -> unI64 7892 + '\x1ed7'# -> unI64 7894 + '\x1ed9'# -> unI64 7896 + '\x1edb'# -> unI64 7898 + '\x1edd'# -> unI64 7900 + '\x1edf'# -> unI64 7902 + '\x1ee1'# -> unI64 7904 + '\x1ee3'# -> unI64 7906 + '\x1ee5'# -> unI64 7908 + '\x1ee7'# -> unI64 7910 + '\x1ee9'# -> unI64 7912 + '\x1eeb'# -> unI64 7914 + '\x1eed'# -> unI64 7916 + '\x1eef'# -> unI64 7918 + '\x1ef1'# -> unI64 7920 + '\x1ef3'# -> unI64 7922 + '\x1ef5'# -> unI64 7924 + '\x1ef7'# -> unI64 7926 + '\x1ef9'# -> unI64 7928 + '\x1efb'# -> unI64 7930 + '\x1efd'# -> unI64 7932 + '\x1eff'# -> unI64 7934 + '\x1f00'# -> unI64 7944 + '\x1f01'# -> unI64 7945 + '\x1f02'# -> unI64 7946 + '\x1f03'# -> unI64 7947 + '\x1f04'# -> unI64 7948 + '\x1f05'# -> unI64 7949 + '\x1f06'# -> unI64 7950 + '\x1f07'# -> unI64 7951 + '\x1f10'# -> unI64 7960 + '\x1f11'# -> unI64 7961 + '\x1f12'# -> unI64 7962 + '\x1f13'# -> unI64 7963 + '\x1f14'# -> unI64 7964 + '\x1f15'# -> unI64 7965 + '\x1f20'# -> unI64 7976 + '\x1f21'# -> unI64 7977 + '\x1f22'# -> unI64 7978 + '\x1f23'# -> unI64 7979 + '\x1f24'# -> unI64 7980 + '\x1f25'# -> unI64 7981 + '\x1f26'# -> unI64 7982 + '\x1f27'# -> unI64 7983 + '\x1f30'# -> unI64 7992 + '\x1f31'# -> unI64 7993 + '\x1f32'# -> unI64 7994 + '\x1f33'# -> unI64 7995 + '\x1f34'# -> unI64 7996 + '\x1f35'# -> unI64 7997 + '\x1f36'# -> unI64 7998 + '\x1f37'# -> unI64 7999 + '\x1f40'# -> unI64 8008 + '\x1f41'# -> unI64 8009 + '\x1f42'# -> unI64 8010 + '\x1f43'# -> unI64 8011 + '\x1f44'# -> unI64 8012 + '\x1f45'# -> unI64 8013 + '\x1f51'# -> unI64 8025 + '\x1f53'# -> unI64 8027 + '\x1f55'# -> unI64 8029 + '\x1f57'# -> unI64 8031 + '\x1f60'# -> unI64 8040 + '\x1f61'# -> unI64 8041 + '\x1f62'# -> unI64 8042 + '\x1f63'# -> unI64 8043 + '\x1f64'# -> unI64 8044 + '\x1f65'# -> unI64 8045 + '\x1f66'# -> unI64 8046 + '\x1f67'# -> unI64 8047 + '\x1f70'# -> unI64 8122 + '\x1f71'# -> unI64 8123 + '\x1f72'# -> unI64 8136 + '\x1f73'# -> unI64 8137 + '\x1f74'# -> unI64 8138 + '\x1f75'# -> unI64 8139 + '\x1f76'# -> unI64 8154 + '\x1f77'# -> unI64 8155 + '\x1f78'# -> unI64 8184 + '\x1f79'# -> unI64 8185 + '\x1f7a'# -> unI64 8170 + '\x1f7b'# -> unI64 8171 + '\x1f7c'# -> unI64 8186 + '\x1f7d'# -> unI64 8187 + '\x1f80'# -> unI64 8072 + '\x1f81'# -> unI64 8073 + '\x1f82'# -> unI64 8074 + '\x1f83'# -> unI64 8075 + '\x1f84'# -> unI64 8076 + '\x1f85'# -> unI64 8077 + '\x1f86'# -> unI64 8078 + '\x1f87'# -> unI64 8079 + '\x1f90'# -> unI64 8088 + '\x1f91'# -> unI64 8089 + '\x1f92'# -> unI64 8090 + '\x1f93'# -> unI64 8091 + '\x1f94'# -> unI64 8092 + '\x1f95'# -> unI64 8093 + '\x1f96'# -> unI64 8094 + '\x1f97'# -> unI64 8095 + '\x1fa0'# -> unI64 8104 + '\x1fa1'# -> unI64 8105 + '\x1fa2'# -> unI64 8106 + '\x1fa3'# -> unI64 8107 + '\x1fa4'# -> unI64 8108 + '\x1fa5'# -> unI64 8109 + '\x1fa6'# -> unI64 8110 + '\x1fa7'# -> unI64 8111 + '\x1fb0'# -> unI64 8120 + '\x1fb1'# -> unI64 8121 + '\x1fb3'# -> unI64 8124 + '\x1fbe'# -> unI64 921 + '\x1fc3'# -> unI64 8140 + '\x1fd0'# -> unI64 8152 + '\x1fd1'# -> unI64 8153 + '\x1fe0'# -> unI64 8168 + '\x1fe1'# -> unI64 8169 + '\x1fe5'# -> unI64 8172 + '\x1ff3'# -> unI64 8188 + '\x214e'# -> unI64 8498 + '\x2170'# -> unI64 8544 + '\x2171'# -> unI64 8545 + '\x2172'# -> unI64 8546 + '\x2173'# -> unI64 8547 + '\x2174'# -> unI64 8548 + '\x2175'# -> unI64 8549 + '\x2176'# -> unI64 8550 + '\x2177'# -> unI64 8551 + '\x2178'# -> unI64 8552 + '\x2179'# -> unI64 8553 + '\x217a'# -> unI64 8554 + '\x217b'# -> unI64 8555 + '\x217c'# -> unI64 8556 + '\x217d'# -> unI64 8557 + '\x217e'# -> unI64 8558 + '\x217f'# -> unI64 8559 + '\x2184'# -> unI64 8579 + '\x24d0'# -> unI64 9398 + '\x24d1'# -> unI64 9399 + '\x24d2'# -> unI64 9400 + '\x24d3'# -> unI64 9401 + '\x24d4'# -> unI64 9402 + '\x24d5'# -> unI64 9403 + '\x24d6'# -> unI64 9404 + '\x24d7'# -> unI64 9405 + '\x24d8'# -> unI64 9406 + '\x24d9'# -> unI64 9407 + '\x24da'# -> unI64 9408 + '\x24db'# -> unI64 9409 + '\x24dc'# -> unI64 9410 + '\x24dd'# -> unI64 9411 + '\x24de'# -> unI64 9412 + '\x24df'# -> unI64 9413 + '\x24e0'# -> unI64 9414 + '\x24e1'# -> unI64 9415 + '\x24e2'# -> unI64 9416 + '\x24e3'# -> unI64 9417 + '\x24e4'# -> unI64 9418 + '\x24e5'# -> unI64 9419 + '\x24e6'# -> unI64 9420 + '\x24e7'# -> unI64 9421 + '\x24e8'# -> unI64 9422 + '\x24e9'# -> unI64 9423 + '\x2c30'# -> unI64 11264 + '\x2c31'# -> unI64 11265 + '\x2c32'# -> unI64 11266 + '\x2c33'# -> unI64 11267 + '\x2c34'# -> unI64 11268 + '\x2c35'# -> unI64 11269 + '\x2c36'# -> unI64 11270 + '\x2c37'# -> unI64 11271 + '\x2c38'# -> unI64 11272 + '\x2c39'# -> unI64 11273 + '\x2c3a'# -> unI64 11274 + '\x2c3b'# -> unI64 11275 + '\x2c3c'# -> unI64 11276 + '\x2c3d'# -> unI64 11277 + '\x2c3e'# -> unI64 11278 + '\x2c3f'# -> unI64 11279 + '\x2c40'# -> unI64 11280 + '\x2c41'# -> unI64 11281 + '\x2c42'# -> unI64 11282 + '\x2c43'# -> unI64 11283 + '\x2c44'# -> unI64 11284 + '\x2c45'# -> unI64 11285 + '\x2c46'# -> unI64 11286 + '\x2c47'# -> unI64 11287 + '\x2c48'# -> unI64 11288 + '\x2c49'# -> unI64 11289 + '\x2c4a'# -> unI64 11290 + '\x2c4b'# -> unI64 11291 + '\x2c4c'# -> unI64 11292 + '\x2c4d'# -> unI64 11293 + '\x2c4e'# -> unI64 11294 + '\x2c4f'# -> unI64 11295 + '\x2c50'# -> unI64 11296 + '\x2c51'# -> unI64 11297 + '\x2c52'# -> unI64 11298 + '\x2c53'# -> unI64 11299 + '\x2c54'# -> unI64 11300 + '\x2c55'# -> unI64 11301 + '\x2c56'# -> unI64 11302 + '\x2c57'# -> unI64 11303 + '\x2c58'# -> unI64 11304 + '\x2c59'# -> unI64 11305 + '\x2c5a'# -> unI64 11306 + '\x2c5b'# -> unI64 11307 + '\x2c5c'# -> unI64 11308 + '\x2c5d'# -> unI64 11309 + '\x2c5e'# -> unI64 11310 + '\x2c5f'# -> unI64 11311 + '\x2c61'# -> unI64 11360 + '\x2c65'# -> unI64 570 + '\x2c66'# -> unI64 574 + '\x2c68'# -> unI64 11367 + '\x2c6a'# -> unI64 11369 + '\x2c6c'# -> unI64 11371 + '\x2c73'# -> unI64 11378 + '\x2c76'# -> unI64 11381 + '\x2c81'# -> unI64 11392 + '\x2c83'# -> unI64 11394 + '\x2c85'# -> unI64 11396 + '\x2c87'# -> unI64 11398 + '\x2c89'# -> unI64 11400 + '\x2c8b'# -> unI64 11402 + '\x2c8d'# -> unI64 11404 + '\x2c8f'# -> unI64 11406 + '\x2c91'# -> unI64 11408 + '\x2c93'# -> unI64 11410 + '\x2c95'# -> unI64 11412 + '\x2c97'# -> unI64 11414 + '\x2c99'# -> unI64 11416 + '\x2c9b'# -> unI64 11418 + '\x2c9d'# -> unI64 11420 + '\x2c9f'# -> unI64 11422 + '\x2ca1'# -> unI64 11424 + '\x2ca3'# -> unI64 11426 + '\x2ca5'# -> unI64 11428 + '\x2ca7'# -> unI64 11430 + '\x2ca9'# -> unI64 11432 + '\x2cab'# -> unI64 11434 + '\x2cad'# -> unI64 11436 + '\x2caf'# -> unI64 11438 + '\x2cb1'# -> unI64 11440 + '\x2cb3'# -> unI64 11442 + '\x2cb5'# -> unI64 11444 + '\x2cb7'# -> unI64 11446 + '\x2cb9'# -> unI64 11448 + '\x2cbb'# -> unI64 11450 + '\x2cbd'# -> unI64 11452 + '\x2cbf'# -> unI64 11454 + '\x2cc1'# -> unI64 11456 + '\x2cc3'# -> unI64 11458 + '\x2cc5'# -> unI64 11460 + '\x2cc7'# -> unI64 11462 + '\x2cc9'# -> unI64 11464 + '\x2ccb'# -> unI64 11466 + '\x2ccd'# -> unI64 11468 + '\x2ccf'# -> unI64 11470 + '\x2cd1'# -> unI64 11472 + '\x2cd3'# -> unI64 11474 + '\x2cd5'# -> unI64 11476 + '\x2cd7'# -> unI64 11478 + '\x2cd9'# -> unI64 11480 + '\x2cdb'# -> unI64 11482 + '\x2cdd'# -> unI64 11484 + '\x2cdf'# -> unI64 11486 + '\x2ce1'# -> unI64 11488 + '\x2ce3'# -> unI64 11490 + '\x2cec'# -> unI64 11499 + '\x2cee'# -> unI64 11501 + '\x2cf3'# -> unI64 11506 + '\x2d00'# -> unI64 4256 + '\x2d01'# -> unI64 4257 + '\x2d02'# -> unI64 4258 + '\x2d03'# -> unI64 4259 + '\x2d04'# -> unI64 4260 + '\x2d05'# -> unI64 4261 + '\x2d06'# -> unI64 4262 + '\x2d07'# -> unI64 4263 + '\x2d08'# -> unI64 4264 + '\x2d09'# -> unI64 4265 + '\x2d0a'# -> unI64 4266 + '\x2d0b'# -> unI64 4267 + '\x2d0c'# -> unI64 4268 + '\x2d0d'# -> unI64 4269 + '\x2d0e'# -> unI64 4270 + '\x2d0f'# -> unI64 4271 + '\x2d10'# -> unI64 4272 + '\x2d11'# -> unI64 4273 + '\x2d12'# -> unI64 4274 + '\x2d13'# -> unI64 4275 + '\x2d14'# -> unI64 4276 + '\x2d15'# -> unI64 4277 + '\x2d16'# -> unI64 4278 + '\x2d17'# -> unI64 4279 + '\x2d18'# -> unI64 4280 + '\x2d19'# -> unI64 4281 + '\x2d1a'# -> unI64 4282 + '\x2d1b'# -> unI64 4283 + '\x2d1c'# -> unI64 4284 + '\x2d1d'# -> unI64 4285 + '\x2d1e'# -> unI64 4286 + '\x2d1f'# -> unI64 4287 + '\x2d20'# -> unI64 4288 + '\x2d21'# -> unI64 4289 + '\x2d22'# -> unI64 4290 + '\x2d23'# -> unI64 4291 + '\x2d24'# -> unI64 4292 + '\x2d25'# -> unI64 4293 + '\x2d27'# -> unI64 4295 + '\x2d2d'# -> unI64 4301 + '\xa641'# -> unI64 42560 + '\xa643'# -> unI64 42562 + '\xa645'# -> unI64 42564 + '\xa647'# -> unI64 42566 + '\xa649'# -> unI64 42568 + '\xa64b'# -> unI64 42570 + '\xa64d'# -> unI64 42572 + '\xa64f'# -> unI64 42574 + '\xa651'# -> unI64 42576 + '\xa653'# -> unI64 42578 + '\xa655'# -> unI64 42580 + '\xa657'# -> unI64 42582 + '\xa659'# -> unI64 42584 + '\xa65b'# -> unI64 42586 + '\xa65d'# -> unI64 42588 + '\xa65f'# -> unI64 42590 + '\xa661'# -> unI64 42592 + '\xa663'# -> unI64 42594 + '\xa665'# -> unI64 42596 + '\xa667'# -> unI64 42598 + '\xa669'# -> unI64 42600 + '\xa66b'# -> unI64 42602 + '\xa66d'# -> unI64 42604 + '\xa681'# -> unI64 42624 + '\xa683'# -> unI64 42626 + '\xa685'# -> unI64 42628 + '\xa687'# -> unI64 42630 + '\xa689'# -> unI64 42632 + '\xa68b'# -> unI64 42634 + '\xa68d'# -> unI64 42636 + '\xa68f'# -> unI64 42638 + '\xa691'# -> unI64 42640 + '\xa693'# -> unI64 42642 + '\xa695'# -> unI64 42644 + '\xa697'# -> unI64 42646 + '\xa699'# -> unI64 42648 + '\xa69b'# -> unI64 42650 + '\xa723'# -> unI64 42786 + '\xa725'# -> unI64 42788 + '\xa727'# -> unI64 42790 + '\xa729'# -> unI64 42792 + '\xa72b'# -> unI64 42794 + '\xa72d'# -> unI64 42796 + '\xa72f'# -> unI64 42798 + '\xa733'# -> unI64 42802 + '\xa735'# -> unI64 42804 + '\xa737'# -> unI64 42806 + '\xa739'# -> unI64 42808 + '\xa73b'# -> unI64 42810 + '\xa73d'# -> unI64 42812 + '\xa73f'# -> unI64 42814 + '\xa741'# -> unI64 42816 + '\xa743'# -> unI64 42818 + '\xa745'# -> unI64 42820 + '\xa747'# -> unI64 42822 + '\xa749'# -> unI64 42824 + '\xa74b'# -> unI64 42826 + '\xa74d'# -> unI64 42828 + '\xa74f'# -> unI64 42830 + '\xa751'# -> unI64 42832 + '\xa753'# -> unI64 42834 + '\xa755'# -> unI64 42836 + '\xa757'# -> unI64 42838 + '\xa759'# -> unI64 42840 + '\xa75b'# -> unI64 42842 + '\xa75d'# -> unI64 42844 + '\xa75f'# -> unI64 42846 + '\xa761'# -> unI64 42848 + '\xa763'# -> unI64 42850 + '\xa765'# -> unI64 42852 + '\xa767'# -> unI64 42854 + '\xa769'# -> unI64 42856 + '\xa76b'# -> unI64 42858 + '\xa76d'# -> unI64 42860 + '\xa76f'# -> unI64 42862 + '\xa77a'# -> unI64 42873 + '\xa77c'# -> unI64 42875 + '\xa77f'# -> unI64 42878 + '\xa781'# -> unI64 42880 + '\xa783'# -> unI64 42882 + '\xa785'# -> unI64 42884 + '\xa787'# -> unI64 42886 + '\xa78c'# -> unI64 42891 + '\xa791'# -> unI64 42896 + '\xa793'# -> unI64 42898 + '\xa794'# -> unI64 42948 + '\xa797'# -> unI64 42902 + '\xa799'# -> unI64 42904 + '\xa79b'# -> unI64 42906 + '\xa79d'# -> unI64 42908 + '\xa79f'# -> unI64 42910 + '\xa7a1'# -> unI64 42912 + '\xa7a3'# -> unI64 42914 + '\xa7a5'# -> unI64 42916 + '\xa7a7'# -> unI64 42918 + '\xa7a9'# -> unI64 42920 + '\xa7b5'# -> unI64 42932 + '\xa7b7'# -> unI64 42934 + '\xa7b9'# -> unI64 42936 + '\xa7bb'# -> unI64 42938 + '\xa7bd'# -> unI64 42940 + '\xa7bf'# -> unI64 42942 + '\xa7c1'# -> unI64 42944 + '\xa7c3'# -> unI64 42946 + '\xa7c8'# -> unI64 42951 + '\xa7ca'# -> unI64 42953 + '\xa7d1'# -> unI64 42960 + '\xa7d7'# -> unI64 42966 + '\xa7d9'# -> unI64 42968 + '\xa7f6'# -> unI64 42997 + '\xab53'# -> unI64 42931 + '\xab70'# -> unI64 5024 + '\xab71'# -> unI64 5025 + '\xab72'# -> unI64 5026 + '\xab73'# -> unI64 5027 + '\xab74'# -> unI64 5028 + '\xab75'# -> unI64 5029 + '\xab76'# -> unI64 5030 + '\xab77'# -> unI64 5031 + '\xab78'# -> unI64 5032 + '\xab79'# -> unI64 5033 + '\xab7a'# -> unI64 5034 + '\xab7b'# -> unI64 5035 + '\xab7c'# -> unI64 5036 + '\xab7d'# -> unI64 5037 + '\xab7e'# -> unI64 5038 + '\xab7f'# -> unI64 5039 + '\xab80'# -> unI64 5040 + '\xab81'# -> unI64 5041 + '\xab82'# -> unI64 5042 + '\xab83'# -> unI64 5043 + '\xab84'# -> unI64 5044 + '\xab85'# -> unI64 5045 + '\xab86'# -> unI64 5046 + '\xab87'# -> unI64 5047 + '\xab88'# -> unI64 5048 + '\xab89'# -> unI64 5049 + '\xab8a'# -> unI64 5050 + '\xab8b'# -> unI64 5051 + '\xab8c'# -> unI64 5052 + '\xab8d'# -> unI64 5053 + '\xab8e'# -> unI64 5054 + '\xab8f'# -> unI64 5055 + '\xab90'# -> unI64 5056 + '\xab91'# -> unI64 5057 + '\xab92'# -> unI64 5058 + '\xab93'# -> unI64 5059 + '\xab94'# -> unI64 5060 + '\xab95'# -> unI64 5061 + '\xab96'# -> unI64 5062 + '\xab97'# -> unI64 5063 + '\xab98'# -> unI64 5064 + '\xab99'# -> unI64 5065 + '\xab9a'# -> unI64 5066 + '\xab9b'# -> unI64 5067 + '\xab9c'# -> unI64 5068 + '\xab9d'# -> unI64 5069 + '\xab9e'# -> unI64 5070 + '\xab9f'# -> unI64 5071 + '\xaba0'# -> unI64 5072 + '\xaba1'# -> unI64 5073 + '\xaba2'# -> unI64 5074 + '\xaba3'# -> unI64 5075 + '\xaba4'# -> unI64 5076 + '\xaba5'# -> unI64 5077 + '\xaba6'# -> unI64 5078 + '\xaba7'# -> unI64 5079 + '\xaba8'# -> unI64 5080 + '\xaba9'# -> unI64 5081 + '\xabaa'# -> unI64 5082 + '\xabab'# -> unI64 5083 + '\xabac'# -> unI64 5084 + '\xabad'# -> unI64 5085 + '\xabae'# -> unI64 5086 + '\xabaf'# -> unI64 5087 + '\xabb0'# -> unI64 5088 + '\xabb1'# -> unI64 5089 + '\xabb2'# -> unI64 5090 + '\xabb3'# -> unI64 5091 + '\xabb4'# -> unI64 5092 + '\xabb5'# -> unI64 5093 + '\xabb6'# -> unI64 5094 + '\xabb7'# -> unI64 5095 + '\xabb8'# -> unI64 5096 + '\xabb9'# -> unI64 5097 + '\xabba'# -> unI64 5098 + '\xabbb'# -> unI64 5099 + '\xabbc'# -> unI64 5100 + '\xabbd'# -> unI64 5101 + '\xabbe'# -> unI64 5102 + '\xabbf'# -> unI64 5103 + '\xff41'# -> unI64 65313 + '\xff42'# -> unI64 65314 + '\xff43'# -> unI64 65315 + '\xff44'# -> unI64 65316 + '\xff45'# -> unI64 65317 + '\xff46'# -> unI64 65318 + '\xff47'# -> unI64 65319 + '\xff48'# -> unI64 65320 + '\xff49'# -> unI64 65321 + '\xff4a'# -> unI64 65322 + '\xff4b'# -> unI64 65323 + '\xff4c'# -> unI64 65324 + '\xff4d'# -> unI64 65325 + '\xff4e'# -> unI64 65326 + '\xff4f'# -> unI64 65327 + '\xff50'# -> unI64 65328 + '\xff51'# -> unI64 65329 + '\xff52'# -> unI64 65330 + '\xff53'# -> unI64 65331 + '\xff54'# -> unI64 65332 + '\xff55'# -> unI64 65333 + '\xff56'# -> unI64 65334 + '\xff57'# -> unI64 65335 + '\xff58'# -> unI64 65336 + '\xff59'# -> unI64 65337 + '\xff5a'# -> unI64 65338 + '\x10428'# -> unI64 66560 + '\x10429'# -> unI64 66561 + '\x1042a'# -> unI64 66562 + '\x1042b'# -> unI64 66563 + '\x1042c'# -> unI64 66564 + '\x1042d'# -> unI64 66565 + '\x1042e'# -> unI64 66566 + '\x1042f'# -> unI64 66567 + '\x10430'# -> unI64 66568 + '\x10431'# -> unI64 66569 + '\x10432'# -> unI64 66570 + '\x10433'# -> unI64 66571 + '\x10434'# -> unI64 66572 + '\x10435'# -> unI64 66573 + '\x10436'# -> unI64 66574 + '\x10437'# -> unI64 66575 + '\x10438'# -> unI64 66576 + '\x10439'# -> unI64 66577 + '\x1043a'# -> unI64 66578 + '\x1043b'# -> unI64 66579 + '\x1043c'# -> unI64 66580 + '\x1043d'# -> unI64 66581 + '\x1043e'# -> unI64 66582 + '\x1043f'# -> unI64 66583 + '\x10440'# -> unI64 66584 + '\x10441'# -> unI64 66585 + '\x10442'# -> unI64 66586 + '\x10443'# -> unI64 66587 + '\x10444'# -> unI64 66588 + '\x10445'# -> unI64 66589 + '\x10446'# -> unI64 66590 + '\x10447'# -> unI64 66591 + '\x10448'# -> unI64 66592 + '\x10449'# -> unI64 66593 + '\x1044a'# -> unI64 66594 + '\x1044b'# -> unI64 66595 + '\x1044c'# -> unI64 66596 + '\x1044d'# -> unI64 66597 + '\x1044e'# -> unI64 66598 + '\x1044f'# -> unI64 66599 + '\x104d8'# -> unI64 66736 + '\x104d9'# -> unI64 66737 + '\x104da'# -> unI64 66738 + '\x104db'# -> unI64 66739 + '\x104dc'# -> unI64 66740 + '\x104dd'# -> unI64 66741 + '\x104de'# -> unI64 66742 + '\x104df'# -> unI64 66743 + '\x104e0'# -> unI64 66744 + '\x104e1'# -> unI64 66745 + '\x104e2'# -> unI64 66746 + '\x104e3'# -> unI64 66747 + '\x104e4'# -> unI64 66748 + '\x104e5'# -> unI64 66749 + '\x104e6'# -> unI64 66750 + '\x104e7'# -> unI64 66751 + '\x104e8'# -> unI64 66752 + '\x104e9'# -> unI64 66753 + '\x104ea'# -> unI64 66754 + '\x104eb'# -> unI64 66755 + '\x104ec'# -> unI64 66756 + '\x104ed'# -> unI64 66757 + '\x104ee'# -> unI64 66758 + '\x104ef'# -> unI64 66759 + '\x104f0'# -> unI64 66760 + '\x104f1'# -> unI64 66761 + '\x104f2'# -> unI64 66762 + '\x104f3'# -> unI64 66763 + '\x104f4'# -> unI64 66764 + '\x104f5'# -> unI64 66765 + '\x104f6'# -> unI64 66766 + '\x104f7'# -> unI64 66767 + '\x104f8'# -> unI64 66768 + '\x104f9'# -> unI64 66769 + '\x104fa'# -> unI64 66770 + '\x104fb'# -> unI64 66771 + '\x10597'# -> unI64 66928 + '\x10598'# -> unI64 66929 + '\x10599'# -> unI64 66930 + '\x1059a'# -> unI64 66931 + '\x1059b'# -> unI64 66932 + '\x1059c'# -> unI64 66933 + '\x1059d'# -> unI64 66934 + '\x1059e'# -> unI64 66935 + '\x1059f'# -> unI64 66936 + '\x105a0'# -> unI64 66937 + '\x105a1'# -> unI64 66938 + '\x105a3'# -> unI64 66940 + '\x105a4'# -> unI64 66941 + '\x105a5'# -> unI64 66942 + '\x105a6'# -> unI64 66943 + '\x105a7'# -> unI64 66944 + '\x105a8'# -> unI64 66945 + '\x105a9'# -> unI64 66946 + '\x105aa'# -> unI64 66947 + '\x105ab'# -> unI64 66948 + '\x105ac'# -> unI64 66949 + '\x105ad'# -> unI64 66950 + '\x105ae'# -> unI64 66951 + '\x105af'# -> unI64 66952 + '\x105b0'# -> unI64 66953 + '\x105b1'# -> unI64 66954 + '\x105b3'# -> unI64 66956 + '\x105b4'# -> unI64 66957 + '\x105b5'# -> unI64 66958 + '\x105b6'# -> unI64 66959 + '\x105b7'# -> unI64 66960 + '\x105b8'# -> unI64 66961 + '\x105b9'# -> unI64 66962 + '\x105bb'# -> unI64 66964 + '\x105bc'# -> unI64 66965 + '\x10cc0'# -> unI64 68736 + '\x10cc1'# -> unI64 68737 + '\x10cc2'# -> unI64 68738 + '\x10cc3'# -> unI64 68739 + '\x10cc4'# -> unI64 68740 + '\x10cc5'# -> unI64 68741 + '\x10cc6'# -> unI64 68742 + '\x10cc7'# -> unI64 68743 + '\x10cc8'# -> unI64 68744 + '\x10cc9'# -> unI64 68745 + '\x10cca'# -> unI64 68746 + '\x10ccb'# -> unI64 68747 + '\x10ccc'# -> unI64 68748 + '\x10ccd'# -> unI64 68749 + '\x10cce'# -> unI64 68750 + '\x10ccf'# -> unI64 68751 + '\x10cd0'# -> unI64 68752 + '\x10cd1'# -> unI64 68753 + '\x10cd2'# -> unI64 68754 + '\x10cd3'# -> unI64 68755 + '\x10cd4'# -> unI64 68756 + '\x10cd5'# -> unI64 68757 + '\x10cd6'# -> unI64 68758 + '\x10cd7'# -> unI64 68759 + '\x10cd8'# -> unI64 68760 + '\x10cd9'# -> unI64 68761 + '\x10cda'# -> unI64 68762 + '\x10cdb'# -> unI64 68763 + '\x10cdc'# -> unI64 68764 + '\x10cdd'# -> unI64 68765 + '\x10cde'# -> unI64 68766 + '\x10cdf'# -> unI64 68767 + '\x10ce0'# -> unI64 68768 + '\x10ce1'# -> unI64 68769 + '\x10ce2'# -> unI64 68770 + '\x10ce3'# -> unI64 68771 + '\x10ce4'# -> unI64 68772 + '\x10ce5'# -> unI64 68773 + '\x10ce6'# -> unI64 68774 + '\x10ce7'# -> unI64 68775 + '\x10ce8'# -> unI64 68776 + '\x10ce9'# -> unI64 68777 + '\x10cea'# -> unI64 68778 + '\x10ceb'# -> unI64 68779 + '\x10cec'# -> unI64 68780 + '\x10ced'# -> unI64 68781 + '\x10cee'# -> unI64 68782 + '\x10cef'# -> unI64 68783 + '\x10cf0'# -> unI64 68784 + '\x10cf1'# -> unI64 68785 + '\x10cf2'# -> unI64 68786 + '\x118c0'# -> unI64 71840 + '\x118c1'# -> unI64 71841 + '\x118c2'# -> unI64 71842 + '\x118c3'# -> unI64 71843 + '\x118c4'# -> unI64 71844 + '\x118c5'# -> unI64 71845 + '\x118c6'# -> unI64 71846 + '\x118c7'# -> unI64 71847 + '\x118c8'# -> unI64 71848 + '\x118c9'# -> unI64 71849 + '\x118ca'# -> unI64 71850 + '\x118cb'# -> unI64 71851 + '\x118cc'# -> unI64 71852 + '\x118cd'# -> unI64 71853 + '\x118ce'# -> unI64 71854 + '\x118cf'# -> unI64 71855 + '\x118d0'# -> unI64 71856 + '\x118d1'# -> unI64 71857 + '\x118d2'# -> unI64 71858 + '\x118d3'# -> unI64 71859 + '\x118d4'# -> unI64 71860 + '\x118d5'# -> unI64 71861 + '\x118d6'# -> unI64 71862 + '\x118d7'# -> unI64 71863 + '\x118d8'# -> unI64 71864 + '\x118d9'# -> unI64 71865 + '\x118da'# -> unI64 71866 + '\x118db'# -> unI64 71867 + '\x118dc'# -> unI64 71868 + '\x118dd'# -> unI64 71869 + '\x118de'# -> unI64 71870 + '\x118df'# -> unI64 71871 + '\x16e60'# -> unI64 93760 + '\x16e61'# -> unI64 93761 + '\x16e62'# -> unI64 93762 + '\x16e63'# -> unI64 93763 + '\x16e64'# -> unI64 93764 + '\x16e65'# -> unI64 93765 + '\x16e66'# -> unI64 93766 + '\x16e67'# -> unI64 93767 + '\x16e68'# -> unI64 93768 + '\x16e69'# -> unI64 93769 + '\x16e6a'# -> unI64 93770 + '\x16e6b'# -> unI64 93771 + '\x16e6c'# -> unI64 93772 + '\x16e6d'# -> unI64 93773 + '\x16e6e'# -> unI64 93774 + '\x16e6f'# -> unI64 93775 + '\x16e70'# -> unI64 93776 + '\x16e71'# -> unI64 93777 + '\x16e72'# -> unI64 93778 + '\x16e73'# -> unI64 93779 + '\x16e74'# -> unI64 93780 + '\x16e75'# -> unI64 93781 + '\x16e76'# -> unI64 93782 + '\x16e77'# -> unI64 93783 + '\x16e78'# -> unI64 93784 + '\x16e79'# -> unI64 93785 + '\x16e7a'# -> unI64 93786 + '\x16e7b'# -> unI64 93787 + '\x16e7c'# -> unI64 93788 + '\x16e7d'# -> unI64 93789 + '\x16e7e'# -> unI64 93790 + '\x16e7f'# -> unI64 93791 + '\x1e922'# -> unI64 125184 + '\x1e923'# -> unI64 125185 + '\x1e924'# -> unI64 125186 + '\x1e925'# -> unI64 125187 + '\x1e926'# -> unI64 125188 + '\x1e927'# -> unI64 125189 + '\x1e928'# -> unI64 125190 + '\x1e929'# -> unI64 125191 + '\x1e92a'# -> unI64 125192 + '\x1e92b'# -> unI64 125193 + '\x1e92c'# -> unI64 125194 + '\x1e92d'# -> unI64 125195 + '\x1e92e'# -> unI64 125196 + '\x1e92f'# -> unI64 125197 + '\x1e930'# -> unI64 125198 + '\x1e931'# -> unI64 125199 + '\x1e932'# -> unI64 125200 + '\x1e933'# -> unI64 125201 + '\x1e934'# -> unI64 125202 + '\x1e935'# -> unI64 125203 + '\x1e936'# -> unI64 125204 + '\x1e937'# -> unI64 125205 + '\x1e938'# -> unI64 125206 + '\x1e939'# -> unI64 125207 + '\x1e93a'# -> unI64 125208 + '\x1e93b'# -> unI64 125209 + '\x1e93c'# -> unI64 125210 + '\x1e93d'# -> unI64 125211 + '\x1e93e'# -> unI64 125212 + '\x1e93f'# -> unI64 125213 + '\x1e940'# -> unI64 125214 + '\x1e941'# -> unI64 125215 + '\x1e942'# -> unI64 125216 + '\x1e943'# -> unI64 125217 + _ -> unI64 0 +foldMapping :: Char# -> _ {- unboxed Int64 -} +{-# NOINLINE foldMapping #-} +foldMapping = \case + -- MICRO SIGN + '\x00b5'# -> unI64 956 + -- LATIN SMALL LETTER SHARP S + '\x00df'# -> unI64 241172595 + -- LATIN CAPITAL LETTER I WITH DOT ABOVE + '\x0130'# -> unI64 1625292905 + -- LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + '\x0149'# -> unI64 230687420 + -- LATIN SMALL LETTER LONG S + '\x017f'# -> unI64 115 + -- LATIN SMALL LETTER J WITH CARON + '\x01f0'# -> unI64 1635778666 + -- COMBINING GREEK YPOGEGRAMMENI + '\x0345'# -> unI64 953 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + '\x0390'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + '\x03b0'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER FINAL SIGMA + '\x03c2'# -> unI64 963 + -- GREEK BETA SYMBOL + '\x03d0'# -> unI64 946 + -- GREEK THETA SYMBOL + '\x03d1'# -> unI64 952 + -- GREEK PHI SYMBOL + '\x03d5'# -> unI64 966 + -- GREEK PI SYMBOL + '\x03d6'# -> unI64 960 + -- GREEK KAPPA SYMBOL + '\x03f0'# -> unI64 954 + -- GREEK RHO SYMBOL + '\x03f1'# -> unI64 961 + -- GREEK LUNATE EPSILON SYMBOL + '\x03f5'# -> unI64 949 + -- ARMENIAN SMALL LIGATURE ECH YIWN + '\x0587'# -> unI64 2956985701 + -- CHEROKEE SMALL LETTER YE + '\x13f8'# -> unI64 5104 + -- CHEROKEE SMALL LETTER YI + '\x13f9'# -> unI64 5105 + -- CHEROKEE SMALL LETTER YO + '\x13fa'# -> unI64 5106 + -- CHEROKEE SMALL LETTER YU + '\x13fb'# -> unI64 5107 + -- CHEROKEE SMALL LETTER YV + '\x13fc'# -> unI64 5108 + -- CHEROKEE SMALL LETTER MV + '\x13fd'# -> unI64 5109 + -- CYRILLIC SMALL LETTER ROUNDED VE + '\x1c80'# -> unI64 1074 + -- CYRILLIC SMALL LETTER LONG-LEGGED DE + '\x1c81'# -> unI64 1076 + -- CYRILLIC SMALL LETTER NARROW O + '\x1c82'# -> unI64 1086 + -- CYRILLIC SMALL LETTER WIDE ES + '\x1c83'# -> unI64 1089 + -- CYRILLIC SMALL LETTER TALL TE + '\x1c84'# -> unI64 1090 + -- CYRILLIC SMALL LETTER THREE-LEGGED TE + '\x1c85'# -> unI64 1090 + -- CYRILLIC SMALL LETTER TALL HARD SIGN + '\x1c86'# -> unI64 1098 + -- CYRILLIC SMALL LETTER TALL YAT + '\x1c87'# -> unI64 1123 + -- CYRILLIC SMALL LETTER UNBLENDED UK + '\x1c88'# -> unI64 42571 + -- LATIN SMALL LETTER H WITH LINE BELOW + '\x1e96'# -> unI64 1713373288 + -- LATIN SMALL LETTER T WITH DIAERESIS + '\x1e97'# -> unI64 1627390068 + -- LATIN SMALL LETTER W WITH RING ABOVE + '\x1e98'# -> unI64 1631584375 + -- LATIN SMALL LETTER Y WITH RING ABOVE + '\x1e99'# -> unI64 1631584377 + -- LATIN SMALL LETTER A WITH RIGHT HALF RING + '\x1e9a'# -> unI64 1472200801 + -- LATIN SMALL LETTER LONG S WITH DOT ABOVE + '\x1e9b'# -> unI64 7777 + -- LATIN CAPITAL LETTER SHARP S + '\x1e9e'# -> unI64 241172595 + -- GREEK SMALL LETTER UPSILON WITH PSILI + '\x1f50'# -> unI64 1650459589 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + '\x1f52'# -> unI64 3377701370987461 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + '\x1f54'# -> unI64 3382099417498565 + -- GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + '\x1f56'# -> unI64 3667972440720325 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + '\x1f80'# -> unI64 1998593792 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + '\x1f81'# -> unI64 1998593793 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f82'# -> unI64 1998593794 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f83'# -> unI64 1998593795 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f84'# -> unI64 1998593796 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f85'# -> unI64 1998593797 + -- GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f86'# -> unI64 1998593798 + -- GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f87'# -> unI64 1998593799 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + '\x1f88'# -> unI64 1998593792 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + '\x1f89'# -> unI64 1998593793 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f8a'# -> unI64 1998593794 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f8b'# -> unI64 1998593795 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f8c'# -> unI64 1998593796 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f8d'# -> unI64 1998593797 + -- GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8e'# -> unI64 1998593798 + -- GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f8f'# -> unI64 1998593799 + -- GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + '\x1f90'# -> unI64 1998593824 + -- GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + '\x1f91'# -> unI64 1998593825 + -- GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1f92'# -> unI64 1998593826 + -- GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1f93'# -> unI64 1998593827 + -- GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1f94'# -> unI64 1998593828 + -- GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1f95'# -> unI64 1998593829 + -- GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f96'# -> unI64 1998593830 + -- GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1f97'# -> unI64 1998593831 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + '\x1f98'# -> unI64 1998593824 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + '\x1f99'# -> unI64 1998593825 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1f9a'# -> unI64 1998593826 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1f9b'# -> unI64 1998593827 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1f9c'# -> unI64 1998593828 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1f9d'# -> unI64 1998593829 + -- GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9e'# -> unI64 1998593830 + -- GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1f9f'# -> unI64 1998593831 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + '\x1fa0'# -> unI64 1998593888 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + '\x1fa1'# -> unI64 1998593889 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + '\x1fa2'# -> unI64 1998593890 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + '\x1fa3'# -> unI64 1998593891 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + '\x1fa4'# -> unI64 1998593892 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + '\x1fa5'# -> unI64 1998593893 + -- GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa6'# -> unI64 1998593894 + -- GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + '\x1fa7'# -> unI64 1998593895 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + '\x1fa8'# -> unI64 1998593888 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + '\x1fa9'# -> unI64 1998593889 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + '\x1faa'# -> unI64 1998593890 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + '\x1fab'# -> unI64 1998593891 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + '\x1fac'# -> unI64 1998593892 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + '\x1fad'# -> unI64 1998593893 + -- GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + '\x1fae'# -> unI64 1998593894 + -- GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + '\x1faf'# -> unI64 1998593895 + -- GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + '\x1fb2'# -> unI64 1998593904 + -- GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + '\x1fb3'# -> unI64 1998586801 + -- GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + '\x1fb4'# -> unI64 1998586796 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI + '\x1fb6'# -> unI64 1749025713 + -- GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fb7'# -> unI64 4191340074107825 + -- GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + '\x1fbc'# -> unI64 1998586801 + -- GREEK PROSGEGRAMMENI + '\x1fbe'# -> unI64 953 + -- GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + '\x1fc2'# -> unI64 1998593908 + -- GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + '\x1fc3'# -> unI64 1998586807 + -- GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + '\x1fc4'# -> unI64 1998586798 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI + '\x1fc6'# -> unI64 1749025719 + -- GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1fc7'# -> unI64 4191340074107831 + -- GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + '\x1fcc'# -> unI64 1998586807 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + '\x1fd2'# -> unI64 3377701347918777 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + '\x1fd3'# -> unI64 3382099394429881 + -- GREEK SMALL LETTER IOTA WITH PERISPOMENI + '\x1fd6'# -> unI64 1749025721 + -- GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + '\x1fd7'# -> unI64 3667972417651641 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + '\x1fe2'# -> unI64 3377701347918789 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + '\x1fe3'# -> unI64 3382099394429893 + -- GREEK SMALL LETTER RHO WITH PSILI + '\x1fe4'# -> unI64 1650459585 + -- GREEK SMALL LETTER UPSILON WITH PERISPOMENI + '\x1fe6'# -> unI64 1749025733 + -- GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + '\x1fe7'# -> unI64 3667972417651653 + -- GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + '\x1ff2'# -> unI64 1998593916 + -- GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + '\x1ff3'# -> unI64 1998586825 + -- GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + '\x1ff4'# -> unI64 1998586830 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI + '\x1ff6'# -> unI64 1749025737 + -- GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + '\x1ff7'# -> unI64 4191340074107849 + -- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + '\x1ffc'# -> unI64 1998586825 + -- CHEROKEE SMALL LETTER A + '\xab70'# -> unI64 5024 + -- CHEROKEE SMALL LETTER E + '\xab71'# -> unI64 5025 + -- CHEROKEE SMALL LETTER I + '\xab72'# -> unI64 5026 + -- CHEROKEE SMALL LETTER O + '\xab73'# -> unI64 5027 + -- CHEROKEE SMALL LETTER U + '\xab74'# -> unI64 5028 + -- CHEROKEE SMALL LETTER V + '\xab75'# -> unI64 5029 + -- CHEROKEE SMALL LETTER GA + '\xab76'# -> unI64 5030 + -- CHEROKEE SMALL LETTER KA + '\xab77'# -> unI64 5031 + -- CHEROKEE SMALL LETTER GE + '\xab78'# -> unI64 5032 + -- CHEROKEE SMALL LETTER GI + '\xab79'# -> unI64 5033 + -- CHEROKEE SMALL LETTER GO + '\xab7a'# -> unI64 5034 + -- CHEROKEE SMALL LETTER GU + '\xab7b'# -> unI64 5035 + -- CHEROKEE SMALL LETTER GV + '\xab7c'# -> unI64 5036 + -- CHEROKEE SMALL LETTER HA + '\xab7d'# -> unI64 5037 + -- CHEROKEE SMALL LETTER HE + '\xab7e'# -> unI64 5038 + -- CHEROKEE SMALL LETTER HI + '\xab7f'# -> unI64 5039 + -- CHEROKEE SMALL LETTER HO + '\xab80'# -> unI64 5040 + -- CHEROKEE SMALL LETTER HU + '\xab81'# -> unI64 5041 + -- CHEROKEE SMALL LETTER HV + '\xab82'# -> unI64 5042 + -- CHEROKEE SMALL LETTER LA + '\xab83'# -> unI64 5043 + -- CHEROKEE SMALL LETTER LE + '\xab84'# -> unI64 5044 + -- CHEROKEE SMALL LETTER LI + '\xab85'# -> unI64 5045 + -- CHEROKEE SMALL LETTER LO + '\xab86'# -> unI64 5046 + -- CHEROKEE SMALL LETTER LU + '\xab87'# -> unI64 5047 + -- CHEROKEE SMALL LETTER LV + '\xab88'# -> unI64 5048 + -- CHEROKEE SMALL LETTER MA + '\xab89'# -> unI64 5049 + -- CHEROKEE SMALL LETTER ME + '\xab8a'# -> unI64 5050 + -- CHEROKEE SMALL LETTER MI + '\xab8b'# -> unI64 5051 + -- CHEROKEE SMALL LETTER MO + '\xab8c'# -> unI64 5052 + -- CHEROKEE SMALL LETTER MU + '\xab8d'# -> unI64 5053 + -- CHEROKEE SMALL LETTER NA + '\xab8e'# -> unI64 5054 + -- CHEROKEE SMALL LETTER HNA + '\xab8f'# -> unI64 5055 + -- CHEROKEE SMALL LETTER NAH + '\xab90'# -> unI64 5056 + -- CHEROKEE SMALL LETTER NE + '\xab91'# -> unI64 5057 + -- CHEROKEE SMALL LETTER NI + '\xab92'# -> unI64 5058 + -- CHEROKEE SMALL LETTER NO + '\xab93'# -> unI64 5059 + -- CHEROKEE SMALL LETTER NU + '\xab94'# -> unI64 5060 + -- CHEROKEE SMALL LETTER NV + '\xab95'# -> unI64 5061 + -- CHEROKEE SMALL LETTER QUA + '\xab96'# -> unI64 5062 + -- CHEROKEE SMALL LETTER QUE + '\xab97'# -> unI64 5063 + -- CHEROKEE SMALL LETTER QUI + '\xab98'# -> unI64 5064 + -- CHEROKEE SMALL LETTER QUO + '\xab99'# -> unI64 5065 + -- CHEROKEE SMALL LETTER QUU + '\xab9a'# -> unI64 5066 + -- CHEROKEE SMALL LETTER QUV + '\xab9b'# -> unI64 5067 + -- CHEROKEE SMALL LETTER SA + '\xab9c'# -> unI64 5068 + -- CHEROKEE SMALL LETTER S + '\xab9d'# -> unI64 5069 + -- CHEROKEE SMALL LETTER SE + '\xab9e'# -> unI64 5070 + -- CHEROKEE SMALL LETTER SI + '\xab9f'# -> unI64 5071 + -- CHEROKEE SMALL LETTER SO + '\xaba0'# -> unI64 5072 + -- CHEROKEE SMALL LETTER SU + '\xaba1'# -> unI64 5073 + -- CHEROKEE SMALL LETTER SV + '\xaba2'# -> unI64 5074 + -- CHEROKEE SMALL LETTER DA + '\xaba3'# -> unI64 5075 + -- CHEROKEE SMALL LETTER TA + '\xaba4'# -> unI64 5076 + -- CHEROKEE SMALL LETTER DE + '\xaba5'# -> unI64 5077 + -- CHEROKEE SMALL LETTER TE + '\xaba6'# -> unI64 5078 + -- CHEROKEE SMALL LETTER DI + '\xaba7'# -> unI64 5079 + -- CHEROKEE SMALL LETTER TI + '\xaba8'# -> unI64 5080 + -- CHEROKEE SMALL LETTER DO + '\xaba9'# -> unI64 5081 + -- CHEROKEE SMALL LETTER DU + '\xabaa'# -> unI64 5082 + -- CHEROKEE SMALL LETTER DV + '\xabab'# -> unI64 5083 + -- CHEROKEE SMALL LETTER DLA + '\xabac'# -> unI64 5084 + -- CHEROKEE SMALL LETTER TLA + '\xabad'# -> unI64 5085 + -- CHEROKEE SMALL LETTER TLE + '\xabae'# -> unI64 5086 + -- CHEROKEE SMALL LETTER TLI + '\xabaf'# -> unI64 5087 + -- CHEROKEE SMALL LETTER TLO + '\xabb0'# -> unI64 5088 + -- CHEROKEE SMALL LETTER TLU + '\xabb1'# -> unI64 5089 + -- CHEROKEE SMALL LETTER TLV + '\xabb2'# -> unI64 5090 + -- CHEROKEE SMALL LETTER TSA + '\xabb3'# -> unI64 5091 + -- CHEROKEE SMALL LETTER TSE + '\xabb4'# -> unI64 5092 + -- CHEROKEE SMALL LETTER TSI + '\xabb5'# -> unI64 5093 + -- CHEROKEE SMALL LETTER TSO + '\xabb6'# -> unI64 5094 + -- CHEROKEE SMALL LETTER TSU + '\xabb7'# -> unI64 5095 + -- CHEROKEE SMALL LETTER TSV + '\xabb8'# -> unI64 5096 + -- CHEROKEE SMALL LETTER WA + '\xabb9'# -> unI64 5097 + -- CHEROKEE SMALL LETTER WE + '\xabba'# -> unI64 5098 + -- CHEROKEE SMALL LETTER WI + '\xabbb'# -> unI64 5099 + -- CHEROKEE SMALL LETTER WO + '\xabbc'# -> unI64 5100 + -- CHEROKEE SMALL LETTER WU + '\xabbd'# -> unI64 5101 + -- CHEROKEE SMALL LETTER WV + '\xabbe'# -> unI64 5102 + -- CHEROKEE SMALL LETTER YA + '\xabbf'# -> unI64 5103 + -- LATIN SMALL LIGATURE FF + '\xfb00'# -> unI64 213909606 + -- LATIN SMALL LIGATURE FI + '\xfb01'# -> unI64 220201062 + -- LATIN SMALL LIGATURE FL + '\xfb02'# -> unI64 226492518 + -- LATIN SMALL LIGATURE FFI + '\xfb03'# -> unI64 461795097575526 + -- LATIN SMALL LIGATURE FFL + '\xfb04'# -> unI64 474989237108838 + -- LATIN SMALL LIGATURE LONG S T + '\xfb05'# -> unI64 243269747 + -- LATIN SMALL LIGATURE ST + '\xfb06'# -> unI64 243269747 + -- ARMENIAN SMALL LIGATURE MEN NOW + '\xfb13'# -> unI64 2931819892 + -- ARMENIAN SMALL LIGATURE MEN ECH + '\xfb14'# -> unI64 2896168308 + -- ARMENIAN SMALL LIGATURE MEN INI + '\xfb15'# -> unI64 2908751220 + -- ARMENIAN SMALL LIGATURE VEW NOW + '\xfb16'# -> unI64 2931819902 + -- ARMENIAN SMALL LIGATURE MEN XEH + '\xfb17'# -> unI64 2912945524 + '\x0041'# -> unI64 97 + '\x0042'# -> unI64 98 + '\x0043'# -> unI64 99 + '\x0044'# -> unI64 100 + '\x0045'# -> unI64 101 + '\x0046'# -> unI64 102 + '\x0047'# -> unI64 103 + '\x0048'# -> unI64 104 + '\x0049'# -> unI64 105 + '\x004a'# -> unI64 106 + '\x004b'# -> unI64 107 + '\x004c'# -> unI64 108 + '\x004d'# -> unI64 109 + '\x004e'# -> unI64 110 + '\x004f'# -> unI64 111 + '\x0050'# -> unI64 112 + '\x0051'# -> unI64 113 + '\x0052'# -> unI64 114 + '\x0053'# -> unI64 115 + '\x0054'# -> unI64 116 + '\x0055'# -> unI64 117 + '\x0056'# -> unI64 118 + '\x0057'# -> unI64 119 + '\x0058'# -> unI64 120 + '\x0059'# -> unI64 121 + '\x005a'# -> unI64 122 + '\x00c0'# -> unI64 224 + '\x00c1'# -> unI64 225 + '\x00c2'# -> unI64 226 + '\x00c3'# -> unI64 227 + '\x00c4'# -> unI64 228 + '\x00c5'# -> unI64 229 + '\x00c6'# -> unI64 230 + '\x00c7'# -> unI64 231 + '\x00c8'# -> unI64 232 + '\x00c9'# -> unI64 233 + '\x00ca'# -> unI64 234 + '\x00cb'# -> unI64 235 + '\x00cc'# -> unI64 236 + '\x00cd'# -> unI64 237 + '\x00ce'# -> unI64 238 + '\x00cf'# -> unI64 239 + '\x00d0'# -> unI64 240 + '\x00d1'# -> unI64 241 + '\x00d2'# -> unI64 242 + '\x00d3'# -> unI64 243 + '\x00d4'# -> unI64 244 + '\x00d5'# -> unI64 245 + '\x00d6'# -> unI64 246 + '\x00d8'# -> unI64 248 + '\x00d9'# -> unI64 249 + '\x00da'# -> unI64 250 + '\x00db'# -> unI64 251 + '\x00dc'# -> unI64 252 + '\x00dd'# -> unI64 253 + '\x00de'# -> unI64 254 + '\x0100'# -> unI64 257 + '\x0102'# -> unI64 259 + '\x0104'# -> unI64 261 + '\x0106'# -> unI64 263 + '\x0108'# -> unI64 265 + '\x010a'# -> unI64 267 + '\x010c'# -> unI64 269 + '\x010e'# -> unI64 271 + '\x0110'# -> unI64 273 + '\x0112'# -> unI64 275 + '\x0114'# -> unI64 277 + '\x0116'# -> unI64 279 + '\x0118'# -> unI64 281 + '\x011a'# -> unI64 283 + '\x011c'# -> unI64 285 + '\x011e'# -> unI64 287 + '\x0120'# -> unI64 289 + '\x0122'# -> unI64 291 + '\x0124'# -> unI64 293 + '\x0126'# -> unI64 295 + '\x0128'# -> unI64 297 + '\x012a'# -> unI64 299 + '\x012c'# -> unI64 301 + '\x012e'# -> unI64 303 + '\x0132'# -> unI64 307 + '\x0134'# -> unI64 309 + '\x0136'# -> unI64 311 + '\x0139'# -> unI64 314 + '\x013b'# -> unI64 316 + '\x013d'# -> unI64 318 + '\x013f'# -> unI64 320 + '\x0141'# -> unI64 322 + '\x0143'# -> unI64 324 + '\x0145'# -> unI64 326 + '\x0147'# -> unI64 328 + '\x014a'# -> unI64 331 + '\x014c'# -> unI64 333 + '\x014e'# -> unI64 335 + '\x0150'# -> unI64 337 + '\x0152'# -> unI64 339 + '\x0154'# -> unI64 341 + '\x0156'# -> unI64 343 + '\x0158'# -> unI64 345 + '\x015a'# -> unI64 347 + '\x015c'# -> unI64 349 + '\x015e'# -> unI64 351 + '\x0160'# -> unI64 353 + '\x0162'# -> unI64 355 + '\x0164'# -> unI64 357 + '\x0166'# -> unI64 359 + '\x0168'# -> unI64 361 + '\x016a'# -> unI64 363 + '\x016c'# -> unI64 365 + '\x016e'# -> unI64 367 + '\x0170'# -> unI64 369 + '\x0172'# -> unI64 371 + '\x0174'# -> unI64 373 + '\x0176'# -> unI64 375 + '\x0178'# -> unI64 255 + '\x0179'# -> unI64 378 + '\x017b'# -> unI64 380 + '\x017d'# -> unI64 382 + '\x0181'# -> unI64 595 + '\x0182'# -> unI64 387 + '\x0184'# -> unI64 389 + '\x0186'# -> unI64 596 + '\x0187'# -> unI64 392 + '\x0189'# -> unI64 598 + '\x018a'# -> unI64 599 + '\x018b'# -> unI64 396 + '\x018e'# -> unI64 477 + '\x018f'# -> unI64 601 + '\x0190'# -> unI64 603 + '\x0191'# -> unI64 402 + '\x0193'# -> unI64 608 + '\x0194'# -> unI64 611 + '\x0196'# -> unI64 617 + '\x0197'# -> unI64 616 + '\x0198'# -> unI64 409 + '\x019c'# -> unI64 623 + '\x019d'# -> unI64 626 + '\x019f'# -> unI64 629 + '\x01a0'# -> unI64 417 + '\x01a2'# -> unI64 419 + '\x01a4'# -> unI64 421 + '\x01a6'# -> unI64 640 + '\x01a7'# -> unI64 424 + '\x01a9'# -> unI64 643 + '\x01ac'# -> unI64 429 + '\x01ae'# -> unI64 648 + '\x01af'# -> unI64 432 + '\x01b1'# -> unI64 650 + '\x01b2'# -> unI64 651 + '\x01b3'# -> unI64 436 + '\x01b5'# -> unI64 438 + '\x01b7'# -> unI64 658 + '\x01b8'# -> unI64 441 + '\x01bc'# -> unI64 445 + '\x01c4'# -> unI64 454 + '\x01c5'# -> unI64 454 + '\x01c7'# -> unI64 457 + '\x01c8'# -> unI64 457 + '\x01ca'# -> unI64 460 + '\x01cb'# -> unI64 460 + '\x01cd'# -> unI64 462 + '\x01cf'# -> unI64 464 + '\x01d1'# -> unI64 466 + '\x01d3'# -> unI64 468 + '\x01d5'# -> unI64 470 + '\x01d7'# -> unI64 472 + '\x01d9'# -> unI64 474 + '\x01db'# -> unI64 476 + '\x01de'# -> unI64 479 + '\x01e0'# -> unI64 481 + '\x01e2'# -> unI64 483 + '\x01e4'# -> unI64 485 + '\x01e6'# -> unI64 487 + '\x01e8'# -> unI64 489 + '\x01ea'# -> unI64 491 + '\x01ec'# -> unI64 493 + '\x01ee'# -> unI64 495 + '\x01f1'# -> unI64 499 + '\x01f2'# -> unI64 499 + '\x01f4'# -> unI64 501 + '\x01f6'# -> unI64 405 + '\x01f7'# -> unI64 447 + '\x01f8'# -> unI64 505 + '\x01fa'# -> unI64 507 + '\x01fc'# -> unI64 509 + '\x01fe'# -> unI64 511 + '\x0200'# -> unI64 513 + '\x0202'# -> unI64 515 + '\x0204'# -> unI64 517 + '\x0206'# -> unI64 519 + '\x0208'# -> unI64 521 + '\x020a'# -> unI64 523 + '\x020c'# -> unI64 525 + '\x020e'# -> unI64 527 + '\x0210'# -> unI64 529 + '\x0212'# -> unI64 531 + '\x0214'# -> unI64 533 + '\x0216'# -> unI64 535 + '\x0218'# -> unI64 537 + '\x021a'# -> unI64 539 + '\x021c'# -> unI64 541 + '\x021e'# -> unI64 543 + '\x0220'# -> unI64 414 + '\x0222'# -> unI64 547 + '\x0224'# -> unI64 549 + '\x0226'# -> unI64 551 + '\x0228'# -> unI64 553 + '\x022a'# -> unI64 555 + '\x022c'# -> unI64 557 + '\x022e'# -> unI64 559 + '\x0230'# -> unI64 561 + '\x0232'# -> unI64 563 + '\x023a'# -> unI64 11365 + '\x023b'# -> unI64 572 + '\x023d'# -> unI64 410 + '\x023e'# -> unI64 11366 + '\x0241'# -> unI64 578 + '\x0243'# -> unI64 384 + '\x0244'# -> unI64 649 + '\x0245'# -> unI64 652 + '\x0246'# -> unI64 583 + '\x0248'# -> unI64 585 + '\x024a'# -> unI64 587 + '\x024c'# -> unI64 589 + '\x024e'# -> unI64 591 + '\x0370'# -> unI64 881 + '\x0372'# -> unI64 883 + '\x0376'# -> unI64 887 + '\x037f'# -> unI64 1011 + '\x0386'# -> unI64 940 + '\x0388'# -> unI64 941 + '\x0389'# -> unI64 942 + '\x038a'# -> unI64 943 + '\x038c'# -> unI64 972 + '\x038e'# -> unI64 973 + '\x038f'# -> unI64 974 + '\x0391'# -> unI64 945 + '\x0392'# -> unI64 946 + '\x0393'# -> unI64 947 + '\x0394'# -> unI64 948 + '\x0395'# -> unI64 949 + '\x0396'# -> unI64 950 + '\x0397'# -> unI64 951 + '\x0398'# -> unI64 952 + '\x0399'# -> unI64 953 + '\x039a'# -> unI64 954 + '\x039b'# -> unI64 955 + '\x039c'# -> unI64 956 + '\x039d'# -> unI64 957 + '\x039e'# -> unI64 958 + '\x039f'# -> unI64 959 + '\x03a0'# -> unI64 960 + '\x03a1'# -> unI64 961 + '\x03a3'# -> unI64 963 + '\x03a4'# -> unI64 964 + '\x03a5'# -> unI64 965 + '\x03a6'# -> unI64 966 + '\x03a7'# -> unI64 967 + '\x03a8'# -> unI64 968 + '\x03a9'# -> unI64 969 + '\x03aa'# -> unI64 970 + '\x03ab'# -> unI64 971 + '\x03cf'# -> unI64 983 + '\x03d8'# -> unI64 985 + '\x03da'# -> unI64 987 + '\x03dc'# -> unI64 989 + '\x03de'# -> unI64 991 + '\x03e0'# -> unI64 993 + '\x03e2'# -> unI64 995 + '\x03e4'# -> unI64 997 + '\x03e6'# -> unI64 999 + '\x03e8'# -> unI64 1001 + '\x03ea'# -> unI64 1003 + '\x03ec'# -> unI64 1005 + '\x03ee'# -> unI64 1007 + '\x03f4'# -> unI64 952 + '\x03f7'# -> unI64 1016 + '\x03f9'# -> unI64 1010 + '\x03fa'# -> unI64 1019 + '\x03fd'# -> unI64 891 + '\x03fe'# -> unI64 892 + '\x03ff'# -> unI64 893 + '\x0400'# -> unI64 1104 + '\x0401'# -> unI64 1105 + '\x0402'# -> unI64 1106 + '\x0403'# -> unI64 1107 + '\x0404'# -> unI64 1108 + '\x0405'# -> unI64 1109 + '\x0406'# -> unI64 1110 + '\x0407'# -> unI64 1111 + '\x0408'# -> unI64 1112 + '\x0409'# -> unI64 1113 + '\x040a'# -> unI64 1114 + '\x040b'# -> unI64 1115 + '\x040c'# -> unI64 1116 + '\x040d'# -> unI64 1117 + '\x040e'# -> unI64 1118 + '\x040f'# -> unI64 1119 + '\x0410'# -> unI64 1072 + '\x0411'# -> unI64 1073 + '\x0412'# -> unI64 1074 + '\x0413'# -> unI64 1075 + '\x0414'# -> unI64 1076 + '\x0415'# -> unI64 1077 + '\x0416'# -> unI64 1078 + '\x0417'# -> unI64 1079 + '\x0418'# -> unI64 1080 + '\x0419'# -> unI64 1081 + '\x041a'# -> unI64 1082 + '\x041b'# -> unI64 1083 + '\x041c'# -> unI64 1084 + '\x041d'# -> unI64 1085 + '\x041e'# -> unI64 1086 + '\x041f'# -> unI64 1087 + '\x0420'# -> unI64 1088 + '\x0421'# -> unI64 1089 + '\x0422'# -> unI64 1090 + '\x0423'# -> unI64 1091 + '\x0424'# -> unI64 1092 + '\x0425'# -> unI64 1093 + '\x0426'# -> unI64 1094 + '\x0427'# -> unI64 1095 + '\x0428'# -> unI64 1096 + '\x0429'# -> unI64 1097 + '\x042a'# -> unI64 1098 + '\x042b'# -> unI64 1099 + '\x042c'# -> unI64 1100 + '\x042d'# -> unI64 1101 + '\x042e'# -> unI64 1102 + '\x042f'# -> unI64 1103 + '\x0460'# -> unI64 1121 + '\x0462'# -> unI64 1123 + '\x0464'# -> unI64 1125 + '\x0466'# -> unI64 1127 + '\x0468'# -> unI64 1129 + '\x046a'# -> unI64 1131 + '\x046c'# -> unI64 1133 + '\x046e'# -> unI64 1135 + '\x0470'# -> unI64 1137 + '\x0472'# -> unI64 1139 + '\x0474'# -> unI64 1141 + '\x0476'# -> unI64 1143 + '\x0478'# -> unI64 1145 + '\x047a'# -> unI64 1147 + '\x047c'# -> unI64 1149 + '\x047e'# -> unI64 1151 + '\x0480'# -> unI64 1153 + '\x048a'# -> unI64 1163 + '\x048c'# -> unI64 1165 + '\x048e'# -> unI64 1167 + '\x0490'# -> unI64 1169 + '\x0492'# -> unI64 1171 + '\x0494'# -> unI64 1173 + '\x0496'# -> unI64 1175 + '\x0498'# -> unI64 1177 + '\x049a'# -> unI64 1179 + '\x049c'# -> unI64 1181 + '\x049e'# -> unI64 1183 + '\x04a0'# -> unI64 1185 + '\x04a2'# -> unI64 1187 + '\x04a4'# -> unI64 1189 + '\x04a6'# -> unI64 1191 + '\x04a8'# -> unI64 1193 + '\x04aa'# -> unI64 1195 + '\x04ac'# -> unI64 1197 + '\x04ae'# -> unI64 1199 + '\x04b0'# -> unI64 1201 + '\x04b2'# -> unI64 1203 + '\x04b4'# -> unI64 1205 + '\x04b6'# -> unI64 1207 + '\x04b8'# -> unI64 1209 + '\x04ba'# -> unI64 1211 + '\x04bc'# -> unI64 1213 + '\x04be'# -> unI64 1215 + '\x04c0'# -> unI64 1231 + '\x04c1'# -> unI64 1218 + '\x04c3'# -> unI64 1220 + '\x04c5'# -> unI64 1222 + '\x04c7'# -> unI64 1224 + '\x04c9'# -> unI64 1226 + '\x04cb'# -> unI64 1228 + '\x04cd'# -> unI64 1230 + '\x04d0'# -> unI64 1233 + '\x04d2'# -> unI64 1235 + '\x04d4'# -> unI64 1237 + '\x04d6'# -> unI64 1239 + '\x04d8'# -> unI64 1241 + '\x04da'# -> unI64 1243 + '\x04dc'# -> unI64 1245 + '\x04de'# -> unI64 1247 + '\x04e0'# -> unI64 1249 + '\x04e2'# -> unI64 1251 + '\x04e4'# -> unI64 1253 + '\x04e6'# -> unI64 1255 + '\x04e8'# -> unI64 1257 + '\x04ea'# -> unI64 1259 + '\x04ec'# -> unI64 1261 + '\x04ee'# -> unI64 1263 + '\x04f0'# -> unI64 1265 + '\x04f2'# -> unI64 1267 + '\x04f4'# -> unI64 1269 + '\x04f6'# -> unI64 1271 + '\x04f8'# -> unI64 1273 + '\x04fa'# -> unI64 1275 + '\x04fc'# -> unI64 1277 + '\x04fe'# -> unI64 1279 + '\x0500'# -> unI64 1281 + '\x0502'# -> unI64 1283 + '\x0504'# -> unI64 1285 + '\x0506'# -> unI64 1287 + '\x0508'# -> unI64 1289 + '\x050a'# -> unI64 1291 + '\x050c'# -> unI64 1293 + '\x050e'# -> unI64 1295 + '\x0510'# -> unI64 1297 + '\x0512'# -> unI64 1299 + '\x0514'# -> unI64 1301 + '\x0516'# -> unI64 1303 + '\x0518'# -> unI64 1305 + '\x051a'# -> unI64 1307 + '\x051c'# -> unI64 1309 + '\x051e'# -> unI64 1311 + '\x0520'# -> unI64 1313 + '\x0522'# -> unI64 1315 + '\x0524'# -> unI64 1317 + '\x0526'# -> unI64 1319 + '\x0528'# -> unI64 1321 + '\x052a'# -> unI64 1323 + '\x052c'# -> unI64 1325 + '\x052e'# -> unI64 1327 + '\x0531'# -> unI64 1377 + '\x0532'# -> unI64 1378 + '\x0533'# -> unI64 1379 + '\x0534'# -> unI64 1380 + '\x0535'# -> unI64 1381 + '\x0536'# -> unI64 1382 + '\x0537'# -> unI64 1383 + '\x0538'# -> unI64 1384 + '\x0539'# -> unI64 1385 + '\x053a'# -> unI64 1386 + '\x053b'# -> unI64 1387 + '\x053c'# -> unI64 1388 + '\x053d'# -> unI64 1389 + '\x053e'# -> unI64 1390 + '\x053f'# -> unI64 1391 + '\x0540'# -> unI64 1392 + '\x0541'# -> unI64 1393 + '\x0542'# -> unI64 1394 + '\x0543'# -> unI64 1395 + '\x0544'# -> unI64 1396 + '\x0545'# -> unI64 1397 + '\x0546'# -> unI64 1398 + '\x0547'# -> unI64 1399 + '\x0548'# -> unI64 1400 + '\x0549'# -> unI64 1401 + '\x054a'# -> unI64 1402 + '\x054b'# -> unI64 1403 + '\x054c'# -> unI64 1404 + '\x054d'# -> unI64 1405 + '\x054e'# -> unI64 1406 + '\x054f'# -> unI64 1407 + '\x0550'# -> unI64 1408 + '\x0551'# -> unI64 1409 + '\x0552'# -> unI64 1410 + '\x0553'# -> unI64 1411 + '\x0554'# -> unI64 1412 + '\x0555'# -> unI64 1413 + '\x0556'# -> unI64 1414 + '\x10a0'# -> unI64 11520 + '\x10a1'# -> unI64 11521 + '\x10a2'# -> unI64 11522 + '\x10a3'# -> unI64 11523 + '\x10a4'# -> unI64 11524 + '\x10a5'# -> unI64 11525 + '\x10a6'# -> unI64 11526 + '\x10a7'# -> unI64 11527 + '\x10a8'# -> unI64 11528 + '\x10a9'# -> unI64 11529 + '\x10aa'# -> unI64 11530 + '\x10ab'# -> unI64 11531 + '\x10ac'# -> unI64 11532 + '\x10ad'# -> unI64 11533 + '\x10ae'# -> unI64 11534 + '\x10af'# -> unI64 11535 + '\x10b0'# -> unI64 11536 + '\x10b1'# -> unI64 11537 + '\x10b2'# -> unI64 11538 + '\x10b3'# -> unI64 11539 + '\x10b4'# -> unI64 11540 + '\x10b5'# -> unI64 11541 + '\x10b6'# -> unI64 11542 + '\x10b7'# -> unI64 11543 + '\x10b8'# -> unI64 11544 + '\x10b9'# -> unI64 11545 + '\x10ba'# -> unI64 11546 + '\x10bb'# -> unI64 11547 + '\x10bc'# -> unI64 11548 + '\x10bd'# -> unI64 11549 + '\x10be'# -> unI64 11550 + '\x10bf'# -> unI64 11551 + '\x10c0'# -> unI64 11552 + '\x10c1'# -> unI64 11553 + '\x10c2'# -> unI64 11554 + '\x10c3'# -> unI64 11555 + '\x10c4'# -> unI64 11556 + '\x10c5'# -> unI64 11557 + '\x10c7'# -> unI64 11559 + '\x10cd'# -> unI64 11565 + '\x13a0'# -> unI64 43888 + '\x13a1'# -> unI64 43889 + '\x13a2'# -> unI64 43890 + '\x13a3'# -> unI64 43891 + '\x13a4'# -> unI64 43892 + '\x13a5'# -> unI64 43893 + '\x13a6'# -> unI64 43894 + '\x13a7'# -> unI64 43895 + '\x13a8'# -> unI64 43896 + '\x13a9'# -> unI64 43897 + '\x13aa'# -> unI64 43898 + '\x13ab'# -> unI64 43899 + '\x13ac'# -> unI64 43900 + '\x13ad'# -> unI64 43901 + '\x13ae'# -> unI64 43902 + '\x13af'# -> unI64 43903 + '\x13b0'# -> unI64 43904 + '\x13b1'# -> unI64 43905 + '\x13b2'# -> unI64 43906 + '\x13b3'# -> unI64 43907 + '\x13b4'# -> unI64 43908 + '\x13b5'# -> unI64 43909 + '\x13b6'# -> unI64 43910 + '\x13b7'# -> unI64 43911 + '\x13b8'# -> unI64 43912 + '\x13b9'# -> unI64 43913 + '\x13ba'# -> unI64 43914 + '\x13bb'# -> unI64 43915 + '\x13bc'# -> unI64 43916 + '\x13bd'# -> unI64 43917 + '\x13be'# -> unI64 43918 + '\x13bf'# -> unI64 43919 + '\x13c0'# -> unI64 43920 + '\x13c1'# -> unI64 43921 + '\x13c2'# -> unI64 43922 + '\x13c3'# -> unI64 43923 + '\x13c4'# -> unI64 43924 + '\x13c5'# -> unI64 43925 + '\x13c6'# -> unI64 43926 + '\x13c7'# -> unI64 43927 + '\x13c8'# -> unI64 43928 + '\x13c9'# -> unI64 43929 + '\x13ca'# -> unI64 43930 + '\x13cb'# -> unI64 43931 + '\x13cc'# -> unI64 43932 + '\x13cd'# -> unI64 43933 + '\x13ce'# -> unI64 43934 + '\x13cf'# -> unI64 43935 + '\x13d0'# -> unI64 43936 + '\x13d1'# -> unI64 43937 + '\x13d2'# -> unI64 43938 + '\x13d3'# -> unI64 43939 + '\x13d4'# -> unI64 43940 + '\x13d5'# -> unI64 43941 + '\x13d6'# -> unI64 43942 + '\x13d7'# -> unI64 43943 + '\x13d8'# -> unI64 43944 + '\x13d9'# -> unI64 43945 + '\x13da'# -> unI64 43946 + '\x13db'# -> unI64 43947 + '\x13dc'# -> unI64 43948 + '\x13dd'# -> unI64 43949 + '\x13de'# -> unI64 43950 + '\x13df'# -> unI64 43951 + '\x13e0'# -> unI64 43952 + '\x13e1'# -> unI64 43953 + '\x13e2'# -> unI64 43954 + '\x13e3'# -> unI64 43955 + '\x13e4'# -> unI64 43956 + '\x13e5'# -> unI64 43957 + '\x13e6'# -> unI64 43958 + '\x13e7'# -> unI64 43959 + '\x13e8'# -> unI64 43960 + '\x13e9'# -> unI64 43961 + '\x13ea'# -> unI64 43962 + '\x13eb'# -> unI64 43963 + '\x13ec'# -> unI64 43964 + '\x13ed'# -> unI64 43965 + '\x13ee'# -> unI64 43966 + '\x13ef'# -> unI64 43967 + '\x13f0'# -> unI64 5112 + '\x13f1'# -> unI64 5113 + '\x13f2'# -> unI64 5114 + '\x13f3'# -> unI64 5115 + '\x13f4'# -> unI64 5116 + '\x13f5'# -> unI64 5117 + '\x1c90'# -> unI64 4304 + '\x1c91'# -> unI64 4305 + '\x1c92'# -> unI64 4306 + '\x1c93'# -> unI64 4307 + '\x1c94'# -> unI64 4308 + '\x1c95'# -> unI64 4309 + '\x1c96'# -> unI64 4310 + '\x1c97'# -> unI64 4311 + '\x1c98'# -> unI64 4312 + '\x1c99'# -> unI64 4313 + '\x1c9a'# -> unI64 4314 + '\x1c9b'# -> unI64 4315 + '\x1c9c'# -> unI64 4316 + '\x1c9d'# -> unI64 4317 + '\x1c9e'# -> unI64 4318 + '\x1c9f'# -> unI64 4319 + '\x1ca0'# -> unI64 4320 + '\x1ca1'# -> unI64 4321 + '\x1ca2'# -> unI64 4322 + '\x1ca3'# -> unI64 4323 + '\x1ca4'# -> unI64 4324 + '\x1ca5'# -> unI64 4325 + '\x1ca6'# -> unI64 4326 + '\x1ca7'# -> unI64 4327 + '\x1ca8'# -> unI64 4328 + '\x1ca9'# -> unI64 4329 + '\x1caa'# -> unI64 4330 + '\x1cab'# -> unI64 4331 + '\x1cac'# -> unI64 4332 + '\x1cad'# -> unI64 4333 + '\x1cae'# -> unI64 4334 + '\x1caf'# -> unI64 4335 + '\x1cb0'# -> unI64 4336 + '\x1cb1'# -> unI64 4337 + '\x1cb2'# -> unI64 4338 + '\x1cb3'# -> unI64 4339 + '\x1cb4'# -> unI64 4340 + '\x1cb5'# -> unI64 4341 + '\x1cb6'# -> unI64 4342 + '\x1cb7'# -> unI64 4343 + '\x1cb8'# -> unI64 4344 + '\x1cb9'# -> unI64 4345 + '\x1cba'# -> unI64 4346 + '\x1cbd'# -> unI64 4349 + '\x1cbe'# -> unI64 4350 + '\x1cbf'# -> unI64 4351 + '\x1e00'# -> unI64 7681 + '\x1e02'# -> unI64 7683 + '\x1e04'# -> unI64 7685 + '\x1e06'# -> unI64 7687 + '\x1e08'# -> unI64 7689 + '\x1e0a'# -> unI64 7691 + '\x1e0c'# -> unI64 7693 + '\x1e0e'# -> unI64 7695 + '\x1e10'# -> unI64 7697 + '\x1e12'# -> unI64 7699 + '\x1e14'# -> unI64 7701 + '\x1e16'# -> unI64 7703 + '\x1e18'# -> unI64 7705 + '\x1e1a'# -> unI64 7707 + '\x1e1c'# -> unI64 7709 + '\x1e1e'# -> unI64 7711 + '\x1e20'# -> unI64 7713 + '\x1e22'# -> unI64 7715 + '\x1e24'# -> unI64 7717 + '\x1e26'# -> unI64 7719 + '\x1e28'# -> unI64 7721 + '\x1e2a'# -> unI64 7723 + '\x1e2c'# -> unI64 7725 + '\x1e2e'# -> unI64 7727 + '\x1e30'# -> unI64 7729 + '\x1e32'# -> unI64 7731 + '\x1e34'# -> unI64 7733 + '\x1e36'# -> unI64 7735 + '\x1e38'# -> unI64 7737 + '\x1e3a'# -> unI64 7739 + '\x1e3c'# -> unI64 7741 + '\x1e3e'# -> unI64 7743 + '\x1e40'# -> unI64 7745 + '\x1e42'# -> unI64 7747 + '\x1e44'# -> unI64 7749 + '\x1e46'# -> unI64 7751 + '\x1e48'# -> unI64 7753 + '\x1e4a'# -> unI64 7755 + '\x1e4c'# -> unI64 7757 + '\x1e4e'# -> unI64 7759 + '\x1e50'# -> unI64 7761 + '\x1e52'# -> unI64 7763 + '\x1e54'# -> unI64 7765 + '\x1e56'# -> unI64 7767 + '\x1e58'# -> unI64 7769 + '\x1e5a'# -> unI64 7771 + '\x1e5c'# -> unI64 7773 + '\x1e5e'# -> unI64 7775 + '\x1e60'# -> unI64 7777 + '\x1e62'# -> unI64 7779 + '\x1e64'# -> unI64 7781 + '\x1e66'# -> unI64 7783 + '\x1e68'# -> unI64 7785 + '\x1e6a'# -> unI64 7787 + '\x1e6c'# -> unI64 7789 + '\x1e6e'# -> unI64 7791 + '\x1e70'# -> unI64 7793 + '\x1e72'# -> unI64 7795 + '\x1e74'# -> unI64 7797 + '\x1e76'# -> unI64 7799 + '\x1e78'# -> unI64 7801 + '\x1e7a'# -> unI64 7803 + '\x1e7c'# -> unI64 7805 + '\x1e7e'# -> unI64 7807 + '\x1e80'# -> unI64 7809 + '\x1e82'# -> unI64 7811 + '\x1e84'# -> unI64 7813 + '\x1e86'# -> unI64 7815 + '\x1e88'# -> unI64 7817 + '\x1e8a'# -> unI64 7819 + '\x1e8c'# -> unI64 7821 + '\x1e8e'# -> unI64 7823 + '\x1e90'# -> unI64 7825 + '\x1e92'# -> unI64 7827 + '\x1e94'# -> unI64 7829 + '\x1ea0'# -> unI64 7841 + '\x1ea2'# -> unI64 7843 + '\x1ea4'# -> unI64 7845 + '\x1ea6'# -> unI64 7847 + '\x1ea8'# -> unI64 7849 + '\x1eaa'# -> unI64 7851 + '\x1eac'# -> unI64 7853 + '\x1eae'# -> unI64 7855 + '\x1eb0'# -> unI64 7857 + '\x1eb2'# -> unI64 7859 + '\x1eb4'# -> unI64 7861 + '\x1eb6'# -> unI64 7863 + '\x1eb8'# -> unI64 7865 + '\x1eba'# -> unI64 7867 + '\x1ebc'# -> unI64 7869 + '\x1ebe'# -> unI64 7871 + '\x1ec0'# -> unI64 7873 + '\x1ec2'# -> unI64 7875 + '\x1ec4'# -> unI64 7877 + '\x1ec6'# -> unI64 7879 + '\x1ec8'# -> unI64 7881 + '\x1eca'# -> unI64 7883 + '\x1ecc'# -> unI64 7885 + '\x1ece'# -> unI64 7887 + '\x1ed0'# -> unI64 7889 + '\x1ed2'# -> unI64 7891 + '\x1ed4'# -> unI64 7893 + '\x1ed6'# -> unI64 7895 + '\x1ed8'# -> unI64 7897 + '\x1eda'# -> unI64 7899 + '\x1edc'# -> unI64 7901 + '\x1ede'# -> unI64 7903 + '\x1ee0'# -> unI64 7905 + '\x1ee2'# -> unI64 7907 + '\x1ee4'# -> unI64 7909 + '\x1ee6'# -> unI64 7911 + '\x1ee8'# -> unI64 7913 + '\x1eea'# -> unI64 7915 + '\x1eec'# -> unI64 7917 + '\x1eee'# -> unI64 7919 + '\x1ef0'# -> unI64 7921 + '\x1ef2'# -> unI64 7923 + '\x1ef4'# -> unI64 7925 + '\x1ef6'# -> unI64 7927 + '\x1ef8'# -> unI64 7929 + '\x1efa'# -> unI64 7931 + '\x1efc'# -> unI64 7933 + '\x1efe'# -> unI64 7935 + '\x1f08'# -> unI64 7936 + '\x1f09'# -> unI64 7937 + '\x1f0a'# -> unI64 7938 + '\x1f0b'# -> unI64 7939 + '\x1f0c'# -> unI64 7940 + '\x1f0d'# -> unI64 7941 + '\x1f0e'# -> unI64 7942 + '\x1f0f'# -> unI64 7943 + '\x1f18'# -> unI64 7952 + '\x1f19'# -> unI64 7953 + '\x1f1a'# -> unI64 7954 + '\x1f1b'# -> unI64 7955 + '\x1f1c'# -> unI64 7956 + '\x1f1d'# -> unI64 7957 + '\x1f28'# -> unI64 7968 + '\x1f29'# -> unI64 7969 + '\x1f2a'# -> unI64 7970 + '\x1f2b'# -> unI64 7971 + '\x1f2c'# -> unI64 7972 + '\x1f2d'# -> unI64 7973 + '\x1f2e'# -> unI64 7974 + '\x1f2f'# -> unI64 7975 + '\x1f38'# -> unI64 7984 + '\x1f39'# -> unI64 7985 + '\x1f3a'# -> unI64 7986 + '\x1f3b'# -> unI64 7987 + '\x1f3c'# -> unI64 7988 + '\x1f3d'# -> unI64 7989 + '\x1f3e'# -> unI64 7990 + '\x1f3f'# -> unI64 7991 + '\x1f48'# -> unI64 8000 + '\x1f49'# -> unI64 8001 + '\x1f4a'# -> unI64 8002 + '\x1f4b'# -> unI64 8003 + '\x1f4c'# -> unI64 8004 + '\x1f4d'# -> unI64 8005 + '\x1f59'# -> unI64 8017 + '\x1f5b'# -> unI64 8019 + '\x1f5d'# -> unI64 8021 + '\x1f5f'# -> unI64 8023 + '\x1f68'# -> unI64 8032 + '\x1f69'# -> unI64 8033 + '\x1f6a'# -> unI64 8034 + '\x1f6b'# -> unI64 8035 + '\x1f6c'# -> unI64 8036 + '\x1f6d'# -> unI64 8037 + '\x1f6e'# -> unI64 8038 + '\x1f6f'# -> unI64 8039 + '\x1fb8'# -> unI64 8112 + '\x1fb9'# -> unI64 8113 + '\x1fba'# -> unI64 8048 + '\x1fbb'# -> unI64 8049 + '\x1fc8'# -> unI64 8050 + '\x1fc9'# -> unI64 8051 + '\x1fca'# -> unI64 8052 + '\x1fcb'# -> unI64 8053 + '\x1fd8'# -> unI64 8144 + '\x1fd9'# -> unI64 8145 + '\x1fda'# -> unI64 8054 + '\x1fdb'# -> unI64 8055 + '\x1fe8'# -> unI64 8160 + '\x1fe9'# -> unI64 8161 + '\x1fea'# -> unI64 8058 + '\x1feb'# -> unI64 8059 + '\x1fec'# -> unI64 8165 + '\x1ff8'# -> unI64 8056 + '\x1ff9'# -> unI64 8057 + '\x1ffa'# -> unI64 8060 + '\x1ffb'# -> unI64 8061 + '\x2126'# -> unI64 969 + '\x212a'# -> unI64 107 + '\x212b'# -> unI64 229 + '\x2132'# -> unI64 8526 + '\x2160'# -> unI64 8560 + '\x2161'# -> unI64 8561 + '\x2162'# -> unI64 8562 + '\x2163'# -> unI64 8563 + '\x2164'# -> unI64 8564 + '\x2165'# -> unI64 8565 + '\x2166'# -> unI64 8566 + '\x2167'# -> unI64 8567 + '\x2168'# -> unI64 8568 + '\x2169'# -> unI64 8569 + '\x216a'# -> unI64 8570 + '\x216b'# -> unI64 8571 + '\x216c'# -> unI64 8572 + '\x216d'# -> unI64 8573 + '\x216e'# -> unI64 8574 + '\x216f'# -> unI64 8575 + '\x2183'# -> unI64 8580 + '\x24b6'# -> unI64 9424 + '\x24b7'# -> unI64 9425 + '\x24b8'# -> unI64 9426 + '\x24b9'# -> unI64 9427 + '\x24ba'# -> unI64 9428 + '\x24bb'# -> unI64 9429 + '\x24bc'# -> unI64 9430 + '\x24bd'# -> unI64 9431 + '\x24be'# -> unI64 9432 + '\x24bf'# -> unI64 9433 + '\x24c0'# -> unI64 9434 + '\x24c1'# -> unI64 9435 + '\x24c2'# -> unI64 9436 + '\x24c3'# -> unI64 9437 + '\x24c4'# -> unI64 9438 + '\x24c5'# -> unI64 9439 + '\x24c6'# -> unI64 9440 + '\x24c7'# -> unI64 9441 + '\x24c8'# -> unI64 9442 + '\x24c9'# -> unI64 9443 + '\x24ca'# -> unI64 9444 + '\x24cb'# -> unI64 9445 + '\x24cc'# -> unI64 9446 + '\x24cd'# -> unI64 9447 + '\x24ce'# -> unI64 9448 + '\x24cf'# -> unI64 9449 + '\x2c00'# -> unI64 11312 + '\x2c01'# -> unI64 11313 + '\x2c02'# -> unI64 11314 + '\x2c03'# -> unI64 11315 + '\x2c04'# -> unI64 11316 + '\x2c05'# -> unI64 11317 + '\x2c06'# -> unI64 11318 + '\x2c07'# -> unI64 11319 + '\x2c08'# -> unI64 11320 + '\x2c09'# -> unI64 11321 + '\x2c0a'# -> unI64 11322 + '\x2c0b'# -> unI64 11323 + '\x2c0c'# -> unI64 11324 + '\x2c0d'# -> unI64 11325 + '\x2c0e'# -> unI64 11326 + '\x2c0f'# -> unI64 11327 + '\x2c10'# -> unI64 11328 + '\x2c11'# -> unI64 11329 + '\x2c12'# -> unI64 11330 + '\x2c13'# -> unI64 11331 + '\x2c14'# -> unI64 11332 + '\x2c15'# -> unI64 11333 + '\x2c16'# -> unI64 11334 + '\x2c17'# -> unI64 11335 + '\x2c18'# -> unI64 11336 + '\x2c19'# -> unI64 11337 + '\x2c1a'# -> unI64 11338 + '\x2c1b'# -> unI64 11339 + '\x2c1c'# -> unI64 11340 + '\x2c1d'# -> unI64 11341 + '\x2c1e'# -> unI64 11342 + '\x2c1f'# -> unI64 11343 + '\x2c20'# -> unI64 11344 + '\x2c21'# -> unI64 11345 + '\x2c22'# -> unI64 11346 + '\x2c23'# -> unI64 11347 + '\x2c24'# -> unI64 11348 + '\x2c25'# -> unI64 11349 + '\x2c26'# -> unI64 11350 + '\x2c27'# -> unI64 11351 + '\x2c28'# -> unI64 11352 + '\x2c29'# -> unI64 11353 + '\x2c2a'# -> unI64 11354 + '\x2c2b'# -> unI64 11355 + '\x2c2c'# -> unI64 11356 + '\x2c2d'# -> unI64 11357 + '\x2c2e'# -> unI64 11358 + '\x2c2f'# -> unI64 11359 + '\x2c60'# -> unI64 11361 + '\x2c62'# -> unI64 619 + '\x2c63'# -> unI64 7549 + '\x2c64'# -> unI64 637 + '\x2c67'# -> unI64 11368 + '\x2c69'# -> unI64 11370 + '\x2c6b'# -> unI64 11372 + '\x2c6d'# -> unI64 593 + '\x2c6e'# -> unI64 625 + '\x2c6f'# -> unI64 592 + '\x2c70'# -> unI64 594 + '\x2c72'# -> unI64 11379 + '\x2c75'# -> unI64 11382 + '\x2c7e'# -> unI64 575 + '\x2c7f'# -> unI64 576 + '\x2c80'# -> unI64 11393 + '\x2c82'# -> unI64 11395 + '\x2c84'# -> unI64 11397 + '\x2c86'# -> unI64 11399 + '\x2c88'# -> unI64 11401 + '\x2c8a'# -> unI64 11403 + '\x2c8c'# -> unI64 11405 + '\x2c8e'# -> unI64 11407 + '\x2c90'# -> unI64 11409 + '\x2c92'# -> unI64 11411 + '\x2c94'# -> unI64 11413 + '\x2c96'# -> unI64 11415 + '\x2c98'# -> unI64 11417 + '\x2c9a'# -> unI64 11419 + '\x2c9c'# -> unI64 11421 + '\x2c9e'# -> unI64 11423 + '\x2ca0'# -> unI64 11425 + '\x2ca2'# -> unI64 11427 + '\x2ca4'# -> unI64 11429 + '\x2ca6'# -> unI64 11431 + '\x2ca8'# -> unI64 11433 + '\x2caa'# -> unI64 11435 + '\x2cac'# -> unI64 11437 + '\x2cae'# -> unI64 11439 + '\x2cb0'# -> unI64 11441 + '\x2cb2'# -> unI64 11443 + '\x2cb4'# -> unI64 11445 + '\x2cb6'# -> unI64 11447 + '\x2cb8'# -> unI64 11449 + '\x2cba'# -> unI64 11451 + '\x2cbc'# -> unI64 11453 + '\x2cbe'# -> unI64 11455 + '\x2cc0'# -> unI64 11457 + '\x2cc2'# -> unI64 11459 + '\x2cc4'# -> unI64 11461 + '\x2cc6'# -> unI64 11463 + '\x2cc8'# -> unI64 11465 + '\x2cca'# -> unI64 11467 + '\x2ccc'# -> unI64 11469 + '\x2cce'# -> unI64 11471 + '\x2cd0'# -> unI64 11473 + '\x2cd2'# -> unI64 11475 + '\x2cd4'# -> unI64 11477 + '\x2cd6'# -> unI64 11479 + '\x2cd8'# -> unI64 11481 + '\x2cda'# -> unI64 11483 + '\x2cdc'# -> unI64 11485 + '\x2cde'# -> unI64 11487 + '\x2ce0'# -> unI64 11489 + '\x2ce2'# -> unI64 11491 + '\x2ceb'# -> unI64 11500 + '\x2ced'# -> unI64 11502 + '\x2cf2'# -> unI64 11507 + '\xa640'# -> unI64 42561 + '\xa642'# -> unI64 42563 + '\xa644'# -> unI64 42565 + '\xa646'# -> unI64 42567 + '\xa648'# -> unI64 42569 + '\xa64a'# -> unI64 42571 + '\xa64c'# -> unI64 42573 + '\xa64e'# -> unI64 42575 + '\xa650'# -> unI64 42577 + '\xa652'# -> unI64 42579 + '\xa654'# -> unI64 42581 + '\xa656'# -> unI64 42583 + '\xa658'# -> unI64 42585 + '\xa65a'# -> unI64 42587 + '\xa65c'# -> unI64 42589 + '\xa65e'# -> unI64 42591 + '\xa660'# -> unI64 42593 + '\xa662'# -> unI64 42595 + '\xa664'# -> unI64 42597 + '\xa666'# -> unI64 42599 + '\xa668'# -> unI64 42601 + '\xa66a'# -> unI64 42603 + '\xa66c'# -> unI64 42605 + '\xa680'# -> unI64 42625 + '\xa682'# -> unI64 42627 + '\xa684'# -> unI64 42629 + '\xa686'# -> unI64 42631 + '\xa688'# -> unI64 42633 + '\xa68a'# -> unI64 42635 + '\xa68c'# -> unI64 42637 + '\xa68e'# -> unI64 42639 + '\xa690'# -> unI64 42641 + '\xa692'# -> unI64 42643 + '\xa694'# -> unI64 42645 + '\xa696'# -> unI64 42647 + '\xa698'# -> unI64 42649 + '\xa69a'# -> unI64 42651 + '\xa722'# -> unI64 42787 + '\xa724'# -> unI64 42789 + '\xa726'# -> unI64 42791 + '\xa728'# -> unI64 42793 + '\xa72a'# -> unI64 42795 + '\xa72c'# -> unI64 42797 + '\xa72e'# -> unI64 42799 + '\xa732'# -> unI64 42803 + '\xa734'# -> unI64 42805 + '\xa736'# -> unI64 42807 + '\xa738'# -> unI64 42809 + '\xa73a'# -> unI64 42811 + '\xa73c'# -> unI64 42813 + '\xa73e'# -> unI64 42815 + '\xa740'# -> unI64 42817 + '\xa742'# -> unI64 42819 + '\xa744'# -> unI64 42821 + '\xa746'# -> unI64 42823 + '\xa748'# -> unI64 42825 + '\xa74a'# -> unI64 42827 + '\xa74c'# -> unI64 42829 + '\xa74e'# -> unI64 42831 + '\xa750'# -> unI64 42833 + '\xa752'# -> unI64 42835 + '\xa754'# -> unI64 42837 + '\xa756'# -> unI64 42839 + '\xa758'# -> unI64 42841 + '\xa75a'# -> unI64 42843 + '\xa75c'# -> unI64 42845 + '\xa75e'# -> unI64 42847 + '\xa760'# -> unI64 42849 + '\xa762'# -> unI64 42851 + '\xa764'# -> unI64 42853 + '\xa766'# -> unI64 42855 + '\xa768'# -> unI64 42857 + '\xa76a'# -> unI64 42859 + '\xa76c'# -> unI64 42861 + '\xa76e'# -> unI64 42863 + '\xa779'# -> unI64 42874 + '\xa77b'# -> unI64 42876 + '\xa77d'# -> unI64 7545 + '\xa77e'# -> unI64 42879 + '\xa780'# -> unI64 42881 + '\xa782'# -> unI64 42883 + '\xa784'# -> unI64 42885 + '\xa786'# -> unI64 42887 + '\xa78b'# -> unI64 42892 + '\xa78d'# -> unI64 613 + '\xa790'# -> unI64 42897 + '\xa792'# -> unI64 42899 + '\xa796'# -> unI64 42903 + '\xa798'# -> unI64 42905 + '\xa79a'# -> unI64 42907 + '\xa79c'# -> unI64 42909 + '\xa79e'# -> unI64 42911 + '\xa7a0'# -> unI64 42913 + '\xa7a2'# -> unI64 42915 + '\xa7a4'# -> unI64 42917 + '\xa7a6'# -> unI64 42919 + '\xa7a8'# -> unI64 42921 + '\xa7aa'# -> unI64 614 + '\xa7ab'# -> unI64 604 + '\xa7ac'# -> unI64 609 + '\xa7ad'# -> unI64 620 + '\xa7ae'# -> unI64 618 + '\xa7b0'# -> unI64 670 + '\xa7b1'# -> unI64 647 + '\xa7b2'# -> unI64 669 + '\xa7b3'# -> unI64 43859 + '\xa7b4'# -> unI64 42933 + '\xa7b6'# -> unI64 42935 + '\xa7b8'# -> unI64 42937 + '\xa7ba'# -> unI64 42939 + '\xa7bc'# -> unI64 42941 + '\xa7be'# -> unI64 42943 + '\xa7c0'# -> unI64 42945 + '\xa7c2'# -> unI64 42947 + '\xa7c4'# -> unI64 42900 + '\xa7c5'# -> unI64 642 + '\xa7c6'# -> unI64 7566 + '\xa7c7'# -> unI64 42952 + '\xa7c9'# -> unI64 42954 + '\xa7d0'# -> unI64 42961 + '\xa7d6'# -> unI64 42967 + '\xa7d8'# -> unI64 42969 + '\xa7f5'# -> unI64 42998 + '\xff21'# -> unI64 65345 + '\xff22'# -> unI64 65346 + '\xff23'# -> unI64 65347 + '\xff24'# -> unI64 65348 + '\xff25'# -> unI64 65349 + '\xff26'# -> unI64 65350 + '\xff27'# -> unI64 65351 + '\xff28'# -> unI64 65352 + '\xff29'# -> unI64 65353 + '\xff2a'# -> unI64 65354 + '\xff2b'# -> unI64 65355 + '\xff2c'# -> unI64 65356 + '\xff2d'# -> unI64 65357 + '\xff2e'# -> unI64 65358 + '\xff2f'# -> unI64 65359 + '\xff30'# -> unI64 65360 + '\xff31'# -> unI64 65361 + '\xff32'# -> unI64 65362 + '\xff33'# -> unI64 65363 + '\xff34'# -> unI64 65364 + '\xff35'# -> unI64 65365 + '\xff36'# -> unI64 65366 + '\xff37'# -> unI64 65367 + '\xff38'# -> unI64 65368 + '\xff39'# -> unI64 65369 + '\xff3a'# -> unI64 65370 + '\x10400'# -> unI64 66600 + '\x10401'# -> unI64 66601 + '\x10402'# -> unI64 66602 + '\x10403'# -> unI64 66603 + '\x10404'# -> unI64 66604 + '\x10405'# -> unI64 66605 + '\x10406'# -> unI64 66606 + '\x10407'# -> unI64 66607 + '\x10408'# -> unI64 66608 + '\x10409'# -> unI64 66609 + '\x1040a'# -> unI64 66610 + '\x1040b'# -> unI64 66611 + '\x1040c'# -> unI64 66612 + '\x1040d'# -> unI64 66613 + '\x1040e'# -> unI64 66614 + '\x1040f'# -> unI64 66615 + '\x10410'# -> unI64 66616 + '\x10411'# -> unI64 66617 + '\x10412'# -> unI64 66618 + '\x10413'# -> unI64 66619 + '\x10414'# -> unI64 66620 + '\x10415'# -> unI64 66621 + '\x10416'# -> unI64 66622 + '\x10417'# -> unI64 66623 + '\x10418'# -> unI64 66624 + '\x10419'# -> unI64 66625 + '\x1041a'# -> unI64 66626 + '\x1041b'# -> unI64 66627 + '\x1041c'# -> unI64 66628 + '\x1041d'# -> unI64 66629 + '\x1041e'# -> unI64 66630 + '\x1041f'# -> unI64 66631 + '\x10420'# -> unI64 66632 + '\x10421'# -> unI64 66633 + '\x10422'# -> unI64 66634 + '\x10423'# -> unI64 66635 + '\x10424'# -> unI64 66636 + '\x10425'# -> unI64 66637 + '\x10426'# -> unI64 66638 + '\x10427'# -> unI64 66639 + '\x104b0'# -> unI64 66776 + '\x104b1'# -> unI64 66777 + '\x104b2'# -> unI64 66778 + '\x104b3'# -> unI64 66779 + '\x104b4'# -> unI64 66780 + '\x104b5'# -> unI64 66781 + '\x104b6'# -> unI64 66782 + '\x104b7'# -> unI64 66783 + '\x104b8'# -> unI64 66784 + '\x104b9'# -> unI64 66785 + '\x104ba'# -> unI64 66786 + '\x104bb'# -> unI64 66787 + '\x104bc'# -> unI64 66788 + '\x104bd'# -> unI64 66789 + '\x104be'# -> unI64 66790 + '\x104bf'# -> unI64 66791 + '\x104c0'# -> unI64 66792 + '\x104c1'# -> unI64 66793 + '\x104c2'# -> unI64 66794 + '\x104c3'# -> unI64 66795 + '\x104c4'# -> unI64 66796 + '\x104c5'# -> unI64 66797 + '\x104c6'# -> unI64 66798 + '\x104c7'# -> unI64 66799 + '\x104c8'# -> unI64 66800 + '\x104c9'# -> unI64 66801 + '\x104ca'# -> unI64 66802 + '\x104cb'# -> unI64 66803 + '\x104cc'# -> unI64 66804 + '\x104cd'# -> unI64 66805 + '\x104ce'# -> unI64 66806 + '\x104cf'# -> unI64 66807 + '\x104d0'# -> unI64 66808 + '\x104d1'# -> unI64 66809 + '\x104d2'# -> unI64 66810 + '\x104d3'# -> unI64 66811 + '\x10570'# -> unI64 66967 + '\x10571'# -> unI64 66968 + '\x10572'# -> unI64 66969 + '\x10573'# -> unI64 66970 + '\x10574'# -> unI64 66971 + '\x10575'# -> unI64 66972 + '\x10576'# -> unI64 66973 + '\x10577'# -> unI64 66974 + '\x10578'# -> unI64 66975 + '\x10579'# -> unI64 66976 + '\x1057a'# -> unI64 66977 + '\x1057c'# -> unI64 66979 + '\x1057d'# -> unI64 66980 + '\x1057e'# -> unI64 66981 + '\x1057f'# -> unI64 66982 + '\x10580'# -> unI64 66983 + '\x10581'# -> unI64 66984 + '\x10582'# -> unI64 66985 + '\x10583'# -> unI64 66986 + '\x10584'# -> unI64 66987 + '\x10585'# -> unI64 66988 + '\x10586'# -> unI64 66989 + '\x10587'# -> unI64 66990 + '\x10588'# -> unI64 66991 + '\x10589'# -> unI64 66992 + '\x1058a'# -> unI64 66993 + '\x1058c'# -> unI64 66995 + '\x1058d'# -> unI64 66996 + '\x1058e'# -> unI64 66997 + '\x1058f'# -> unI64 66998 + '\x10590'# -> unI64 66999 + '\x10591'# -> unI64 67000 + '\x10592'# -> unI64 67001 + '\x10594'# -> unI64 67003 + '\x10595'# -> unI64 67004 + '\x10c80'# -> unI64 68800 + '\x10c81'# -> unI64 68801 + '\x10c82'# -> unI64 68802 + '\x10c83'# -> unI64 68803 + '\x10c84'# -> unI64 68804 + '\x10c85'# -> unI64 68805 + '\x10c86'# -> unI64 68806 + '\x10c87'# -> unI64 68807 + '\x10c88'# -> unI64 68808 + '\x10c89'# -> unI64 68809 + '\x10c8a'# -> unI64 68810 + '\x10c8b'# -> unI64 68811 + '\x10c8c'# -> unI64 68812 + '\x10c8d'# -> unI64 68813 + '\x10c8e'# -> unI64 68814 + '\x10c8f'# -> unI64 68815 + '\x10c90'# -> unI64 68816 + '\x10c91'# -> unI64 68817 + '\x10c92'# -> unI64 68818 + '\x10c93'# -> unI64 68819 + '\x10c94'# -> unI64 68820 + '\x10c95'# -> unI64 68821 + '\x10c96'# -> unI64 68822 + '\x10c97'# -> unI64 68823 + '\x10c98'# -> unI64 68824 + '\x10c99'# -> unI64 68825 + '\x10c9a'# -> unI64 68826 + '\x10c9b'# -> unI64 68827 + '\x10c9c'# -> unI64 68828 + '\x10c9d'# -> unI64 68829 + '\x10c9e'# -> unI64 68830 + '\x10c9f'# -> unI64 68831 + '\x10ca0'# -> unI64 68832 + '\x10ca1'# -> unI64 68833 + '\x10ca2'# -> unI64 68834 + '\x10ca3'# -> unI64 68835 + '\x10ca4'# -> unI64 68836 + '\x10ca5'# -> unI64 68837 + '\x10ca6'# -> unI64 68838 + '\x10ca7'# -> unI64 68839 + '\x10ca8'# -> unI64 68840 + '\x10ca9'# -> unI64 68841 + '\x10caa'# -> unI64 68842 + '\x10cab'# -> unI64 68843 + '\x10cac'# -> unI64 68844 + '\x10cad'# -> unI64 68845 + '\x10cae'# -> unI64 68846 + '\x10caf'# -> unI64 68847 + '\x10cb0'# -> unI64 68848 + '\x10cb1'# -> unI64 68849 + '\x10cb2'# -> unI64 68850 + '\x118a0'# -> unI64 71872 + '\x118a1'# -> unI64 71873 + '\x118a2'# -> unI64 71874 + '\x118a3'# -> unI64 71875 + '\x118a4'# -> unI64 71876 + '\x118a5'# -> unI64 71877 + '\x118a6'# -> unI64 71878 + '\x118a7'# -> unI64 71879 + '\x118a8'# -> unI64 71880 + '\x118a9'# -> unI64 71881 + '\x118aa'# -> unI64 71882 + '\x118ab'# -> unI64 71883 + '\x118ac'# -> unI64 71884 + '\x118ad'# -> unI64 71885 + '\x118ae'# -> unI64 71886 + '\x118af'# -> unI64 71887 + '\x118b0'# -> unI64 71888 + '\x118b1'# -> unI64 71889 + '\x118b2'# -> unI64 71890 + '\x118b3'# -> unI64 71891 + '\x118b4'# -> unI64 71892 + '\x118b5'# -> unI64 71893 + '\x118b6'# -> unI64 71894 + '\x118b7'# -> unI64 71895 + '\x118b8'# -> unI64 71896 + '\x118b9'# -> unI64 71897 + '\x118ba'# -> unI64 71898 + '\x118bb'# -> unI64 71899 + '\x118bc'# -> unI64 71900 + '\x118bd'# -> unI64 71901 + '\x118be'# -> unI64 71902 + '\x118bf'# -> unI64 71903 + '\x16e40'# -> unI64 93792 + '\x16e41'# -> unI64 93793 + '\x16e42'# -> unI64 93794 + '\x16e43'# -> unI64 93795 + '\x16e44'# -> unI64 93796 + '\x16e45'# -> unI64 93797 + '\x16e46'# -> unI64 93798 + '\x16e47'# -> unI64 93799 + '\x16e48'# -> unI64 93800 + '\x16e49'# -> unI64 93801 + '\x16e4a'# -> unI64 93802 + '\x16e4b'# -> unI64 93803 + '\x16e4c'# -> unI64 93804 + '\x16e4d'# -> unI64 93805 + '\x16e4e'# -> unI64 93806 + '\x16e4f'# -> unI64 93807 + '\x16e50'# -> unI64 93808 + '\x16e51'# -> unI64 93809 + '\x16e52'# -> unI64 93810 + '\x16e53'# -> unI64 93811 + '\x16e54'# -> unI64 93812 + '\x16e55'# -> unI64 93813 + '\x16e56'# -> unI64 93814 + '\x16e57'# -> unI64 93815 + '\x16e58'# -> unI64 93816 + '\x16e59'# -> unI64 93817 + '\x16e5a'# -> unI64 93818 + '\x16e5b'# -> unI64 93819 + '\x16e5c'# -> unI64 93820 + '\x16e5d'# -> unI64 93821 + '\x16e5e'# -> unI64 93822 + '\x16e5f'# -> unI64 93823 + '\x1e900'# -> unI64 125218 + '\x1e901'# -> unI64 125219 + '\x1e902'# -> unI64 125220 + '\x1e903'# -> unI64 125221 + '\x1e904'# -> unI64 125222 + '\x1e905'# -> unI64 125223 + '\x1e906'# -> unI64 125224 + '\x1e907'# -> unI64 125225 + '\x1e908'# -> unI64 125226 + '\x1e909'# -> unI64 125227 + '\x1e90a'# -> unI64 125228 + '\x1e90b'# -> unI64 125229 + '\x1e90c'# -> unI64 125230 + '\x1e90d'# -> unI64 125231 + '\x1e90e'# -> unI64 125232 + '\x1e90f'# -> unI64 125233 + '\x1e910'# -> unI64 125234 + '\x1e911'# -> unI64 125235 + '\x1e912'# -> unI64 125236 + '\x1e913'# -> unI64 125237 + '\x1e914'# -> unI64 125238 + '\x1e915'# -> unI64 125239 + '\x1e916'# -> unI64 125240 + '\x1e917'# -> unI64 125241 + '\x1e918'# -> unI64 125242 + '\x1e919'# -> unI64 125243 + '\x1e91a'# -> unI64 125244 + '\x1e91b'# -> unI64 125245 + '\x1e91c'# -> unI64 125246 + '\x1e91d'# -> unI64 125247 + '\x1e91e'# -> unI64 125248 + '\x1e91f'# -> unI64 125249 + '\x1e920'# -> unI64 125250 + '\x1e921'# -> unI64 125251 + _ -> unI64 0 diff --git a/src/Data/Text/Internal/Fusion/Common.hs b/src/Data/Text/Internal/Fusion/Common.hs --- a/src/Data/Text/Internal/Fusion/Common.hs +++ b/src/Data/Text/Internal/Fusion/Common.hs @@ -1,4 +1,5 @@ -{-# LANGUAGE BangPatterns, MagicHash, Rank2Types #-} +{-# LANGUAGE BangPatterns, MagicHash, Rank2Types, PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} -- | -- Module : Data.Text.Internal.Fusion.Common -- Copyright : (c) Bryan O'Sullivan 2009, 2012 @@ -12,7 +13,25 @@ -- API or name. Functions in this module may not check or enforce -- preconditions expected by public modules. Use at your own risk! -- --- Common stream fusion functionality for text. +-- This module provides a common stream fusion interface for text. +-- The stream interface allows us to write text pipelines which +-- do not allocate intermediate text values. For example, we could +-- guarantee no intermediate text is allocated by writing the following: +-- +-- @ +-- getNucleotides :: 'Data.Text.Internal.Text' -> 'Data.Text.Internal.Text' +-- getNucleotides = +-- 'Data.Text.Internal.Fusion.unstream' +-- . 'filter' isNucleotide +-- . 'toLower' +-- . 'Data.Text.Internal.Fusion.stream' +-- where +-- isNucleotide chr = +-- chr == \'a\' || +-- chr == \'c\' || +-- chr == \'t\' || +-- chr == \'g\' +-- @ module Data.Text.Internal.Fusion.Common ( @@ -101,33 +120,53 @@ , zipWith ) where -import Prelude (Bool(..), Char, Eq(..), Int, Integral, Maybe(..), +import Prelude (Bool(..), Char, Eq, (==), Int, Integral, Maybe(..), Ord(..), Ordering(..), String, (.), ($), (+), (-), (*), (++), (&&), fromIntegral, otherwise) import qualified Data.List as L import qualified Prelude as P -import Data.Bits (shiftL) +import Data.Bits (shiftL, shiftR, (.&.)) import Data.Char (isLetter, isSpace) -import Data.Int (Int64) +import GHC.Int (Int64(..)) +import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader) import Data.Text.Internal.Fusion.Types import Data.Text.Internal.Fusion.CaseMapping (foldMapping, lowerMapping, titleMapping, upperMapping) import Data.Text.Internal.Fusion.Size -import GHC.Prim (Addr#, chr#, indexCharOffAddr#, ord#) -import GHC.Types (Char(..), Int(..)) +import GHC.Exts (Char(..), Char#, chr#) +import GHC.Prim (Addr#, indexWord8OffAddr#) +import GHC.Stack (HasCallStack) +import GHC.Types (Int(..)) +import Data.Text.Internal.Unsafe.Char (unsafeChr8) +import GHC.Word +-- | /O(1)/ Convert a character into a 'Stream' +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'singleton' = 'Data.Text.singleton'@ singleton :: Char -> Stream Char singleton c = Stream next False (codePointsSize 1) where next False = Yield c True next True = Done {-# INLINE [0] singleton #-} +-- | /O(n)/ Convert a list into a 'Stream'. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'streamList' = 'Data.Text.pack'@ streamList :: [a] -> Stream a {-# INLINE [0] streamList #-} streamList s = Stream next s unknownSize where next [] = Done next (x:xs) = Yield x xs +-- | /O(n)/ Convert a 'Stream' into a list. +-- +-- __Properties__ +-- +-- @'unstreamList' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.unpack'@ unstreamList :: Stream a -> [a] unstreamList (Stream next s0 _len) = unfold s0 where unfold !s = case next s of @@ -143,28 +182,25 @@ -- -- This encoding uses the byte sequence "\xc0\x80" to represent NUL, -- and the string is NUL-terminated. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'streamCString#' addr# = 'Data.Text.Show.unpackCString#' addr#@ streamCString# :: Addr# -> Stream Char streamCString# addr = Stream step 0 unknownSize where step !i | b == 0 = Done - | b <= 0x7f = Yield (C# b#) (i+1) - | b <= 0xdf = let !c = chr $ ((b-0xc0) `shiftL` 6) + next 1 - in Yield c (i+2) - | b <= 0xef = let !c = chr $ ((b-0xe0) `shiftL` 12) + - (next 1 `shiftL` 6) + - next 2 - in Yield c (i+3) - | otherwise = let !c = chr $ ((b-0xf0) `shiftL` 18) + - (next 1 `shiftL` 12) + - (next 2 `shiftL` 6) + - next 3 - in Yield c (i+4) - where b = I# (ord# b#) - next n = I# (ord# (at# (i+n))) - 0x80 - !b# = at# i - at# (I# i#) = indexCharOffAddr# addr i# - chr (I# i#) = C# (chr# i#) + | otherwise = Yield chr (i + l) + where b = at# i + l = utf8LengthByLeader b + next n = at# (i+n) + chr = case l of + 1 -> unsafeChr8 b + 2 -> chr2 b (next 1) + 3 -> chr3 b (next 1) (next 2) + _ -> chr4 b (next 1) (next 2) (next 3) + at# (I# i#) = W8# (indexWord8OffAddr# addr i#) {-# INLINE [0] streamCString# #-} -- ---------------------------------------------------------------------------- @@ -174,6 +210,10 @@ | C1 !s -- | /O(n)/ Adds a character to the front of a Stream Char. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'cons' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.cons' c @ cons :: Char -> Stream Char -> Stream Char cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len + codePointsSize 1) where @@ -188,6 +228,10 @@ | J !a -- | /O(n)/ Adds a character to the end of a stream. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' . 'snoc' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.snoc' c @ snoc :: Stream Char -> Char -> Stream Char snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len + codePointsSize 1) where @@ -202,6 +246,10 @@ | R !r -- | /O(n)/ Appends one Stream to the other. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' ('append' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.append' t1 t2@ append :: Stream Char -> Stream Char -> Stream Char append (Stream next0 s01 len1) (Stream next1 s02 len2) = Stream next (L s01) (len1 + len2) @@ -216,9 +264,13 @@ Yield x s2' -> Yield x (R s2') {-# INLINE [0] append #-} --- | /O(1)/ Returns the first character of a Text, which must be non-empty. --- Subject to array fusion. -head :: Stream Char -> Char +-- | /O(1)/ Returns the first character of a 'Stream' 'Char', which must be non-empty. +-- This is a partial function, consider using 'uncons'. +-- +-- __Properties__ +-- +-- @ 'head' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.head' @ +head :: HasCallStack => Stream Char -> Char head (Stream next s0 _len) = loop_head s0 where loop_head !s = case next s of @@ -227,12 +279,18 @@ Done -> head_empty {-# INLINE [0] head #-} -head_empty :: a +head_empty :: HasCallStack => a head_empty = streamError "head" "Empty stream" {-# NOINLINE head_empty #-} --- | /O(1)/ Returns the first character and remainder of a 'Stream --- Char', or 'Nothing' if empty. Subject to array fusion. +-- | /O(1)/ Returns the first character and remainder of a 'Stream' +-- 'Char', or 'Nothing' if empty. +-- +-- __Properties__ +-- +-- @ 'Data.Functor.fmap' 'Data.Tuple.fst' . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.fst' . 'Data.Text.uncons' @ +-- +-- @ 'Data.Functor.fmap' ('Data.Text.Internal.Fusion.unstream' . 'Data.Tuple.snd') . 'uncons' . 'Data.Text.Internal.Fusion.stream' = 'Data.Functor.fmap' 'Data.Tuple.snd' . 'Data.Text.uncons' @ uncons :: Stream Char -> Maybe (Char, Stream Char) uncons (Stream next s0 len) = loop_uncons s0 where @@ -242,9 +300,13 @@ Done -> Nothing {-# INLINE [0] uncons #-} --- | /O(n)/ Returns the last character of a 'Stream Char', which must +-- | /O(n)/ Returns the last character of a 'Stream' 'Char', which must -- be non-empty. -last :: Stream Char -> Char +-- +-- __Properties__ +-- +-- @ 'last' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.last' @ +last :: HasCallStack => Stream Char -> Char last (Stream next s0 _len) = loop0_last s0 where loop0_last !s = case next s of @@ -257,9 +319,13 @@ Yield x' s' -> loop_last x' s' {-# INLINE[0] last #-} --- | /O(1)/ Returns all characters after the head of a Stream Char, which must --- be non-empty. -tail :: Stream Char -> Stream Char +-- | /O(1)/ Returns all characters after the head of a 'Stream' 'Char', which must +-- be non-empty. This is a partial function, consider using 'uncons'. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'tail' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.tail' @ +tail :: HasCallStack => Stream Char -> Stream Char tail (Stream next0 s0 len) = Stream next (C0 s0) (len - codePointsSize 1) where next (C0 s) = case next0 s of @@ -275,9 +341,13 @@ data Init s = Init0 !s | Init1 {-# UNPACK #-} !Char !s --- | /O(1)/ Returns all but the last character of a Stream Char, which +-- | /O(1)/ Returns all but the last character of a 'Stream' 'Char', which -- must be non-empty. -init :: Stream Char -> Stream Char +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'init' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.init' @ +init :: HasCallStack => Stream Char -> Stream Char init (Stream next0 s0 len) = Stream next (Init0 s0) (len - codePointsSize 1) where next (Init0 s) = case next0 s of @@ -290,7 +360,11 @@ Yield x' s' -> Yield x (Init1 x' s') {-# INLINE [0] init #-} --- | /O(1)/ Tests whether a Stream Char is empty or not. +-- | /O(1)/ Tests whether a 'Stream' 'Char' is empty or not. +-- +-- __Properties__ +-- +-- @ 'null' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.null' @ null :: Stream Char -> Bool null (Stream next s0 _len) = loop_null s0 where @@ -311,7 +385,6 @@ {-# INLINE[0] lengthI #-} -- | /O(n)/ Compares the count of characters in a string to a number. --- Subject to fusion. -- -- This function gives the same answer as comparing against the result -- of 'lengthI', but can short circuit if the count of characters is @@ -335,6 +408,10 @@ {-# INLINE[0] compareLengthI #-} -- | /O(n)/ Indicate whether a string contains exactly one element. +-- +-- __Properties__ +-- +-- @ 'isSingleton' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.isSingleton' @ isSingleton :: Stream Char -> Bool isSingleton (Stream next s0 _len) = loop 0 s0 where @@ -349,8 +426,12 @@ -- ---------------------------------------------------------------------------- -- * Stream transformations --- | /O(n)/ 'map' @f @xs is the Stream Char obtained by applying @f@ +-- | /O(n)/ 'map' @f @xs is the 'Stream' 'Char' obtained by applying @f@ -- to each element of @xs@. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'map' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.map' f @ map :: (Char -> Char) -> Stream Char -> Stream Char map f (Stream next0 s0 len) = Stream next s0 len where @@ -371,6 +452,10 @@ -- | /O(n)/ Take a character and place it between each of the -- characters of a 'Stream Char'. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'intersperse' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.intersperse' c @ intersperse :: Char -> Stream Char -> Stream Char intersperse c (Stream next0 s0 len) = Stream next (I1 s0) (len + unknownSize) where @@ -398,25 +483,35 @@ -- characters. -- | Map a 'Stream' through the given case-mapping function. -caseConvert :: (forall s. Char -> s -> Step (CC s) Char) +caseConvert :: (Char# -> _ {- unboxed Int64 -}) -> Stream Char -> Stream Char caseConvert remap (Stream next0 s0 len) = - Stream next (CC s0 '\0' '\0') (len `unionSize` (3*len)) + Stream next (CC s0 0) (len `unionSize` (3*len)) where - next (CC s '\0' _) = + next (CC s 0) = case next0 s of Done -> Done - Skip s' -> Skip (CC s' '\0' '\0') - Yield c s' -> remap c s' - next (CC s a b) = Yield a (CC s b '\0') + Skip s' -> Skip (CC s' 0) + Yield c@(C# c#) s' -> case I64# (remap c#) of + 0 -> Yield c (CC s' 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC s' b) + next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b) +chopOffChar :: Int64 -> (Char, Int64) +chopOffChar ab = (chr a, ab `shiftR` 21) + where + chr (I# n) = C# (chr# n) + mask = (1 `shiftL` 21) - 1 + a = fromIntegral $ ab .&. mask + -- | /O(n)/ Convert a string to folded case. This function is mainly -- useful for performing caseless (or case insensitive) string -- comparisons. -- -- A string @x@ is a caseless match for a string @y@ if and only if: -- --- @toCaseFold x == toCaseFold y@ +-- @'toCaseFold' x == 'toCaseFold' y@ -- -- The result string may be longer than the input string, and may -- differ from applying 'toLower' to the input string. For instance, @@ -432,6 +527,10 @@ -- conversion. The result string may be longer than the input string. -- For instance, the German eszett (U+00DF) maps to the two-letter -- sequence SS. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toUpper' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toUpper' @ toUpper :: Stream Char -> Stream Char toUpper = caseConvert upperMapping {-# INLINE [0] toUpper #-} @@ -441,6 +540,10 @@ -- For instance, the Latin capital letter I with dot above (U+0130) -- maps to the sequence Latin small letter i (U+0069) followed by -- combining dot above (U+0307). +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toLower' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toLower' @ toLower :: Stream Char -> Stream Char toLower = caseConvert lowerMapping {-# INLINE [0] toLower #-} @@ -463,21 +566,30 @@ -- guides disagree on whether the book name \"The Hill of the Red -- Fox\" is correctly title cased—but this function will -- capitalize /every/ word. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'toTitle' . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.toTitle' @ toTitle :: Stream Char -> Stream Char -toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') (len + unknownSize) +toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) 0) (len + unknownSize) where - next (CC (letter :*: s) '\0' _) = + next (CC (letter :*: s) 0) = case next0 s of Done -> Done - Skip s' -> Skip (CC (letter :*: s') '\0' '\0') - Yield c s' - | nonSpace -> if letter - then lowerMapping c (nonSpace :*: s') - else titleMapping c (letter' :*: s') - | otherwise -> Yield c (CC (letter' :*: s') '\0' '\0') + Skip s' -> Skip (CC (letter :*: s') 0) + Yield c@(C# c#) s' + | nonSpace, letter -> case I64# (lowerMapping c#) of + 0 -> Yield c (CC (nonSpace :*: s') 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC (nonSpace :*: s') b) + | nonSpace -> case I64# (titleMapping c#) of + 0 -> Yield c (CC (letter' :*: s') 0) + ab -> let (a, b) = chopOffChar ab in + Yield a (CC (letter' :*: s') b) + | otherwise -> Yield c (CC (letter' :*: s') 0) where nonSpace = P.not (isSpace c) letter' = isLetter c - next (CC s a b) = Yield a (CC s b '\0') + next (CC s ab) = let (a, b) = chopOffChar ab in Yield a (CC s b) {-# INLINE [0] toTitle #-} data Justify i s = Just1 !i !s @@ -502,8 +614,12 @@ -- * Reducing Streams (folds) -- | foldl, applied to a binary operator, a starting value (typically the --- left-identity of the operator), and a Stream, reduces the Stream using the +-- left-identity of the operator), and a 'Stream', reduces the 'Stream' using the -- binary operator, from left to right. +-- +-- __Properties__ +-- +-- @ 'foldl' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl' f z0 @ foldl :: (b -> Char -> b) -> b -> Stream Char -> b foldl f z0 (Stream next s0 _len) = loop_foldl z0 s0 where @@ -514,6 +630,10 @@ {-# INLINE [0] foldl #-} -- | A strict version of foldl. +-- +-- __Properties__ +-- +-- @ 'foldl'' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl'' f z0 @ foldl' :: (b -> Char -> b) -> b -> Stream Char -> b foldl' f z0 (Stream next s0 _len) = loop_foldl' z0 s0 where @@ -525,7 +645,11 @@ -- | foldl1 is a variant of foldl that has no starting value argument, -- and thus must be applied to non-empty Streams. -foldl1 :: (Char -> Char -> Char) -> Stream Char -> Char +-- +-- __Properties__ +-- +-- @ 'foldl1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1' f @ +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char foldl1 f (Stream next s0 _len) = loop0_foldl1 s0 where loop0_foldl1 !s = case next s of @@ -539,7 +663,11 @@ {-# INLINE [0] foldl1 #-} -- | A strict version of foldl1. -foldl1' :: (Char -> Char -> Char) -> Stream Char -> Char +-- +-- __Properties__ +-- +-- @ 'foldl1'' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldl1'' f @ +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char foldl1' f (Stream next s0 _len) = loop0_foldl1' s0 where loop0_foldl1' !s = case next s of @@ -555,6 +683,10 @@ -- | 'foldr', applied to a binary operator, a starting value (typically the -- right-identity of the operator), and a stream, reduces the stream using the -- binary operator, from right to left. +-- +-- __Properties__ +-- +-- @ 'foldr' f z0 . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr' f z0 @ foldr :: (Char -> b -> b) -> b -> Stream Char -> b foldr f z (Stream next s0 _len) = loop_foldr s0 where @@ -566,8 +698,11 @@ -- | foldr1 is a variant of 'foldr' that has no starting value argument, -- and thus must be applied to non-empty streams. --- Subject to array fusion. -foldr1 :: (Char -> Char -> Char) -> Stream Char -> Char +-- +-- __Properties__ +-- +-- @ 'foldr1' f . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.foldr1' f @ +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Stream Char -> Char foldr1 f (Stream next s0 _len) = loop0_foldr1 s0 where loop0_foldr1 !s = case next s of @@ -581,6 +716,12 @@ Yield x' s' -> f x (loop_foldr1 x' s') {-# INLINE [0] foldr1 #-} +-- | intercalate str strs interts the stream str in between the streams strs and +-- concatenates the result. +-- +-- __Properties__ +-- +-- @ 'intercalate' s = 'concat' . 'L.intersperse' s @ intercalate :: Stream Char -> [Stream Char] -> Stream Char intercalate s = concat . (L.intersperse s) {-# INLINE [0] intercalate #-} @@ -588,19 +729,31 @@ -- ---------------------------------------------------------------------------- -- ** Special folds --- | /O(n)/ Concatenate a list of streams. Subject to array fusion. +-- | /O(n)/ Concatenate a list of streams. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'concat' . 'Data.Functor.fmap' 'Data.Text.Internal.Fusion.stream' = 'Data.Text.concat'@ concat :: [Stream Char] -> Stream Char concat = L.foldr append empty {-# INLINE [0] concat #-} -- | Map a function over a stream that results in a stream and concatenate the -- results. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'concatMap' ('Data.Text.Fusion.stream' . f) . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.concatMap' f@ concatMap :: (Char -> Stream Char) -> Stream Char -> Stream Char concatMap f = foldr (append . f) empty {-# INLINE [0] concatMap #-} -- | /O(n)/ any @p @xs determines if any character in the stream -- @xs@ satisfies the predicate @p@. +-- +-- __Properties__ +-- +-- @'any' f . 'Data.Text.Fusion.stream' = 'Data.Text.any' f@ any :: (Char -> Bool) -> Stream Char -> Bool any p (Stream next0 s0 _len) = loop_any s0 where @@ -613,6 +766,10 @@ -- | /O(n)/ all @p @xs determines if all characters in the 'Text' -- @xs@ satisfy the predicate @p@. +-- +-- __Properties__ +-- +-- @'all' f . 'Data.Text.Fusion.stream' = 'Data.Text.all' f@ all :: (Char -> Bool) -> Stream Char -> Bool all p (Stream next0 s0 _len) = loop_all s0 where @@ -625,7 +782,11 @@ -- | /O(n)/ maximum returns the maximum value from a stream, which must be -- non-empty. -maximum :: Stream Char -> Char +-- +-- __Properties__ +-- +-- @'maximum' . 'Data.Text.Fusion.stream' = 'Data.Text.maximum'@ +maximum :: HasCallStack => Stream Char -> Char maximum (Stream next0 s0 _len) = loop0_maximum s0 where loop0_maximum !s = case next0 s of @@ -642,7 +803,11 @@ -- | /O(n)/ minimum returns the minimum value from a 'Text', which must be -- non-empty. -minimum :: Stream Char -> Char +-- +-- __Properties__ +-- +-- @'minimum' . 'Data.Text.Fusion.stream' = 'Data.Text.minimum'@ +minimum :: HasCallStack => Stream Char -> Char minimum (Stream next0 s0 _len) = loop0_minimum s0 where loop0_minimum !s = case next0 s of @@ -659,7 +824,18 @@ -- ----------------------------------------------------------------------------- -- * Building streams - +-- +-- | /O(n)/ 'scanl' is similar to 'foldl', but returns a stream of +-- successive reduced values from the left. Conceptually, if we +-- write the input stream as a list then we have: +-- +-- > scanl f z [x1, x2, ...] == [z, z 'f' x1, (z 'f' x1) 'f' x2, ...] +-- +-- __Properties__ +-- +-- @'head' ('scanl' f z xs) = z@ +-- +-- @'last' ('scanl' f z xs) = 'foldl' f z xs@ scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low where @@ -675,6 +851,8 @@ -- ----------------------------------------------------------------------------- -- ** Generating and unfolding streams +-- | /O(n)/ 'replicateCharI' @n@ @c@ is a 'Stream' 'Char' of length @n@ with @c@ the +-- value of every element. replicateCharI :: Integral a => a -> Char -> Stream Char replicateCharI !n !c | n < 0 = empty @@ -686,6 +864,9 @@ data RI s = RI !s {-# UNPACK #-} !Int64 + +-- | /O(n*m)/ 'replicateI' @n@ @t@ is a 'Stream' 'Char' consisting of the input +-- @t@ repeated @n@ times. replicateI :: Int64 -> Stream Char -> Stream Char replicateI n (Stream next0 s0 len) = Stream next (RI s0 0) (int64ToSize (max 0 n) * len) @@ -704,6 +885,10 @@ -- Nothing if it is done producing the stream or returns Just -- (a,b), in which case, a is the next Char in the string, and b is -- the seed value for further production. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'unfoldr' f z = 'Data.Text.unfoldr' f z@ unfoldr :: (a -> Maybe (Char,a)) -> a -> Stream Char unfoldr f s0 = Stream next s0 unknownSize where @@ -717,6 +902,10 @@ -- value. However, the length of the result is limited by the -- first argument to 'unfoldrNI'. This function is more efficient than -- 'unfoldr' when the length of the result is known. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' ('unfoldrNI' n f z) = 'Data.Text.unfoldrN' n f z@ unfoldrNI :: Integral a => a -> (b -> Maybe (Char,b)) -> b -> Stream Char unfoldrNI n f s0 | n < 0 = empty | otherwise = Stream next (0 :*: s0) (maxSize $ fromIntegral (n*2)) @@ -734,6 +923,10 @@ -- | /O(n)/ @'take' n@, applied to a stream, returns the prefix of the -- stream of length @n@, or the stream itself if @n@ is greater than the -- length of the stream. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'take' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.take' n@ take :: Integral a => a -> Stream Char -> Stream Char take n0 (Stream next0 s0 len) = Stream next (n0' :*: s0) (smaller len (codePointsSize $ fromIntegral n0')) @@ -754,6 +947,10 @@ -- | /O(n)/ @'drop' n@, applied to a stream, returns the suffix of the -- stream after the first @n@ characters, or the empty stream if @n@ -- is greater than the length of the stream. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'drop' n . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.drop' n@ drop :: Integral a => a -> Stream Char -> Stream Char drop n0 (Stream next0 s0 len) = Stream next (JS n0' s0) (len - codePointsSize (fromIntegral n0')) @@ -775,6 +972,10 @@ -- | 'takeWhile', applied to a predicate @p@ and a stream, returns the -- longest prefix (possibly empty) of elements that satisfy @p@. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'takeWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.takeWhile' p@ takeWhile :: (Char -> Bool) -> Stream Char -> Stream Char takeWhile p (Stream next0 s0 len) = Stream next s0 (len - unknownSize) where @@ -787,6 +988,10 @@ {-# INLINE [0] takeWhile #-} -- | @'dropWhile' p xs@ returns the suffix remaining after @'takeWhile' p xs@. +-- +-- __Properties__ +-- +-- @'Data.Text.Internal.Fusion.unstream' . 'dropWhile' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.dropWhile' p@ dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char dropWhile p (Stream next0 s0 len) = Stream next (L s0) (len - unknownSize) where @@ -804,6 +1009,10 @@ -- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns -- 'True' iff the first is a prefix of the second. +-- +-- __Properties__ +-- +-- @ 'isPrefixOf' ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2) = 'Data.Text.isPrefixOf' t1 t2@ isPrefixOf :: (Eq a) => Stream a -> Stream a -> Bool isPrefixOf (Stream next1 s1 _) (Stream next2 s2 _) = loop (next1 s1) (next2 s2) where @@ -823,6 +1032,10 @@ -- ** Searching by equality -- | /O(n)/ 'elem' is the stream membership predicate. +-- +-- __Properties__ +-- +-- @ 'elem' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.elem' c@ elem :: Char -> Stream Char -> Bool elem w (Stream next s0 _len) = loop_elem s0 where @@ -839,7 +1052,10 @@ -- | /O(n)/ The 'findBy' function takes a predicate and a stream, -- and returns the first element in matching the predicate, or 'Nothing' -- if there is no such element. - +-- +-- __Properties__ +-- +-- @ 'findBy' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.find' p@ findBy :: (Char -> Bool) -> Stream Char -> Maybe Char findBy p (Stream next s0 _len) = loop_find s0 where @@ -851,7 +1067,11 @@ {-# INLINE [0] findBy #-} -- | /O(n)/ Stream index (subscript) operator, starting from 0. -indexI :: Integral a => Stream Char -> a -> Char +-- +-- __Properties__ +-- +-- @ 'indexI' ('Data.Text.Internal.Fusion.stream' t) n = 'Data.Text.index' t n@ +indexI :: (HasCallStack, Integral a) => Stream Char -> a -> Char indexI (Stream next s0 _len) n0 | n0 < 0 = streamError "index" "Negative index" | otherwise = loop_index n0 s0 @@ -866,6 +1086,10 @@ -- | /O(n)/ 'filter', applied to a predicate and a stream, -- returns a stream containing those characters that satisfy the -- predicate. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.unstream' . 'filter' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.filter' p @ filter :: (Char -> Bool) -> Stream Char -> Stream Char filter p (Stream next0 s0 len) = Stream next s0 (len - unknownSize) -- HINT maybe too high @@ -885,6 +1109,10 @@ -- | The 'findIndexI' function takes a predicate and a stream and -- returns the index of the first element in the stream satisfying the -- predicate. +-- +-- __Properties__ +-- +-- @'findIndexI' p . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.findIndex' p @ findIndexI :: Integral a => (Char -> Bool) -> Stream Char -> Maybe a findIndexI p s = case findIndicesI p s of (i:_) -> Just i @@ -913,6 +1141,10 @@ -- | zipWith generalises 'zip' by zipping with the function given as -- the first argument, instead of a tupling function. +-- +-- __Properties__ +-- +-- @ 'Data.Text.Internal.Fusion.unstream' ('zipWith' f ('Data.Text.Internal.Fusion.stream' t1) ('Data.Text.Internal.Fusion.stream' t2)) = 'Data.Text.zipWith' f t1 t2@ zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) = Stream next (Z1 sa0 sb0) (smaller len1 len2) @@ -930,6 +1162,10 @@ -- | /O(n)/ The 'countCharI' function returns the number of times the -- query element appears in the given stream. +-- +-- __Properties__ +-- +-- @'countCharI' c . 'Data.Text.Internal.Fusion.stream' = 'Data.Text.countChar' c @ countCharI :: Integral a => Char -> Stream Char -> a countCharI a (Stream next s0 _len) = loop 0 s0 where @@ -940,13 +1176,13 @@ | otherwise -> loop i s' {-# INLINE [0] countCharI #-} -streamError :: String -> String -> a +streamError :: HasCallStack => String -> String -> a streamError func msg = P.error $ "Data.Text.Internal.Fusion.Common." ++ func ++ ": " ++ msg -emptyError :: String -> a +emptyError :: HasCallStack => String -> a emptyError func = internalError func "Empty input" -internalError :: String -> a +internalError :: HasCallStack => String -> a internalError func = streamError func "Internal error" int64ToSize :: Int64 -> Size diff --git a/src/Data/Text/Internal/Fusion/Size.hs b/src/Data/Text/Internal/Fusion/Size.hs --- a/src/Data/Text/Internal/Fusion/Size.hs +++ b/src/Data/Text/Internal/Fusion/Size.hs @@ -37,13 +37,13 @@ , isEmpty ) where -import Data.Char (ord) +import Data.Text.Internal.Encoding.Utf8 (utf8Length) import Data.Text.Internal (mul) #if defined(ASSERTS) import Control.Exception (assert) #endif --- | A size in UTF-16 code units. +-- | A size in UTF-8 code units (which is bytes). data Size = Between {-# UNPACK #-} !Int {-# UNPACK #-} !Int -- ^ Lower and upper bounds on size. | Unknown -- ^ Unknown size. deriving (Eq, Show) @@ -55,9 +55,7 @@ -- | The 'Size' of the given code point. charSize :: Char -> Size -charSize c - | ord c < 0x10000 = exactSize 1 - | otherwise = exactSize 2 +charSize = exactSize . utf8Length -- | The 'Size' of @n@ code points. codePointsSize :: Int -> Size @@ -65,7 +63,7 @@ #if defined(ASSERTS) assert (n >= 0) #endif - Between n (2*n) + Between n (4*n) {-# INLINE codePointsSize #-} exactSize :: Int -> Size @@ -160,7 +158,7 @@ upperBound k _ = k {-# INLINE upperBound #-} --- | Compute the maximum size from a size hint, if possible. +-- | Compute the minimum size from a size hint, if possible. lowerBound :: Int -> Size -> Int lowerBound _ (Between n _) = n lowerBound k _ = k diff --git a/src/Data/Text/Internal/Fusion/Types.hs b/src/Data/Text/Internal/Fusion/Types.hs --- a/src/Data/Text/Internal/Fusion/Types.hs +++ b/src/Data/Text/Internal/Fusion/Types.hs @@ -29,10 +29,11 @@ ) where import Data.Text.Internal.Fusion.Size +import Data.Int (Int64) import Data.Word (Word8) -- | Specialised tuple for case conversion. -data CC s = CC !s {-# UNPACK #-} !Char {-# UNPACK #-} !Char +data CC s = CC !s {-# UNPACK #-} !Int64 -- | Restreaming state. data RS s @@ -75,11 +76,13 @@ -- unstreaming functions must be able to cope with the hint being too -- small or too large. -- --- The size hint tries to track the UTF-16 code units in a stream, +-- The size hint tries to track the UTF-8 code units in a stream, -- but often counts the number of code points instead. It can easily -- undercount if, for instance, a transformed stream contains astral -- plane code points (those above 0x10000). +-- | A co-recursive type yielding a single element at a time depending +-- on the internal state it carries. data Stream a = forall s. Stream (s -> Step s a) -- stepper function diff --git a/src/Data/Text/Internal/Lazy.hs b/src/Data/Text/Internal/Lazy.hs --- a/src/Data/Text/Internal/Lazy.hs +++ b/src/Data/Text/Internal/Lazy.hs @@ -36,12 +36,15 @@ , defaultChunkSize , smallChunkSize , chunkOverhead + + , equal ) where +import Data.Bits (shiftL) import Data.Text () -import Data.Text.Internal.Unsafe.Shift (shiftL) import Data.Typeable (Typeable) import Foreign.Storable (sizeOf) +import qualified Data.Text.Array as A import qualified Data.Text.Internal as T data Text = Empty @@ -117,3 +120,16 @@ chunkOverhead :: Int chunkOverhead = sizeOf (undefined :: Int) `shiftL` 1 {-# INLINE chunkOverhead #-} + +equal :: Text -> Text -> Bool +equal Empty Empty = True +equal Empty _ = False +equal _ Empty = False +equal (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + case compare lenA lenB of + LT -> A.equal arrA offA arrB offB lenA && + as `equal` Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs + EQ -> A.equal arrA offA arrB offB lenA && + as `equal` bs + GT -> A.equal arrA offA arrB offB lenB && + Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as `equal` bs diff --git a/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs b/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs --- a/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs +++ b/src/Data/Text/Internal/Lazy/Encoding/Fusion.hs @@ -32,6 +32,7 @@ , module Data.Text.Internal.Encoding.Fusion.Common ) where +import Data.Bits (shiftL) import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize) import qualified Data.ByteString as B import qualified Data.ByteString.Unsafe as B @@ -40,9 +41,8 @@ import Data.Text.Encoding.Error import Data.Text.Internal.Fusion (Step(..), Stream(..)) import Data.Text.Internal.Fusion.Size -import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32) -import Data.Text.Internal.Unsafe.Shift (shiftL) -import Data.Text.Internal.Functions (unsafeWithForeignPtr) +import Data.Text.Internal.Unsafe.Char (unsafeChr8, unsafeChr16, unsafeChr32) +import Data.Text.Internal.Unsafe (unsafeWithForeignPtr) import Data.Word (Word8, Word16, Word32) import qualified Data.Text.Internal.Encoding.Utf8 as U8 import qualified Data.Text.Internal.Encoding.Utf16 as U16 @@ -112,7 +112,7 @@ where next (T bs@(Chunk ps _) S0 i) | i + 1 < len && U16.validate1 x1 = - Yield (unsafeChr x1) (T bs S0 (i+2)) + Yield (unsafeChr16 x1) (T bs S0 (i+2)) | i + 3 < len && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) where len = B.length ps @@ -123,7 +123,7 @@ next st@(T bs s i) = case s of S2 w1 w2 | U16.validate1 (c w1 w2) -> - Yield (unsafeChr (c w1 w2)) es + Yield (unsafeChr16 (c w1 w2)) es S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> Yield (U16.chr2 (c w1 w2) (c w3 w4)) es _ -> consume st @@ -152,7 +152,7 @@ where next (T bs@(Chunk ps _) S0 i) | i + 1 < len && U16.validate1 x1 = - Yield (unsafeChr x1) (T bs S0 (i+2)) + Yield (unsafeChr16 x1) (T bs S0 (i+2)) | i + 3 < len && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (T bs S0 (i+4)) where len = B.length ps @@ -163,7 +163,7 @@ next st@(T bs s i) = case s of S2 w1 w2 | U16.validate1 (c w1 w2) -> - Yield (unsafeChr (c w1 w2)) es + Yield (unsafeChr16 (c w1 w2)) es S4 w1 w2 w3 w4 | U16.validate2 (c w1 w2) (c w3 w4) -> Yield (U16.chr2 (c w1 w2) (c w3 w4)) es _ -> consume st diff --git a/src/Data/Text/Internal/Lazy/Fusion.hs b/src/Data/Text/Internal/Lazy/Fusion.hs --- a/src/Data/Text/Internal/Lazy/Fusion.hs +++ b/src/Data/Text/Internal/Lazy/Fusion.hs @@ -26,6 +26,7 @@ ) where import Prelude hiding (length) +import Data.Bits (shiftL) import qualified Data.Text.Internal.Fusion.Common as S import Control.Monad.ST (runST) import Data.Text.Internal.Fusion.Types @@ -34,12 +35,9 @@ import qualified Data.Text.Internal as I import qualified Data.Text.Array as A import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import Data.Text.Internal.Unsafe.Shift (shiftL) import Data.Text.Unsafe (Iter(..), iter) import Data.Int (Int64) -#if defined(ASSERTS) import GHC.Stack (HasCallStack) -#endif default(Int64) @@ -79,8 +77,8 @@ where unknownLength = 4 where inner marr !len s !i - | i + 1 >= chunkSize = finish marr i s - | i + 1 >= len = {-# SCC "unstreamChunks/resize" #-} do + | i + 3 >= chunkSize = finish marr i s + | i + 3 >= len = {-# SCC "unstreamChunks/resize" #-} do let newLen = min (len `shiftL` 1) chunkSize marr' <- A.new newLen A.copyM marr' 0 marr 0 len @@ -93,6 +91,7 @@ Yield x s' -> do d <- unsafeWrite marr i x inner marr len s' (i+d) finish marr len s' = do + A.shrinkM marr len arr <- A.unsafeFreeze marr return (I.Text arr 0 len `Chunk` outer s') {-# INLINE [0] unstreamChunks #-} @@ -124,7 +123,7 @@ {-# INLINE [0] unfoldrN #-} -- | /O(n)/ stream index (subscript) operator, starting from 0. -index :: Stream Char -> Int64 -> Char +index :: HasCallStack => Stream Char -> Int64 -> Char index = S.indexI {-# INLINE [0] index #-} diff --git a/src/Data/Text/Internal/Lazy/Search.hs b/src/Data/Text/Internal/Lazy/Search.hs --- a/src/Data/Text/Internal/Lazy/Search.hs +++ b/src/Data/Text/Internal/Lazy/Search.hs @@ -1,4 +1,6 @@ {-# LANGUAGE BangPatterns, ScopedTypeVariables #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} -- | -- Module : Data.Text.Lazy.Search @@ -22,14 +24,19 @@ indices ) where +import Data.Bits (unsafeShiftL) import qualified Data.Text.Array as A import Data.Int (Int64) -import Data.Word (Word16, Word64) +import Data.Word (Word8, Word64) import qualified Data.Text.Internal as T +import qualified Data.Text as T (concat, isPrefixOf) import Data.Text.Internal.Fusion.Types (PairS(..)) -import Data.Text.Internal.Lazy (Text(..), foldlChunks) +import Data.Text.Internal.Lazy (Text(..), foldrChunks) +import Data.Text.Unsafe (unsafeDupablePerformIO) import Data.Bits ((.|.), (.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftL) +import Foreign.C.Types +import GHC.Exts (ByteArray#) +import System.Posix.Types (CSsize(..)) -- | /O(n+m)/ Find the offsets of all non-overlapping indices of -- @needle@ within @haystack@. @@ -42,110 +49,97 @@ indices :: Text -- ^ Substring to search for (@needle@) -> Text -- ^ Text to search in (@haystack@) -> [Int64] -indices needle@(Chunk n ns) _haystack@(Chunk k ks) - | nlen <= 0 = [] - | nlen == 1 = indicesOne (nindex 0) 0 k ks - | otherwise = advance k ks 0 0 +indices needle + | nlen <= 0 = const [] + | nlen == 1 = indicesOne (A.unsafeIndex narr noff) 0 + | otherwise = advance 0 0 where - advance x@(T.Text _ _ l) xs = scan - where - scan !g !i - | i >= m = case xs of - Empty -> [] - Chunk y ys -> advance y ys g (i-m) + T.Text narr noff nlen = T.concat (foldrChunks (:) [] needle) + + advance !_ !_ Empty = [] + advance !(g :: Int64) !(i :: Int) xxs@(Chunk x@(T.Text xarr@(A.ByteArray xarr#) xoff l) xs) + | i >= l = advance g (i - l) xs | lackingHay (i + nlen) x xs = [] - | c == z && candidateMatch 0 = g : scan (g+nlen) (i+nlen) - | otherwise = scan (g+delta) (i+delta) + | c == z && candidateMatch = g : advance (g + intToInt64 nlen) (i + nlen) xxs + | otherwise = advance (g + intToInt64 delta) (i + delta) xxs where - m = intToInt64 l - c = hindex (i + nlast) + c = index xxs (i + nlast) delta | nextInPattern = nlen + 1 | c == z = skip + 1 - | otherwise = 1 - nextInPattern = mask .&. swizzle (hindex (i+nlen)) == 0 - candidateMatch !j - | j >= nlast = True - | hindex (i+j) /= nindex j = False - | otherwise = candidateMatch (j+1) - hindex = index x xs - nlen = wordLength needle + | l >= i + nlen = case unsafeDupablePerformIO $ + memchr xarr# (intToCSize (xoff + i + nlen)) (intToCSize (l - i - nlen)) z of + -1 -> max 1 (l - i - nlen) + s -> cSsizeToInt s + 1 + | otherwise = 1 + nextInPattern = mask .&. swizzle (index xxs (i + nlen)) == 0 + + candidateMatch + | i + nlen <= l = A.equal narr noff xarr (xoff + i) nlen + | otherwise = A.equal narr noff xarr (xoff + i) (l - i) && + T.Text narr (noff + l - i) (nlen - l + i) `isPrefixOf` xs + nlast = nlen - 1 - nindex = index n ns - z = foldlChunks fin 0 needle - where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1) - (mask :: Word64) :*: skip = buildTable n ns 0 0 0 (nlen-2) + z = A.unsafeIndex narr (noff + nlen - 1) + (mask :: Word64) :*: skip = buildTable 0 0 0 (nlen-2) - swizzle :: Word16 -> Word64 - swizzle w = 1 `shiftL` (word16ToInt w .&. 0x3f) + swizzle :: Word8 -> Word64 + swizzle w = 1 `unsafeShiftL` (word8ToInt w .&. 0x3f) - buildTable (T.Text xarr xoff xlen) xs = go - where - go !(g::Int64) !i !msk !skp - | i >= xlast = case xs of - Empty -> (msk .|. swizzle z) :*: skp - Chunk y ys -> buildTable y ys g 0 msk' skp' - | otherwise = go (g+1) (i+1) msk' skp' - where c = A.unsafeIndex xarr (xoff+i) + buildTable !g !i !msk !skp + | i >= nlast = (msk .|. swizzle z) :*: skp + | otherwise = buildTable (g+1) (i+1) msk' skp' + where c = A.unsafeIndex narr (noff+i) msk' = msk .|. swizzle c skp' | c == z = nlen - g - 2 | otherwise = skp - xlast = xlen - 1 -- | Check whether an attempt to index into the haystack at the -- given offset would fail. - lackingHay :: Int64 -> T.Text -> Text -> Bool - lackingHay q = go 0 - where - go p (T.Text _ _ l) ps = p' < q && case ps of - Empty -> True - Chunk r rs -> go p' r rs - where p' = p + intToInt64 l -indices _ _ = [] + lackingHay :: Int -> T.Text -> Text -> Bool + lackingHay q (T.Text _ _ l) ps = l < q && case ps of + Empty -> True + Chunk r rs -> lackingHay (q - l) r rs -- | Fast index into a partly unpacked 'Text'. We take into account -- the possibility that the caller might try to access one element -- past the end. -index :: T.Text -> Text -> Int64 -> Word16 -index (T.Text arr off len) xs !i - | j < len = A.unsafeIndex arr (off+j) - | otherwise = case xs of - Empty - -- out of bounds, but legal - | j == len -> 0 - -- should never happen, due to lackingHay above - | otherwise -> emptyError "index" - Chunk c cs -> index c cs (i-intToInt64 len) - where j = int64ToInt i +index :: Text -> Int -> Word8 +index Empty !_ = 0 +index (Chunk (T.Text arr off len) xs) !i + | i < len = A.unsafeIndex arr (off + i) + | otherwise = index xs (i - len) --- | A variant of 'indices' that scans linearly for a single 'Word16'. -indicesOne :: Word16 -> Int64 -> T.Text -> Text -> [Int64] +-- | A variant of 'indices' that scans linearly for a single 'Word8'. +indicesOne :: Word8 -> Int64 -> Text -> [Int64] indicesOne c = chunk where - chunk :: Int64 -> T.Text -> Text -> [Int64] - chunk !i (T.Text oarr ooff olen) os = go 0 + chunk :: Int64 -> Text -> [Int64] + chunk !_ Empty = [] + chunk !i (Chunk (T.Text oarr ooff olen) os) = go 0 where - go h | h >= olen = case os of - Empty -> [] - Chunk y ys -> chunk (i+intToInt64 olen) y ys + go h | h >= olen = chunk (i+intToInt64 olen) os | on == c = i + intToInt64 h : go (h+1) | otherwise = go (h+1) where on = A.unsafeIndex oarr (ooff+h) --- | The number of 'Word16' values in a 'Text'. -wordLength :: Text -> Int64 -wordLength = foldlChunks sumLength 0 - where - sumLength :: Int64 -> T.Text -> Int64 - sumLength i (T.Text _ _ l) = i + intToInt64 l - -emptyError :: String -> a -emptyError fun = error ("Data.Text.Lazy.Search." ++ fun ++ ": empty input") +-- | First argument is a strict Text, and second is a lazy one. +isPrefixOf :: T.Text -> Text -> Bool +isPrefixOf (T.Text _ _ xlen) Empty = xlen == 0 +isPrefixOf x@(T.Text xarr xoff xlen) (Chunk y@(T.Text _ _ ylen) ys) + | xlen <= ylen = x `T.isPrefixOf` y + | otherwise = y `T.isPrefixOf` x && T.Text xarr (xoff + ylen) (xlen - ylen) `isPrefixOf` ys intToInt64 :: Int -> Int64 intToInt64 = fromIntegral -int64ToInt :: Int64 -> Int -int64ToInt = fromIntegral +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral -word16ToInt :: Word16 -> Int -word16ToInt = fromIntegral +intToCSize :: Int -> CSize +intToCSize = fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = fromIntegral + +foreign import ccall unsafe "_hs_text_memchr" memchr + :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize diff --git a/src/Data/Text/Internal/Private.hs b/src/Data/Text/Internal/Private.hs --- a/src/Data/Text/Internal/Private.hs +++ b/src/Data/Text/Internal/Private.hs @@ -13,12 +13,14 @@ ( runText , span_ + , spanAscii_ ) where import Control.Monad.ST (ST, runST) import Data.Text.Internal (Text(..), text) import Data.Text.Unsafe (Iter(..), iter) import qualified Data.Text.Array as A +import Data.Word (Word8) #if defined(ASSERTS) import GHC.Stack (HasCallStack) @@ -34,12 +36,26 @@ where Iter c d = iter t i {-# INLINE span_ #-} +-- | For the sake of performance this function does not check +-- that a char is in ASCII range; it is a responsibility of @p@. +-- +-- @since 2.0 +spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #) +spanAscii_ p (Text arr off len) = (# hd, tl #) + where hd = text arr off k + tl = text arr (off + k) (len - k) + !k = loop 0 + loop !i | i < len && p (A.unsafeIndex arr (off + i)) = loop (i + 1) + | otherwise = i +{-# INLINE spanAscii_ #-} + runText :: #if defined(ASSERTS) HasCallStack => #endif (forall s. (A.MArray s -> Int -> ST s Text) -> ST s Text) -> Text runText act = runST (act $ \ !marr !len -> do + A.shrinkM marr len arr <- A.unsafeFreeze marr return $! text arr 0 len) {-# INLINE runText #-} diff --git a/src/Data/Text/Internal/Read.hs b/src/Data/Text/Internal/Read.hs --- a/src/Data/Text/Internal/Read.hs +++ b/src/Data/Text/Internal/Read.hs @@ -61,9 +61,20 @@ hexDigitToInt :: Char -> Int hexDigitToInt c - | c >= '0' && c <= '9' = ord c - ord '0' - | c >= 'a' && c <= 'f' = ord c - (ord 'a' - 10) - | otherwise = ord c - (ord 'A' - 10) + | to0 < 10 = wordToInt to0 + | toa < 6 = wordToInt toa + 10 + | otherwise = wordToInt toA + 10 + where + ordW = intToWord (ord c) + to0 = ordW - intToWord (ord '0') + toa = ordW - intToWord (ord 'a') + toA = ordW - intToWord (ord 'A') digitToInt :: Char -> Int digitToInt c = ord c - ord '0' + +intToWord :: Int -> Word +intToWord = fromIntegral + +wordToInt :: Word -> Int +wordToInt = fromIntegral diff --git a/src/Data/Text/Internal/Search.hs b/src/Data/Text/Internal/Search.hs --- a/src/Data/Text/Internal/Search.hs +++ b/src/Data/Text/Internal/Search.hs @@ -1,4 +1,6 @@ {-# LANGUAGE BangPatterns, ScopedTypeVariables #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UnliftedFFITypes #-} -- | -- Module : Data.Text.Internal.Search @@ -32,64 +34,82 @@ ) where import qualified Data.Text.Array as A -import Data.Word (Word64, Word16) +import Data.Word (Word64, Word8) import Data.Text.Internal (Text(..)) -import Data.Bits ((.|.), (.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftL) +import Data.Bits ((.|.), (.&.), unsafeShiftL) +import Data.Text.Unsafe (unsafeDupablePerformIO) +import Foreign.C.Types +import GHC.Exts (ByteArray#) +import System.Posix.Types (CSsize(..)) data T = {-# UNPACK #-} !Word64 :* {-# UNPACK #-} !Int -- | /O(n+m)/ Find the offsets of all non-overlapping indices of --- @needle@ within @haystack@. The offsets returned represent --- uncorrected indices in the low-level \"needle\" array, to which its --- offset must be added. +-- @needle@ within @haystack@. -- -- In (unlikely) bad cases, this algorithm's complexity degrades -- towards /O(n*m)/. indices :: Text -- ^ Substring to search for (@needle@) -> Text -- ^ Text to search in (@haystack@) -> [Int] -indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen) - | nlen == 1 = scanOne (nindex 0) - | nlen <= 0 || ldiff < 0 = [] - | otherwise = scan 0 +indices needle@(Text narr noff nlen) + | nlen == 1 = scanOne (A.unsafeIndex narr noff) + | nlen <= 0 = const [] + | otherwise = indices' needle +{-# INLINE indices #-} + +-- | nlen must be >= 2, otherwise nindex causes access violation +indices' :: Text -> Text -> [Int] +indices' (Text narr noff nlen) (Text harr@(A.ByteArray harr#) hoff hlen) = loop (hoff + nlen) where - ldiff = hlen - nlen nlast = nlen - 1 - z = nindex nlast + !z = nindex nlast nindex k = A.unsafeIndex narr (noff+k) - hindex k = A.unsafeIndex harr (hoff+k) - hindex' k | k == hlen = 0 - | otherwise = A.unsafeIndex harr (hoff+k) buildTable !i !msk !skp | i >= nlast = (msk .|. swizzle z) :* skp | otherwise = buildTable (i+1) (msk .|. swizzle c) skp' - where c = nindex i + where !c = nindex i skp' | c == z = nlen - i - 2 | otherwise = skp + !(mask :* skip) = buildTable 0 0 (nlen-2) - swizzle :: Word16 -> Word64 - swizzle k = 1 `shiftL` (word16ToInt k .&. 0x3f) + swizzle :: Word8 -> Word64 + swizzle !k = 1 `unsafeShiftL` (word8ToInt k .&. 0x3f) - scan !i - | i > ldiff = [] - | c == z && candidateMatch 0 = i : scan (i + nlen) - | otherwise = scan (i + delta) - where c = hindex (i + nlast) - candidateMatch !j - | j >= nlast = True - | hindex (i+j) /= nindex j = False - | otherwise = candidateMatch (j+1) - delta | nextInPattern = nlen + 1 - | c == z = skip + 1 - | otherwise = 1 - where nextInPattern = mask .&. swizzle (hindex' (i+nlen)) == 0 - !(mask :* skip) = buildTable 0 0 (nlen-2) - scanOne c = loop 0 - where loop !i | i >= hlen = [] - | hindex i == c = i : loop (i+1) - | otherwise = loop (i+1) -{-# INLINE indices #-} + loop !i + | i > hlen + hoff + = [] + | A.unsafeIndex harr (i - 1) == z + = if A.equal narr noff harr (i - nlen) nlen + then i - nlen - hoff : loop (i + nlen) + else loop (i + skip + 1) + | i == hlen + hoff + = [] + | mask .&. swizzle (A.unsafeIndex harr i) == 0 + = loop (i + nlen + 1) + | otherwise + = case unsafeDupablePerformIO $ memchr harr# (intToCSize i) (intToCSize (hlen + hoff - i)) z of + -1 -> [] + x -> loop (i + cSsizeToInt x + 1) +{-# INLINE indices' #-} -word16ToInt :: Word16 -> Int -word16ToInt = fromIntegral +scanOne :: Word8 -> Text -> [Int] +scanOne c (Text harr hoff hlen) = loop 0 + where + loop !i + | i >= hlen = [] + | A.unsafeIndex harr (hoff+i) == c = i : loop (i+1) + | otherwise = loop (i+1) +{-# INLINE scanOne #-} + +word8ToInt :: Word8 -> Int +word8ToInt = fromIntegral + +intToCSize :: Int -> CSize +intToCSize = fromIntegral + +cSsizeToInt :: CSsize -> Int +cSsizeToInt = fromIntegral + +foreign import ccall unsafe "_hs_text_memchr" memchr + :: ByteArray# -> CSize -> CSize -> Word8 -> IO CSsize diff --git a/src/Data/Text/Internal/Unsafe.hs b/src/Data/Text/Internal/Unsafe.hs --- a/src/Data/Text/Internal/Unsafe.hs +++ b/src/Data/Text/Internal/Unsafe.hs @@ -19,8 +19,17 @@ ( inlineInterleaveST , inlinePerformIO + , unsafeWithForeignPtr ) where +import Foreign.Ptr (Ptr) +import Foreign.ForeignPtr (ForeignPtr) +#if MIN_VERSION_base(4,15,0) +import qualified GHC.ForeignPtr (unsafeWithForeignPtr) +#else +import qualified Foreign.ForeignPtr (withForeignPtr) +#endif + import GHC.ST (ST(..)) import GHC.IO (IO(IO)) import GHC.Base (realWorld#) @@ -47,3 +56,10 @@ inlineInterleaveST (ST m) = ST $ \ s -> let r = case m s of (# _, res #) -> res in (# s, r #) {-# INLINE inlineInterleaveST #-} + +unsafeWithForeignPtr :: ForeignPtr a -> (Ptr a -> IO b) -> IO b +#if MIN_VERSION_base(4,15,0) +unsafeWithForeignPtr = GHC.ForeignPtr.unsafeWithForeignPtr +#else +unsafeWithForeignPtr = Foreign.ForeignPtr.withForeignPtr +#endif diff --git a/src/Data/Text/Internal/Unsafe/Char.hs b/src/Data/Text/Internal/Unsafe/Char.hs --- a/src/Data/Text/Internal/Unsafe/Char.hs +++ b/src/Data/Text/Internal/Unsafe/Char.hs @@ -19,15 +19,14 @@ module Data.Text.Internal.Unsafe.Char ( ord - , unsafeChr + , unsafeChr16 , unsafeChr8 , unsafeChr32 , unsafeWrite ) where import Control.Monad.ST (ST) -import Data.Bits ((.&.)) -import Data.Text.Internal.Unsafe.Shift (shiftR) +import Data.Text.Internal.Encoding.Utf8 import GHC.Exts (Char(..), Int(..), chr#, ord#, word2Int#) import GHC.Word (Word8(..), Word16(..), Word32(..)) import qualified Data.Text.Array as A @@ -40,9 +39,10 @@ ord (C# c#) = I# (ord# c#) {-# INLINE ord #-} -unsafeChr :: Word16 -> Char -unsafeChr (W16# w#) = C# (chr# (word2Int# (word16ToWord# w#))) -{-# INLINE unsafeChr #-} +-- | @since 2.0 +unsafeChr16 :: Word16 -> Char +unsafeChr16 (W16# w#) = C# (chr# (word2Int# (word16ToWord# w#))) +{-# INLINE unsafeChr16 #-} unsafeChr8 :: Word8 -> Char unsafeChr8 (W8# w#) = C# (chr# (word2Int# (word8ToWord# w#))) @@ -53,25 +53,36 @@ {-# INLINE unsafeChr32 #-} -- | Write a character into the array at the given offset. Returns --- the number of 'Word16's written. +-- the number of 'Word8's written. unsafeWrite :: #if defined(ASSERTS) HasCallStack => #endif A.MArray s -> Int -> Char -> ST s Int -unsafeWrite marr i c - | n < 0x10000 = do - A.unsafeWrite marr i (intToWord16 n) +unsafeWrite marr i c = case utf8Length c of + 1 -> do + let n0 = intToWord8 (ord c) + A.unsafeWrite marr i n0 return 1 - | otherwise = do - A.unsafeWrite marr i lo - A.unsafeWrite marr (i+1) hi + 2 -> do + let (n0, n1) = ord2 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 return 2 - where n = ord c - m = n - 0x10000 - lo = intToWord16 $ (m `shiftR` 10) + 0xD800 - hi = intToWord16 $ (m .&. 0x3FF) + 0xDC00 + 3 -> do + let (n0, n1, n2) = ord3 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 + A.unsafeWrite marr (i+2) n2 + return 3 + _ -> do + let (n0, n1, n2, n3) = ord4 c + A.unsafeWrite marr i n0 + A.unsafeWrite marr (i+1) n1 + A.unsafeWrite marr (i+2) n2 + A.unsafeWrite marr (i+3) n3 + return 4 {-# INLINE unsafeWrite #-} -intToWord16 :: Int -> Word16 -intToWord16 = fromIntegral +intToWord8 :: Int -> Word8 +intToWord8 = fromIntegral diff --git a/src/Data/Text/Internal/Unsafe/Shift.hs b/src/Data/Text/Internal/Unsafe/Shift.hs deleted file mode 100644 --- a/src/Data/Text/Internal/Unsafe/Shift.hs +++ /dev/null @@ -1,48 +0,0 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE DefaultSignatures #-} -{-# LANGUAGE MagicHash #-} - --- | --- Module : Data.Text.Internal.Unsafe.Shift --- Copyright : (c) Bryan O'Sullivan 2009 --- --- License : BSD-style --- Maintainer : bos@serpentine.com --- Stability : experimental --- Portability : GHC --- --- /Warning/: this is an internal module, and does not have a stable --- API or name. Functions in this module may not check or enforce --- preconditions expected by public modules. Use at your own risk! --- --- Fast, unchecked bit shifting functions. - -module Data.Text.Internal.Unsafe.Shift - ( - UnsafeShift(..) - ) where - -import qualified Data.Bits as Bits -import Data.Word - --- | This is a workaround for poor optimisation in GHC 6.8.2. It --- fails to notice constant-width shifts, and adds a test and branch --- to every shift. This imposes about a 10% performance hit. --- --- These functions are undefined when the amount being shifted by is --- greater than the size in bits of a machine Int#. -class UnsafeShift a where - shiftL :: a -> Int -> a - {-# INLINE shiftL #-} - default shiftL :: Bits.Bits a => a -> Int -> a - shiftL = Bits.unsafeShiftL - - shiftR :: a -> Int -> a - {-# INLINE shiftR #-} - default shiftR :: Bits.Bits a => a -> Int -> a - shiftR = Bits.unsafeShiftR - -instance UnsafeShift Word16 where -instance UnsafeShift Word32 where -instance UnsafeShift Word64 where -instance UnsafeShift Int where diff --git a/src/Data/Text/Lazy.hs b/src/Data/Text/Lazy.hs --- a/src/Data/Text/Lazy.hs +++ b/src/Data/Text/Lazy.hs @@ -2,6 +2,7 @@ {-# LANGUAGE BangPatterns, MagicHash, CPP, TypeFamilies #-} {-# LANGUAGE Trustworthy #-} {-# LANGUAGE TemplateHaskellQuotes #-} +{-# LANGUAGE LambdaCase #-} -- | -- Module : Data.Text.Lazy @@ -198,21 +199,26 @@ ) where import Prelude (Char, Bool(..), Maybe(..), String, - Eq(..), Ord(..), Ordering(..), Read(..), Show(..), - (&&), (||), (+), (-), (.), ($), (++), + Eq, (==), Ord(..), Ordering(..), Read(..), Show(..), + (&&), (+), (-), (.), ($), (++), error, flip, fmap, fromIntegral, not, otherwise, quot) import qualified Prelude as P +import Control.Arrow (first) import Control.DeepSeq (NFData(..)) +import Data.Bits (finiteBitSize) import Data.Int (Int64) import qualified Data.List as L import Data.Char (isSpace) import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex, Constr, mkConstr, DataType, mkDataType, Fixity(Prefix)) import Data.Binary (Binary(get, put)) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE import Data.Monoid (Monoid(..)) import Data.Semigroup (Semigroup(..)) import Data.String (IsString(..)) import qualified Data.Text as T +import qualified Data.Text.Array as A import qualified Data.Text.Internal as T import qualified Data.Text.Internal.Fusion.Common as S import qualified Data.Text.Unsafe as T @@ -220,46 +226,25 @@ import Data.Text.Internal.Fusion.Types (PairS(..)) import Data.Text.Internal.Lazy.Fusion (stream, unstream) import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, - foldrChunks, smallChunkSize) + foldrChunks, smallChunkSize, defaultChunkSize, equal) import Data.Text.Internal (firstf, safe, text) import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8) -import qualified Data.Text.Internal.Functions as F import Data.Text.Internal.Lazy.Search (indices) import qualified GHC.CString as GHC import qualified GHC.Exts as Exts import GHC.Prim (Addr#) +import GHC.Stack (HasCallStack) import qualified Language.Haskell.TH.Lib as TH import qualified Language.Haskell.TH.Syntax as TH import Text.Printf (PrintfArg, formatArg, formatString) -#if defined(ASSERTS) -import GHC.Stack (HasCallStack) -#endif - -- $fusion -- --- Most of the functions in this module are subject to /fusion/, --- meaning that a pipeline of such functions will usually allocate at --- most one 'Text' value. --- --- As an example, consider the following pipeline: --- --- > import Data.Text.Lazy as T --- > import Data.Text.Lazy.Encoding as E --- > import Data.ByteString.Lazy (ByteString) --- > --- > countChars :: ByteString -> Int --- > countChars = T.length . T.toUpper . E.decodeUtf8 --- --- From the type signatures involved, this looks like it should --- allocate one 'ByteString' value, and two 'Text' values. However, --- when a module is compiled with optimisation enabled under GHC, the --- two intermediate 'Text' values will be optimised away, and the --- function will be compiled down to a single loop over the source --- 'ByteString'. --- --- Functions that can be fused by the compiler are documented with the --- phrase \"Subject to fusion\". +-- Starting from @text-1.3@ fusion is no longer implicit, +-- and pipelines of transormations usually allocate intermediate 'Text' values. +-- Users, who observe significant changes to performances, +-- are encouraged to use fusion framework explicitly, employing +-- "Data.Text.Internal.Fusion" and "Data.Text.Internal.Fusion.Common". -- $replacement -- @@ -271,7 +256,8 @@ -- points -- () -- as 'Char' values, including code points from this invalid range. --- This means that there are some 'Char' values that are not valid +-- This means that there are some 'Char' values +-- (corresponding to 'Data.Char.Surrogate' category) that are not valid -- Unicode scalar values, and the functions in this module must handle -- those cases. -- @@ -280,12 +266,7 @@ -- that are not valid Unicode scalar values with the replacement -- character \"�\" (U+FFFD). Functions that perform this -- inspection and replacement are documented with the phrase --- \"Performs replacement on invalid scalar values\". --- --- (One reason for this policy of replacement is that internally, a --- 'Text' value is represented as packed UTF-16 data. Values in the --- range U+D800 through U+DFFF are used by UTF-16 to denote surrogate --- code points, and so cannot be represented. The functions replace +-- \"Performs replacement on invalid scalar values\". The functions replace -- invalid scalar values, instead of dropping them, as a security -- measure. For details, see -- .) @@ -295,20 +276,6 @@ -- >>> import qualified Data.Text as T -- >>> :seti -XOverloadedStrings -equal :: Text -> Text -> Bool -equal Empty Empty = True -equal Empty _ = False -equal _ Empty = False -equal (Chunk a as) (Chunk b bs) = - case compare lenA lenB of - LT -> a == (T.takeWord16 lenA b) && - as `equal` Chunk (T.dropWord16 lenA b) bs - EQ -> a == b && as `equal` bs - GT -> T.takeWord16 lenB a == b && - Chunk (T.dropWord16 lenB a) as `equal` bs - where lenA = T.lengthWord16 a - lenB = T.lengthWord16 b - instance Eq Text where (==) = equal {-# INLINE (==) #-} @@ -320,18 +287,14 @@ compareText Empty Empty = EQ compareText Empty _ = LT compareText _ Empty = GT -compareText (Chunk a0 as) (Chunk b0 bs) = outer a0 b0 - where - outer ta@(T.Text arrA offA lenA) tb@(T.Text arrB offB lenB) = go 0 0 - where - go !i !j - | i >= lenA = compareText as (chunk (T.Text arrB (offB+j) (lenB-j)) bs) - | j >= lenB = compareText (chunk (T.Text arrA (offA+i) (lenA-i)) as) bs - | a < b = LT - | a > b = GT - | otherwise = go (i+di) (j+dj) - where T.Iter a di = T.iter ta i - T.Iter b dj = T.iter tb j +compareText (Chunk (T.Text arrA offA lenA) as) (Chunk (T.Text arrB offB lenB) bs) = + A.compare arrA offA arrB offB (min lenA lenB) <> case lenA `compare` lenB of + LT -> compareText as (Chunk (T.Text arrB (offB + lenA) (lenB - lenA)) bs) + EQ -> compareText as bs + GT -> compareText (Chunk (T.Text arrA (offA + lenB) (lenA - lenB)) as) bs +-- This is not a mistake: on contrary to UTF-16 (https://github.com/haskell/text/pull/208), +-- lexicographic ordering of UTF-8 encoded strings matches lexicographic ordering +-- of underlying bytearrays, no decoding is needed. instance Show Text where showsPrec p ps r = showsPrec p (unpack ps) r @@ -407,7 +370,7 @@ -- | /O(n)/ Convert a 'String' into a 'Text'. -- --- Subject to fusion. Performs replacement on invalid scalar values. +-- Performs replacement on invalid scalar values. pack :: #if defined(ASSERTS) HasCallStack => @@ -417,7 +380,6 @@ {-# INLINE [1] pack #-} -- | /O(n)/ Convert a 'Text' into a 'String'. --- Subject to fusion. unpack :: #if defined(ASSERTS) HasCallStack => @@ -447,19 +409,12 @@ unstream (S.streamList (L.map safe [a])) = Chunk (T.singleton a) Empty #-} --- | /O(1)/ Convert a character into a Text. Subject to fusion. +-- | /O(1)/ Convert a character into a Text. -- Performs replacement on invalid scalar values. singleton :: Char -> Text singleton c = Chunk (T.singleton c) Empty {-# INLINE [1] singleton #-} -{-# RULES -"LAZY TEXT singleton -> fused" [~1] forall c. - singleton c = unstream (S.singleton c) -"LAZY TEXT singleton -> unfused" [1] forall c. - unstream (S.singleton c) = singleton c - #-} - -- | /O(c)/ Convert a list of strict 'T.Text's into a lazy 'Text'. fromChunks :: [T.Text] -> Text fromChunks cs = L.foldr chunk Empty cs @@ -481,47 +436,26 @@ -- ----------------------------------------------------------------------------- -- * Basic functions --- | /O(1)/ Adds a character to the front of a 'Text'. Subject to fusion. +-- | /O(1)/ Adds a character to the front of a 'Text'. cons :: Char -> Text -> Text cons c t = Chunk (T.singleton c) t {-# INLINE [1] cons #-} infixr 5 `cons` -{-# RULES -"LAZY TEXT cons -> fused" [~1] forall c t. - cons c t = unstream (S.cons c (stream t)) -"LAZY TEXT cons -> unfused" [1] forall c t. - unstream (S.cons c (stream t)) = cons c t - #-} - -- | /O(n)/ Adds a character to the end of a 'Text'. This copies the --- entire array in the process, unless fused. Subject to fusion. +-- entire array in the process. snoc :: Text -> Char -> Text snoc t c = foldrChunks Chunk (singleton c) t {-# INLINE [1] snoc #-} -{-# RULES -"LAZY TEXT snoc -> fused" [~1] forall t c. - snoc t c = unstream (S.snoc (stream t) c) -"LAZY TEXT snoc -> unfused" [1] forall t c. - unstream (S.snoc (stream t) c) = snoc t c - #-} - --- | /O(n\/c)/ Appends one 'Text' to another. Subject to fusion. +-- | /O(n\/c)/ Appends one 'Text' to another. append :: Text -> Text -> Text append xs ys = foldrChunks Chunk ys xs {-# INLINE [1] append #-} -{-# RULES -"LAZY TEXT append -> fused" [~1] forall t1 t2. - append t1 t2 = unstream (S.append (stream t1) (stream t2)) -"LAZY TEXT append -> unfused" [1] forall t1 t2. - unstream (S.append (stream t1) (stream t2)) = append t1 t2 - #-} - -- | /O(1)/ Returns the first character and rest of a 'Text', or --- 'Nothing' if empty. Subject to fusion. +-- 'Nothing' if empty. uncons :: Text -> Maybe (Char, Text) uncons Empty = Nothing uncons (Chunk t ts) = Just (T.unsafeHead t, ts') @@ -530,41 +464,27 @@ {-# INLINE uncons #-} -- | /O(1)/ Returns the first character of a 'Text', which must be --- non-empty. Subject to fusion. -head :: Text -> Char +-- non-empty. This is a partial function, consider using 'uncons' instead. +head :: HasCallStack => Text -> Char head t = S.head (stream t) {-# INLINE head #-} -- | /O(1)/ Returns all characters after the head of a 'Text', which --- must be non-empty. Subject to fusion. -tail :: Text -> Text +-- must be non-empty. This is a partial function, consider using 'uncons' instead. +tail :: HasCallStack => Text -> Text tail (Chunk t ts) = chunk (T.tail t) ts tail Empty = emptyError "tail" {-# INLINE [1] tail #-} -{-# RULES -"LAZY TEXT tail -> fused" [~1] forall t. - tail t = unstream (S.tail (stream t)) -"LAZY TEXT tail -> unfused" [1] forall t. - unstream (S.tail (stream t)) = tail t - #-} - -- | /O(n\/c)/ Returns all but the last character of a 'Text', which must --- be non-empty. Subject to fusion. -init :: Text -> Text +-- be non-empty. This is a partial function, consider using 'unsnoc' instead. +init :: HasCallStack => Text -> Text init (Chunk t0 ts0) = go t0 ts0 where go t (Chunk t' ts) = Chunk t (go t' ts) go t Empty = chunk (T.init t) Empty init Empty = emptyError "init" {-# INLINE [1] init #-} -{-# RULES -"LAZY TEXT init -> fused" [~1] forall t. - init t = unstream (S.init (stream t)) -"LAZY TEXT init -> unfused" [1] forall t. - unstream (S.init (stream t)) = init t - #-} - -- | /O(n\/c)/ Returns the 'init' and 'last' of a 'Text', or 'Nothing' if -- empty. -- @@ -576,44 +496,27 @@ unsnoc ts@(Chunk _ _) = Just (init ts, last ts) {-# INLINE unsnoc #-} --- | /O(1)/ Tests whether a 'Text' is empty or not. Subject to --- fusion. +-- | /O(1)/ Tests whether a 'Text' is empty or not. null :: Text -> Bool null Empty = True null _ = False {-# INLINE [1] null #-} -{-# RULES -"LAZY TEXT null -> fused" [~1] forall t. - null t = S.null (stream t) -"LAZY TEXT null -> unfused" [1] forall t. - S.null (stream t) = null t - #-} - -- | /O(1)/ Tests whether a 'Text' contains exactly one character. --- Subject to fusion. isSingleton :: Text -> Bool isSingleton = S.isSingleton . stream {-# INLINE isSingleton #-} -- | /O(n\/c)/ Returns the last character of a 'Text', which must be --- non-empty. Subject to fusion. -last :: Text -> Char +-- non-empty. This is a partial function, consider using 'unsnoc' instead. +last :: HasCallStack => Text -> Char last Empty = emptyError "last" last (Chunk t ts) = go t ts where go _ (Chunk t' ts') = go t' ts' go t' Empty = T.last t' {-# INLINE [1] last #-} -{-# RULES -"LAZY TEXT last -> fused" [~1] forall t. - last t = S.last (stream t) -"LAZY TEXT last -> unfused" [1] forall t. - S.last (stream t) = last t - #-} - -- | /O(n)/ Returns the number of characters in a 'Text'. --- Subject to fusion. length :: Text -> Int64 length = foldlChunks go 0 where @@ -622,14 +525,21 @@ {-# INLINE [1] length #-} {-# RULES -"LAZY TEXT length -> fused" [~1] forall t. - length t = S.length (stream t) -"LAZY TEXT length -> unfused" [1] forall t. - S.length (stream t) = length t - #-} +"TEXT length/map -> length" forall f t. + length (map f t) = length t +"TEXT length/zipWith -> length" forall f t1 t2. + length (zipWith f t1 t2) = min (length t1) (length t2) +"TEXT length/replicate -> n" forall n t. + length (replicate n t) = max 0 n P.* length t +"TEXT length/cons -> length+1" forall c t. + length (cons c t) = 1 + length t +"TEXT length/intersperse -> 2*length-1" forall c t. + length (intersperse c t) = max 0 (2 P.* length t - 1) +"TEXT length/intercalate -> n*length" forall s ts. + length (intercalate s ts) = let lenS = length s in max 0 (P.sum (P.map (\t -> length t + lenS) ts) - lenS) + #-} -- | /O(n)/ Compare the count of characters in a 'Text' to a number. --- Subject to fusion. -- -- This function gives the same answer as comparing against the result -- of 'length', but can short circuit if the count of characters is @@ -643,28 +553,33 @@ -- properties of code. -- | /O(n)/ 'map' @f@ @t@ is the 'Text' obtained by applying @f@ to --- each element of @t@. Subject to fusion. Performs replacement on +-- each element of @t@. Performs replacement on -- invalid scalar values. map :: (Char -> Char) -> Text -> Text -map f t = unstream (S.map (safe . f) (stream t)) +map f = foldrChunks (Chunk . T.map f) Empty {-# INLINE [1] map #-} +{-# RULES +"TEXT map/map -> map" forall f g t. + map f (map g t) = map (f . safe . g) t +#-} + -- | /O(n)/ The 'intercalate' function takes a 'Text' and a list of -- 'Text's and concatenates the list after interspersing the first -- argument between each element of the list. intercalate :: Text -> [Text] -> Text -intercalate t = concat . (F.intersperse t) -{-# INLINE intercalate #-} +intercalate t = concat . L.intersperse t +{-# INLINE [1] intercalate #-} -- | /O(n)/ The 'intersperse' function takes a character and places it --- between the characters of a 'Text'. Subject to fusion. Performs +-- between the characters of a 'Text'. Performs -- replacement on invalid scalar values. intersperse :: Char -> Text -> Text intersperse c t = unstream (S.intersperse (safe c) (stream t)) -{-# INLINE intersperse #-} +{-# INLINE [1] intersperse #-} -- | /O(n)/ Left-justify a string to the given length, using the --- specified fill character on the right. Subject to fusion. Performs +-- specified fill character on the right. Performs -- replacement on invalid scalar values. -- -- Examples: @@ -674,17 +589,10 @@ justifyLeft :: Int64 -> Char -> Text -> Text justifyLeft k c t | len >= k = t - | otherwise = t `append` replicateChar (k-len) c + | otherwise = t `append` replicateChunk (k-len) (T.singleton c) where len = length t {-# INLINE [1] justifyLeft #-} -{-# RULES -"LAZY TEXT justifyLeft -> fused" [~1] forall k c t. - justifyLeft k c t = unstream (S.justifyLeftI k c (stream t)) -"LAZY TEXT justifyLeft -> unfused" [1] forall k c t. - unstream (S.justifyLeftI k c (stream t)) = justifyLeft k c t - #-} - -- | /O(n)/ Right-justify a string to the given length, using the -- specified fill character on the left. Performs replacement on -- invalid scalar values. @@ -696,7 +604,7 @@ justifyRight :: Int64 -> Char -> Text -> Text justifyRight k c t | len >= k = t - | otherwise = replicateChar (k-len) c `append` t + | otherwise = replicateChunk (k-len) (T.singleton c) `append` t where len = length t {-# INLINE justifyRight #-} @@ -710,7 +618,7 @@ center :: Int64 -> Char -> Text -> Text center k c t | len >= k = t - | otherwise = replicateChar l c `append` t `append` replicateChar r c + | otherwise = replicateChunk l (T.singleton c) `append` t `append` replicateChunk r (T.singleton c) where len = length t d = k - len r = d `quot` 2 @@ -759,7 +667,8 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -replace :: Text +replace :: HasCallStack + => Text -- ^ @needle@ to search for. If this string is empty, an -- error will occur. -> Text @@ -782,7 +691,7 @@ -- functions may map one input character to two or three output -- characters. --- | /O(n)/ Convert a string to folded case. Subject to fusion. +-- | /O(n)/ Convert a string to folded case. -- -- This function is mainly useful for performing caseless (or case -- insensitive) string comparisons. @@ -802,7 +711,7 @@ {-# INLINE toCaseFold #-} -- | /O(n)/ Convert a string to lower case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The result string may be longer than the input string. For -- instance, the Latin capital letter I with dot above (U+0130) maps @@ -813,7 +722,7 @@ {-# INLINE toLower #-} -- | /O(n)/ Convert a string to upper case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The result string may be longer than the input string. For -- instance, the German eszett (U+00DF) maps to the two-letter @@ -824,7 +733,7 @@ -- | /O(n)/ Convert a string to title case, using simple case --- conversion. Subject to fusion. +-- conversion. -- -- The first letter of the input is converted to title case, as is -- every subsequent letter that immediately follows a non-letter. @@ -850,51 +759,46 @@ -- | /O(n)/ 'foldl', applied to a binary operator, a starting value -- (typically the left-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from left to right. --- Subject to fusion. foldl :: (a -> Char -> a) -> a -> Text -> a foldl f z t = S.foldl f z (stream t) {-# INLINE foldl #-} -- | /O(n)/ A strict version of 'foldl'. --- Subject to fusion. +-- foldl' :: (a -> Char -> a) -> a -> Text -> a foldl' f z t = S.foldl' f z (stream t) {-# INLINE foldl' #-} -- | /O(n)/ A variant of 'foldl' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to fusion. -foldl1 :: (Char -> Char -> Char) -> Text -> Char +-- and thus must be applied to a non-empty 'Text'. +foldl1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldl1 f t = S.foldl1 f (stream t) {-# INLINE foldl1 #-} --- | /O(n)/ A strict version of 'foldl1'. Subject to fusion. -foldl1' :: (Char -> Char -> Char) -> Text -> Char +-- | /O(n)/ A strict version of 'foldl1'. +foldl1' :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldl1' f t = S.foldl1' f (stream t) {-# INLINE foldl1' #-} -- | /O(n)/ 'foldr', applied to a binary operator, a starting value -- (typically the right-identity of the operator), and a 'Text', -- reduces the 'Text' using the binary operator, from right to left. --- Subject to fusion. foldr :: (Char -> a -> a) -> a -> Text -> a foldr f z t = S.foldr f z (stream t) {-# INLINE foldr #-} -- | /O(n)/ A variant of 'foldr' that has no starting value argument, --- and thus must be applied to a non-empty 'Text'. Subject to --- fusion. -foldr1 :: (Char -> Char -> Char) -> Text -> Char +-- and thus must be applied to a non-empty 'Text'. +foldr1 :: HasCallStack => (Char -> Char -> Char) -> Text -> Char foldr1 f t = S.foldr1 f (stream t) {-# INLINE foldr1 #-} -- | /O(n)/ Concatenate a list of 'Text's. concat :: [Text] -> Text -concat = to - where - go Empty css = to css - go (Chunk c cs) css = Chunk c (go cs css) - to [] = Empty - to (cs:css) = go cs css +concat [] = Empty +concat (Empty : css) = concat css +concat (Chunk c Empty : css) = Chunk c (concat css) +concat (Chunk c cs : css) = Chunk c (concat (cs : css)) {-# INLINE concat #-} -- | /O(n)/ Map a function over a 'Text' that results in a 'Text', and @@ -904,31 +808,31 @@ {-# INLINE concatMap #-} -- | /O(n)/ 'any' @p@ @t@ determines whether any character in the --- 'Text' @t@ satisfies the predicate @p@. Subject to fusion. +-- 'Text' @t@ satisfies the predicate @p@. any :: (Char -> Bool) -> Text -> Bool any p t = S.any p (stream t) {-# INLINE any #-} -- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the --- 'Text' @t@ satisfy the predicate @p@. Subject to fusion. +-- 'Text' @t@ satisfy the predicate @p@. all :: (Char -> Bool) -> Text -> Bool all p t = S.all p (stream t) {-# INLINE all #-} -- | /O(n)/ 'maximum' returns the maximum value from a 'Text', which --- must be non-empty. Subject to fusion. -maximum :: Text -> Char +-- must be non-empty. +maximum :: HasCallStack => Text -> Char maximum t = S.maximum (stream t) {-# INLINE maximum #-} -- | /O(n)/ 'minimum' returns the minimum value from a 'Text', which --- must be non-empty. Subject to fusion. -minimum :: Text -> Char +-- must be non-empty. +minimum :: HasCallStack => Text -> Char minimum t = S.minimum (stream t) {-# INLINE minimum #-} -- | /O(n)/ 'scanl' is similar to 'foldl', but returns a list of --- successive reduced values from the left. Subject to fusion. +-- successive reduced values from the left. -- Performs replacement on invalid scalar values. -- -- > scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...] @@ -1003,19 +907,33 @@ -- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input -- @t@ repeated @n@ times. replicate :: Int64 -> Text -> Text -replicate n t - | null t || n <= 0 = empty - | isSingleton t = replicateChar n (head t) - | otherwise = concat (rep 0) - where rep !i | i >= n = [] - | otherwise = t : rep (i+1) +replicate n + | n <= 0 = P.const Empty + | otherwise = \case + Empty -> Empty + Chunk t Empty -> replicateChunk n t + t -> concat (rep n) + where + rep 0 = [] + rep i = t : rep (i - 1) {-# INLINE [1] replicate #-} +replicateChunk :: Int64 -> T.Text -> Text +replicateChunk !n !t@(T.Text _ _ len) + | n <= 0 = Empty + | otherwise = Chunk headChunk $ P.foldr Chunk Empty (L.genericReplicate q normalChunk) + where + perChunk = defaultChunkSize `quot` len + normalChunk = T.replicate perChunk t + (q, r) = n `P.quotRem` intToInt64 perChunk + headChunk = T.replicate (int64ToInt r) t +{-# INLINE replicateChunk #-} + -- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or -- equivalently, the infinite repetition of the original 'Text'. -- -- @since 1.2.0.5 -cycle :: Text -> Text +cycle :: HasCallStack => Text -> Text cycle Empty = emptyError "cycle" cycle t = let t' = foldrChunks Chunk t' t in t' @@ -1030,26 +948,12 @@ iterate f c = let t c' = Chunk (T.singleton c') (t (f c')) in t c --- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the --- value of every element. Subject to fusion. -replicateChar :: Int64 -> Char -> Text -replicateChar n c = unstream (S.replicateCharI n (safe c)) -{-# INLINE replicateChar #-} - -{-# RULES -"LAZY TEXT replicate/singleton -> replicateChar" [~1] forall n c. - replicate n (singleton c) = replicateChar n c -"LAZY TEXT replicate/unstream/singleton -> replicateChar" [~1] forall n c. - replicate n (unstream (S.singleton c)) = replicateChar n c - #-} - -- | /O(n)/, where @n@ is the length of the result. The 'unfoldr' -- function is analogous to the List 'L.unfoldr'. 'unfoldr' builds a -- 'Text' from a seed value. The function takes the element and -- returns 'Nothing' if it is done producing the 'Text', otherwise -- 'Just' @(a,b)@. In this case, @a@ is the next 'Char' in the -- string, and @b@ is the seed value for further production. --- Subject to fusion. -- Performs replacement on invalid scalar values. unfoldr :: (a -> Maybe (Char,a)) -> a -> Text unfoldr f s = unstream (S.unfoldr (firstf safe . f) s) @@ -1060,7 +964,6 @@ -- first argument to 'unfoldrN'. This function is more efficient than -- 'unfoldr' when the maximum length of the result is known and -- correct, otherwise its performance is similar to 'unfoldr'. --- Subject to fusion. -- Performs replacement on invalid scalar values. unfoldrN :: Int64 -> (a -> Maybe (Char,a)) -> a -> Text unfoldrN n f s = unstream (S.unfoldrN n (firstf safe . f) s) @@ -1068,7 +971,7 @@ -- | /O(n)/ 'take' @n@, applied to a 'Text', returns the prefix of the -- 'Text' of length @n@, or the 'Text' itself if @n@ is greater than --- the length of the Text. Subject to fusion. +-- the length of the Text. take :: Int64 -> Text -> Text take i _ | i <= 0 = Empty take i t0 = take' i t0 @@ -1076,18 +979,16 @@ take' :: Int64 -> Text -> Text take' 0 _ = Empty take' _ Empty = Empty - take' n (Chunk t ts) - | n < len = Chunk (T.take (int64ToInt n) t) Empty - | otherwise = Chunk t (take' (n - len) ts) - where len = intToInt64 (T.length t) -{-# INLINE [1] take #-} + take' n (Chunk t@(T.Text arr off _) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then fromStrict (T.Text arr off m) + else Chunk t (take' (n + intToInt64 m) ts) -{-# RULES -"LAZY TEXT take -> fused" [~1] forall n t. - take n t = unstream (S.take n (stream t)) -"LAZY TEXT take -> unfused" [1] forall n t. - unstream (S.take n (stream t)) = take n t - #-} + | n < l = Chunk (T.take (int64ToInt n) t) Empty + | otherwise = Chunk t (take' (n - l) ts) + where l = intToInt64 (T.length t) +{-# INLINE [1] take #-} -- | /O(n)/ 'takeEnd' @n@ @t@ returns the suffix remaining after -- taking @n@ characters from the end of @t@. @@ -1111,7 +1012,7 @@ -- | /O(n)/ 'drop' @n@, applied to a 'Text', returns the suffix of the -- 'Text' after the first @n@ characters, or the empty 'Text' if @n@ --- is greater than the length of the 'Text'. Subject to fusion. +-- is greater than the length of the 'Text'. drop :: Int64 -> Text -> Text drop i t0 | i <= 0 = t0 @@ -1120,18 +1021,16 @@ drop' :: Int64 -> Text -> Text drop' 0 ts = ts drop' _ Empty = Empty - drop' n (Chunk t ts) - | n < len = Chunk (T.drop (int64ToInt n) t) ts - | otherwise = drop' (n - len) ts - where len = intToInt64 (T.length t) -{-# INLINE [1] drop #-} + drop' n (Chunk t@(T.Text arr off len) ts) + | finiteBitSize (0 :: P.Int) == 64, m <- T.measureOff (int64ToInt n) t = + if m >= 0 + then chunk (T.Text arr (off + m) (len - m)) ts + else drop' (n + intToInt64 m) ts -{-# RULES -"LAZY TEXT drop -> fused" [~1] forall n t. - drop n t = unstream (S.drop n (stream t)) -"LAZY TEXT drop -> unfused" [1] forall n t. - unstream (S.drop n (stream t)) = drop n t - #-} + | n < l = Chunk (T.drop (int64ToInt n) t) ts + | otherwise = drop' (n - l) ts + where l = intToInt64 (T.length t) +{-# INLINE [1] drop #-} -- | /O(n)/ 'dropEnd' @n@ @t@ returns the prefix remaining after -- dropping @n@ characters from the end of @t@. @@ -1154,9 +1053,9 @@ T.dropEnd (int64ToInt m) t : ts where l = intToInt64 (T.length t) --- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word16' +-- | /O(n)/ 'dropWords' @n@ returns the suffix with @n@ 'Word8' -- values dropped, or the empty 'Text' if @n@ is greater than the --- number of 'Word16' values present. +-- number of 'Word8' values present. dropWords :: Int64 -> Text -> Text dropWords i t0 | i <= 0 = t0 @@ -1173,7 +1072,7 @@ -- | /O(n)/ 'takeWhile', applied to a predicate @p@ and a 'Text', -- returns the longest prefix (possibly empty) of elements that --- satisfy @p@. Subject to fusion. +-- satisfy @p@. takeWhile :: (Char -> Bool) -> Text -> Text takeWhile p t0 = takeWhile' t0 where takeWhile' Empty = Empty @@ -1184,12 +1083,6 @@ Nothing -> Chunk t (takeWhile' ts) {-# INLINE [1] takeWhile #-} -{-# RULES -"LAZY TEXT takeWhile -> fused" [~1] forall p t. - takeWhile p t = unstream (S.takeWhile p (stream t)) -"LAZY TEXT takeWhile -> unfused" [1] forall p t. - unstream (S.takeWhile p (stream t)) = takeWhile p t - #-} -- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text', -- returns the longest suffix (possibly empty) of elements that -- satisfy @p@. @@ -1202,14 +1095,14 @@ takeWhileEnd p = takeChunk empty . L.reverse . toChunks where takeChunk acc [] = acc takeChunk acc (t:ts) - | T.lengthWord16 t' < T.lengthWord16 t + | T.lengthWord8 t' < T.lengthWord8 t = chunk t' acc | otherwise = takeChunk (Chunk t' acc) ts where t' = T.takeWhileEnd p t {-# INLINE takeWhileEnd #-} -- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after --- 'takeWhile' @p@ @t@. Subject to fusion. +-- 'takeWhile' @p@ @t@. dropWhile :: (Char -> Bool) -> Text -> Text dropWhile p t0 = dropWhile' t0 where dropWhile' Empty = Empty @@ -1219,13 +1112,6 @@ Nothing -> dropWhile' ts {-# INLINE [1] dropWhile #-} -{-# RULES -"LAZY TEXT dropWhile -> fused" [~1] forall p t. - dropWhile p t = unstream (S.dropWhile p (stream t)) -"LAZY TEXT dropWhile -> unfused" [1] forall p t. - unstream (S.dropWhile p (stream t)) = dropWhile p t - #-} - -- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after -- dropping characters that satisfy the predicate @p@ from the end of -- @t@. @@ -1291,7 +1177,7 @@ where len = intToInt64 (T.length t) -- | /O(n)/ 'splitAtWord' @n t@ returns a strict pair whose first --- element is a prefix of @t@ whose chunks contain @n@ 'Word16' +-- element is a prefix of @t@ whose chunks contain @n@ 'Word8' -- values, and whose second is the remainder of the string. splitAtWord :: Int64 -> Text -> PairS Text Text splitAtWord _ Empty = empty :*: empty @@ -1326,7 +1212,7 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -breakOn :: Text -> Text -> (Text, Text) +breakOn :: HasCallStack => Text -> Text -> (Text, Text) breakOn pat src | null pat = emptyError "breakOn" | otherwise = case indices pat src of @@ -1341,7 +1227,7 @@ -- remainder of @haystack@, following the match. -- -- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c") -breakOnEnd :: Text -> Text -> (Text, Text) +breakOnEnd :: HasCallStack => Text -> Text -> (Text, Text) breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src) in (reverse b, reverse a) {-# INLINE breakOnEnd #-} @@ -1367,7 +1253,8 @@ -- towards /O(n*m)/. -- -- The @needle@ parameter may not be empty. -breakOnAll :: Text -- ^ @needle@ to search for +breakOnAll :: HasCallStack + => Text -- ^ @needle@ to search for -> Text -- ^ @haystack@ in which to search -> [(Text, Text)] breakOnAll pat src @@ -1472,7 +1359,8 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -splitOn :: Text +splitOn :: HasCallStack + => Text -- ^ String to split on. If this string is empty, an error -- will occur. -> Text @@ -1524,14 +1412,36 @@ | otherwise -> a : go b {-# INLINE chunksOf #-} --- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at --- newline 'Char's. The resulting strings do not contain newlines. +-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at newline characters +-- @'\\n'@ (LF, line feed). The resulting strings do not contain newlines. +-- +-- 'lines' __does not__ treat @'\\r'@ (CR, carriage return) as a newline character. lines :: Text -> [Text] lines Empty = [] -lines t = let (l,t') = break ((==) '\n') t - in l : if null t' then [] - else lines (tail t') +lines t = NE.toList $ go t + where + go :: Text -> NonEmpty Text + go Empty = Empty :| [] + go (Chunk x xs) + -- x is non-empty, so T.lines x is non-empty as well + | hasNlEnd x = NE.fromList $ P.map fromStrict (T.lines x) ++ lines xs + | otherwise = case unsnocList (T.lines x) of + Nothing -> impossibleError "lines" + Just (ls, l) -> P.foldr (NE.cons . fromStrict) (prependToHead l (go xs)) ls +prependToHead :: T.Text -> NonEmpty Text -> NonEmpty Text +prependToHead l ~(x :| xs) = chunk l x :| xs -- Lazy pattern is crucial! + +unsnocList :: [a] -> Maybe ([a], a) +unsnocList [] = Nothing +unsnocList (x : xs) = Just $ go x xs + where + go y [] = ([], y) + go y (z : zs) = first (y :) (go z zs) + +hasNlEnd :: T.Text -> Bool +hasNlEnd (T.Text arr off len) = A.unsafeIndex arr (off + len - 1) == 0x0A + -- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's -- representing white space. words :: Text -> [Text] @@ -1541,7 +1451,7 @@ -- | /O(n)/ Joins lines, after appending a terminating newline to -- each. unlines :: [Text] -> Text -unlines = concat . L.map (`snoc` '\n') +unlines = concat . L.foldr (\t acc -> t : singleton '\n' : acc) [] {-# INLINE unlines #-} -- | /O(n)/ Joins words using single space characters. @@ -1550,7 +1460,7 @@ {-# INLINE unwords #-} -- | /O(n)/ The 'isPrefixOf' function takes two 'Text's and returns --- 'True' iff the first is a prefix of the second. Subject to fusion. +-- 'True' iff the first is a prefix of the second. isPrefixOf :: Text -> Text -> Bool isPrefixOf Empty _ = True isPrefixOf _ Empty = False @@ -1564,13 +1474,6 @@ ly = T.length y {-# INLINE [1] isPrefixOf #-} -{-# RULES -"LAZY TEXT isPrefixOf -> fused" [~1] forall s t. - isPrefixOf s t = S.isPrefixOf (stream s) (stream t) -"LAZY TEXT isPrefixOf -> unfused" [1] forall s t. - S.isPrefixOf (stream s) (stream t) = isPrefixOf s t - #-} - -- | /O(n)/ The 'isSuffixOf' function takes two 'Text's and returns -- 'True' iff the first is a suffix of the second. isSuffixOf :: Text -> Text -> Bool @@ -1594,11 +1497,6 @@ | otherwise = not . L.null . indices needle $ haystack {-# INLINE [1] isInfixOf #-} -{-# RULES -"LAZY TEXT isInfixOf/singleton -> S.elem/S.stream" [~1] forall n h. - isInfixOf (singleton n) h = S.elem n (S.stream h) - #-} - ------------------------------------------------------------------------------- -- * View patterns @@ -1678,12 +1576,17 @@ -- returns a 'Text' containing those characters that satisfy the -- predicate. filter :: (Char -> Bool) -> Text -> Text -filter p t = unstream (S.filter p (stream t)) -{-# INLINE filter #-} +filter p = foldrChunks (chunk . T.filter p) Empty +{-# INLINE [1] filter #-} +{-# RULES +"TEXT filter/filter -> filter" forall p q t. + filter p (filter q t) = filter (\c -> p c && q c) t +#-} + -- | /O(n)/ The 'find' function takes a predicate and a 'Text', and -- returns the first element in matching the predicate, or 'Nothing' --- if there is no such element. Subject to fusion. +-- if there is no such element. find :: (Char -> Bool) -> Text -> Maybe Char find p t = S.findBy p (stream t) {-# INLINE find #-} @@ -1705,8 +1608,7 @@ {-# INLINE partition #-} -- | /O(n)/ 'Text' index (subscript) operator, starting from 0. --- Subject to fusion. -index :: Text -> Int64 -> Char +index :: HasCallStack => Text -> Int64 -> Char index t n = S.index (stream t) n {-# INLINE index #-} @@ -1716,10 +1618,10 @@ -- -- In (unlikely) bad cases, this function's time complexity degrades -- towards /O(n*m)/. -count :: Text -> Text -> Int64 -count pat src +count :: HasCallStack => Text -> Text -> Int64 +count pat | null pat = emptyError "count" - | otherwise = go 0 (indices pat src) + | otherwise = go 0 . indices pat where go !n [] = n go !n (_:xs) = go (n+1) xs {-# INLINE [1] count #-} @@ -1730,7 +1632,7 @@ #-} -- | /O(n)/ The 'countChar' function returns the number of times the --- query element appears in the given 'Text'. Subject to fusion. +-- query element appears in the given 'Text'. countChar :: Char -> Text -> Int64 countChar c t = S.countChar c (stream t) @@ -1753,10 +1655,10 @@ revChunks :: [T.Text] -> Text revChunks = L.foldl' (flip chunk) Empty -emptyError :: String -> a +emptyError :: HasCallStack => String -> a emptyError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": empty input") -impossibleError :: String -> a +impossibleError :: HasCallStack => String -> a impossibleError fun = P.error ("Data.Text.Lazy." ++ fun ++ ": impossible case") intToInt64 :: Exts.Int -> Int64 diff --git a/src/Data/Text/Lazy/Builder/Int.hs b/src/Data/Text/Lazy/Builder/Int.hs --- a/src/Data/Text/Lazy/Builder/Int.hs +++ b/src/Data/Text/Lazy/Builder/Int.hs @@ -102,15 +102,15 @@ let i = fromIntegral i0; j = i + i unsafeWrite marr off $ get (j + 1) unsafeWrite marr (off - 1) $ get j - get = word8ToWord16 . B.unsafeIndex digits + get = B.unsafeIndex digits -minus, zero :: Word16 +minus, zero :: Word8 {-# INLINE minus #-} {-# INLINE zero #-} minus = 45 zero = 48 -i2w :: (Integral a) => a -> Word16 +i2w :: (Integral a) => a -> Word8 {-# INLINE i2w #-} i2w v = zero + fromIntegral v @@ -242,6 +242,3 @@ | otherwise = loop (d-1) q <> hexDigit r where q = n `quotInt` base r = n `remInt` base - -word8ToWord16 :: Word8 -> Word16 -word8ToWord16 = fromIntegral diff --git a/src/Data/Text/Lazy/Builder/RealFloat.hs b/src/Data/Text/Lazy/Builder/RealFloat.hs --- a/src/Data/Text/Lazy/Builder/RealFloat.hs +++ b/src/Data/Text/Lazy/Builder/RealFloat.hs @@ -34,7 +34,7 @@ | Generic -- ^ Use decimal notation for values between @0.1@ and -- @9,999,999@, and scientific notation otherwise. - deriving (Enum, Read, Show) + deriving (Enum, Read, Show, Bounded) -- | Show a signed 'RealFloat' value to full precision, -- using standard decimal notation for arguments whose absolute value lies diff --git a/src/Data/Text/Lazy/Encoding.hs b/src/Data/Text/Lazy/Encoding.hs --- a/src/Data/Text/Lazy/Encoding.hs +++ b/src/Data/Text/Lazy/Encoding.hs @@ -1,5 +1,6 @@ {-# LANGUAGE BangPatterns,CPP #-} {-# LANGUAGE Trustworthy #-} + -- | -- Module : Data.Text.Lazy.Encoding -- Copyright : (c) 2009, 2010 Bryan O'Sullivan @@ -55,7 +56,6 @@ import Data.Word (Word8) import qualified Data.ByteString as S import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith) import qualified Data.ByteString.Builder.Prim as BP import qualified Data.ByteString.Lazy as B import qualified Data.ByteString.Lazy.Internal as B @@ -78,11 +78,10 @@ -- 'decodeUtf8With' allows the programmer to determine what to do on a -- decoding error. --- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII +-- | Decode a 'ByteString' containing 7-bit ASCII -- encoded text. decodeASCII :: B.ByteString -> Text -decodeASCII = decodeUtf8 -{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-} +decodeASCII = foldr (chunk . TE.decodeASCII) empty . B.toChunks -- | Decode a 'ByteString' containing Latin-1 (aka ISO-8859-1) encoded text. decodeLatin1 :: B.ByteString -> Text @@ -120,10 +119,6 @@ decodeUtf8 = decodeUtf8With strictDecode {-# INLINE[0] decodeUtf8 #-} --- This rule seems to cause performance loss. -{- RULES "LAZY STREAM stream/decodeUtf8' fusion" [1] - forall bs. F.stream (decodeUtf8' bs) = E.streamUtf8 strictDecode bs #-} - -- | Decode a 'ByteString' containing UTF-8 encoded text.. -- -- If the input contains any invalid UTF-8 data, the relevant @@ -143,17 +138,7 @@ -- | Encode text using UTF-8 encoding. encodeUtf8 :: Text -> B.ByteString -encodeUtf8 Empty = B.empty -encodeUtf8 lt@(Chunk t _) = - B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt - where - -- To improve our small string performance, we use a strategy that - -- allocates a buffer that is guaranteed to be large enough for the - -- encoding of the first chunk, but not larger than the default - -- B.smallChunkSize. We clamp the firstChunkSize to ensure that we don't - -- generate too large buffers which hamper streaming. - firstChunkSize = min B.smallChunkSize (4 * (T.length t + 1)) - strategy = B.safeStrategy firstChunkSize B.defaultChunkSize +encodeUtf8 = foldrChunks (B.Chunk . TE.encodeUtf8) B.Empty -- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding. -- diff --git a/src/Data/Text/Lazy/IO.hs b/src/Data/Text/Lazy/IO.hs --- a/src/Data/Text/Lazy/IO.hs +++ b/src/Data/Text/Lazy/IO.hs @@ -10,16 +10,17 @@ -- -- Efficient locale-sensitive support for lazy text I\/O. -- --- Skip past the synopsis for some important notes on performance and --- portability across different versions of GHC. +-- The functions in this module obey the runtime system's locale, +-- character set encoding, and line ending conversion settings. +-- +-- If you know in advance that you will be working with data that has +-- a specific encoding (e.g. UTF-8), and your application is highly +-- performance sensitive, you may find that it is faster to perform +-- I\/O with bytestrings and to encode and decode yourself than to use +-- the functions in this module. module Data.Text.Lazy.IO ( - -- * Performance - -- $performance - - -- * Locale support - -- $locale -- * File-at-a-time operations readFile , writeFile @@ -58,23 +59,18 @@ import System.IO.Error (isEOFError) import System.IO.Unsafe (unsafeInterleaveIO) --- $performance --- --- The functions in this module obey the runtime system's locale, --- character set encoding, and line ending conversion settings. --- --- If you know in advance that you will be working with data that has --- a specific encoding (e.g. UTF-8), and your application is highly --- performance sensitive, you may find that it is faster to perform --- I\/O with bytestrings and to encode and decode yourself than to use --- the functions in this module. --- --- Whether this will hold depends on the version of GHC you are using, --- the platform you are working on, the data you are working with, and --- the encodings you are using, so be sure to test for yourself. - -- | Read a file and return its contents as a string. The file is -- read lazily, as with 'getContents'. +-- +-- Beware that this function (similarly to 'Prelude.readFile') is locale-dependent. +-- Unexpected system locale may cause your application to read corrupted data or +-- throw runtime exceptions about "invalid argument (invalid byte sequence)" +-- or "invalid argument (invalid character)". This is also slow, because GHC +-- first converts an entire input to UTF-32, which is afterwards converted to UTF-8. +-- +-- If your data is UTF-8, +-- using 'Data.Text.Lazy.Encoding.decodeUtf8' '.' 'Data.ByteString.Lazy.readFile' +-- is a much faster and safer alternative. readFile :: FilePath -> IO Text readFile name = openFile name ReadMode >>= hGetContents @@ -83,7 +79,7 @@ writeFile :: FilePath -> Text -> IO () writeFile p = withFile p WriteMode . flip hPutStr --- | Write a string the end of a file. +-- | Write a string to the end of a file. appendFile :: FilePath -> Text -> IO () appendFile p = withFile p AppendMode . flip hPutStr @@ -161,33 +157,3 @@ -- | Write a string to 'stdout', followed by a newline. putStrLn :: Text -> IO () putStrLn = hPutStrLn stdout - --- $locale --- --- /Note/: The behaviour of functions in this module depends on the --- version of GHC you are using. --- --- Beginning with GHC 6.12, text I\/O is performed using the system or --- handle's current locale and line ending conventions. --- --- Under GHC 6.10 and earlier, the system I\/O libraries /do not --- support/ locale-sensitive I\/O or line ending conversion. On these --- versions of GHC, functions in this library all use UTF-8. What --- does this mean in practice? --- --- * All data that is read will be decoded as UTF-8. --- --- * Before data is written, it is first encoded as UTF-8. --- --- * On both reading and writing, the platform's native newline --- conversion is performed. --- --- If you must use a non-UTF-8 locale on an older version of GHC, you --- will have to perform the transcoding yourself, e.g. as follows: --- --- > import qualified Data.ByteString.Lazy as B --- > import Data.Text.Lazy (Text) --- > import Data.Text.Lazy.Encoding (encodeUtf16) --- > --- > putStr_Utf16LE :: Text -> IO () --- > putStr_Utf16LE t = B.putStr (encodeUtf16LE t) diff --git a/src/Data/Text/Lazy/Read.hs b/src/Data/Text/Lazy/Read.hs --- a/src/Data/Text/Lazy/Read.hs +++ b/src/Data/Text/Lazy/Read.hs @@ -1,5 +1,7 @@ {-# LANGUAGE OverloadedStrings, CPP #-} -{-# LANGUAGE Safe #-} +{-# LANGUAGE Trustworthy #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE UnboxedTuples #-} -- | -- Module : Data.Text.Lazy.Read @@ -21,11 +23,15 @@ ) where import Control.Monad (liftM) -import Data.Char (isDigit, isHexDigit) +import Data.Char (ord) import Data.Int (Int8, Int16, Int32, Int64) import Data.Ratio ((%)) import Data.Text.Internal.Read +import Data.Text.Array as A import Data.Text.Lazy as T +import Data.Text.Internal.Lazy as T (Text(..)) +import qualified Data.Text.Internal as T (Text(..)) +import qualified Data.Text.Internal.Private as T (spanAscii_) import Data.Word (Word, Word8, Word16, Word32, Word64) -- | Read some text. If the read succeeds, return its value and the @@ -59,7 +65,7 @@ decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.span isDigit txt + where (# h, t #) = spanAscii_ (\w -> w - ord8 '0' < 10) txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading @@ -97,7 +103,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t) - where (h,t) = T.span isHexDigit txt + where (# h, t #) = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) -- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and @@ -156,26 +162,30 @@ {-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} {-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} signa p = do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - if sign == '+' then p else negate `liftM` p + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + if sign == ord8 '+' then p else negate `liftM` p -char :: (Char -> Bool) -> Parser Char -char p = P $ \t -> case T.uncons t of - Just (c,t') | p c -> Right (c,t') - _ -> Left "character does not match" +charAscii :: (Word8 -> Bool) -> Parser Word8 +charAscii p = P $ \case + Empty -> Left "character does not match" + -- len is > 0, unless the internal invariant of Text is violated + Chunk (T.Text arr off len) ts -> let c = A.unsafeIndex arr off in + if p c + then Right (c, if len <= 1 then ts else Chunk (T.Text arr (off + 1) (len - 1)) ts) + else Left "character does not match" floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') real <- P decimal T fraction fracDigits <- perhaps (T 0 0) $ do - _ <- char (=='.') - digits <- P $ \t -> Right (int64ToInt . T.length $ T.takeWhile isDigit t, t) + _ <- charAscii (== ord8 '.') + digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in int64ToInt (T.length hd), t) n <- P decimal return $ T n digits - let e c = c == 'e' || c == 'E' - power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int) + let e c = c == ord8 'e' || c == ord8 'E' + power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int) let n = if fracDigits == 0 then if power == 0 then fromInteger real @@ -183,9 +193,23 @@ else if power == 0 then f real fraction (10 ^ fracDigits) else f real fraction (10 ^ fracDigits) * (10 ^^ power) - return $! if sign == '+' + return $! if sign == ord8 '+' then n else -n int64ToInt :: Int64 -> Int int64ToInt = fromIntegral + +ord8 :: Char -> Word8 +ord8 = fromIntegral . ord + +-- | For the sake of performance this function does not check +-- that a char is in ASCII range; it is a responsibility of @p@. +spanAscii_ :: (Word8 -> Bool) -> Text -> (# Text, Text #) +spanAscii_ p = loop + where + loop Empty = (# Empty, Empty #) + loop (Chunk t ts) = let (# t', t''@(T.Text _ _ len) #) = T.spanAscii_ p t in + if len == 0 + then let (# ts', ts'' #) = loop ts in (# Chunk t ts', ts'' #) + else (# Chunk t' Empty, Chunk t'' ts #) diff --git a/src/Data/Text/Read.hs b/src/Data/Text/Read.hs --- a/src/Data/Text/Read.hs +++ b/src/Data/Text/Read.hs @@ -21,11 +21,13 @@ ) where import Control.Monad (liftM) -import Data.Char (isDigit, isHexDigit) +import Data.Char (ord) import Data.Int (Int8, Int16, Int32, Int64) import Data.Ratio ((%)) import Data.Text as T -import Data.Text.Internal.Private (span_) +import Data.Text.Internal as T (Text(..)) +import Data.Text.Array as A +import Data.Text.Internal.Private (spanAscii_) import Data.Text.Internal.Read import Data.Word (Word, Word8, Word16, Word32, Word64) @@ -60,7 +62,7 @@ decimal txt | T.null h = Left "input does not start with a digit" | otherwise = Right (T.foldl' go 0 h, t) - where (# h,t #) = span_ isDigit txt + where (# h,t #) = spanAscii_ (\w -> w - ord8 '0' < 10) txt go n d = (n * 10 + fromIntegral (digitToInt d)) -- | Read a hexadecimal integer, consisting of an optional leading @@ -107,7 +109,7 @@ hex txt | T.null h = Left "input does not start with a hexadecimal digit" | otherwise = Right (T.foldl' go 0 h, t) - where (# h,t #) = span_ isHexDigit txt + where (# h,t #) = spanAscii_ (\w -> w - ord8 '0' < 10 || w - ord8 'A' < 6 || w - ord8 'a' < 6) txt go n d = (n * 16 + fromIntegral (hexDigitToInt d)) -- | Read an optional leading sign character (@\'-\'@ or @\'+\'@) and @@ -166,26 +168,27 @@ {-# SPECIALIZE signa :: Parser Int64 -> Parser Int64 #-} {-# SPECIALIZE signa :: Parser Integer -> Parser Integer #-} signa p = do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') - if sign == '+' then p else negate `liftM` p + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') + if sign == ord8 '+' then p else negate `liftM` p -char :: (Char -> Bool) -> Parser Char -char p = P $ \t -> case T.uncons t of - Just (c,t') | p c -> Right (c,t') - _ -> Left "character does not match" +charAscii :: (Word8 -> Bool) -> Parser Word8 +charAscii p = P $ \(Text arr off len) -> let c = A.unsafeIndex arr off in + if len > 0 && p c + then Right (c, Text arr (off + 1) (len - 1)) + else Left "character does not match" floaty :: Fractional a => (Integer -> Integer -> Integer -> a) -> Reader a {-# INLINE floaty #-} floaty f = runP $ do - sign <- perhaps '+' $ char (\c -> c == '-' || c == '+') + sign <- perhaps (ord8 '+') $ charAscii (\c -> c == ord8 '-' || c == ord8 '+') real <- P decimal T fraction fracDigits <- perhaps (T 0 0) $ do - _ <- char (=='.') - digits <- P $ \t -> Right (T.length $ T.takeWhile isDigit t, t) + _ <- charAscii (== ord8 '.') + digits <- P $ \t -> Right (let (# hd, _ #) = spanAscii_ (\w -> w - ord8 '0' < 10) t in T.length hd, t) n <- P decimal return $ T n digits - let e c = c == 'e' || c == 'E' - power <- perhaps 0 (char e >> signa (P decimal) :: Parser Int) + let e c = c == ord8 'e' || c == ord8 'E' + power <- perhaps 0 (charAscii e >> signa (P decimal) :: Parser Int) let n = if fracDigits == 0 then if power == 0 then fromInteger real @@ -193,6 +196,9 @@ else if power == 0 then f real fraction (10 ^ fracDigits) else f real fraction (10 ^ fracDigits) * (10 ^^ power) - return $! if sign == '+' + return $! if sign == ord8 '+' then n else -n + +ord8 :: Char -> Word8 +ord8 = fromIntegral . ord diff --git a/src/Data/Text/Show.hs b/src/Data/Text/Show.hs --- a/src/Data/Text/Show.hs +++ b/src/Data/Text/Show.hs @@ -1,5 +1,7 @@ {-# LANGUAGE CPP, MagicHash #-} {-# LANGUAGE Trustworthy #-} +{-# LANGUAGE CApiFFI #-} +{-# LANGUAGE ViewPatterns #-} {-# OPTIONS_GHC -fno-warn-orphans #-} @@ -17,15 +19,23 @@ singleton , unpack , unpackCString# + , unpackCStringAscii# ) where -import Control.Monad.ST (ST) +import Control.Monad.ST (ST, runST) import Data.Text.Internal (Text(..), empty_, safe) +import Data.Text.Internal.Encoding.Utf8 (utf8Length) import Data.Text.Internal.Fusion (stream, unstream) import Data.Text.Internal.Unsafe.Char (unsafeWrite) -import GHC.Prim (Addr#) +import GHC.Exts (Ptr(..), Int(..), Addr#, indexWord8OffAddr#) +import GHC.Word (Word8(..)) import qualified Data.Text.Array as A import qualified Data.Text.Internal.Fusion.Common as S +#if !MIN_VERSION_ghc_prim(0,7,0) +import Data.Text.Internal.Unsafe (inlinePerformIO) +import Foreign.C.String (CString) +import Foreign.C.Types (CSize(..)) +#endif import qualified GHC.CString as GHC @@ -36,7 +46,7 @@ instance Show Text where showsPrec p ps r = showsPrec p (unpack ps) r --- | /O(n)/ Convert a 'Text' into a 'String'. Subject to fusion. +-- | /O(n)/ Convert a 'Text' into a 'String'. unpack :: #if defined(ASSERTS) HasCallStack => @@ -45,18 +55,66 @@ unpack = S.unstreamList . stream {-# INLINE [1] unpack #-} --- | /O(n)/ Convert a literal string into a 'Text'. --- --- This is exposed solely for people writing GHC rewrite rules. +-- | /O(n)/ Convert a null-terminated +-- +-- (but with a standard UTF-8 representation of characters from supplementary planes) +-- string to a 'Text'. Counterpart to 'GHC.unpackCStringUtf8#'. +-- No validation is performed, malformed input can lead to memory access violation. -- -- @since 1.2.1.1 unpackCString# :: Addr# -> Text -unpackCString# addr# = unstream (S.streamCString# addr#) -{-# NOINLINE unpackCString# #-} +unpackCString# addr# = runST $ do + let l = addrLen addr# + at (I# i#) = W8# (indexWord8OffAddr# addr# i#) + marr <- A.new l + let go srcOff@(at -> w8) dstOff + | srcOff >= l + = return dstOff + -- Surrogate halves take 3 bytes and are replaced by \xfffd (also 3 bytes long). + -- Cf. Data.Text.Internal.safe + | w8 == 0xed, at (srcOff + 1) >= 0xa0 = do + A.unsafeWrite marr dstOff 0xef + A.unsafeWrite marr (dstOff + 1) 0xbf + A.unsafeWrite marr (dstOff + 2) 0xbd + go (srcOff + 3) (dstOff + 3) + -- Byte sequence "\xc0\x80" is used to represent NUL + | w8 == 0xc0, at (srcOff + 1) == 0x80 + = A.unsafeWrite marr dstOff 0 >> go (srcOff + 2) (dstOff + 1) + | otherwise + = A.unsafeWrite marr dstOff w8 >> go (srcOff + 1) (dstOff + 1) + actualLen <- go 0 0 + A.shrinkM marr actualLen + arr <- A.unsafeFreeze marr + return $ Text arr 0 actualLen +{-# INLINE unpackCString# #-} +-- | /O(n)/ Convert a null-terminated ASCII string to a 'Text'. +-- Counterpart to 'GHC.unpackCString#'. +-- No validation is performed, malformed input can lead to memory access violation. +-- +-- @since 2.0 +unpackCStringAscii# :: Addr# -> Text +unpackCStringAscii# addr# = Text ba 0 l + where + l = addrLen addr# + ba = runST $ do + marr <- A.new l + A.copyFromPointer marr 0 (Ptr addr#) l + A.unsafeFreeze marr +{-# INLINE unpackCStringAscii# #-} + +addrLen :: Addr# -> Int +#if MIN_VERSION_ghc_prim(0,7,0) +addrLen addr# = I# (GHC.cstringLength# addr#) +#else +addrLen addr# = fromIntegral (inlinePerformIO (c_strlen (Ptr addr#))) + +foreign import capi unsafe "string.h strlen" c_strlen :: CString -> IO CSize +#endif + {-# RULES "TEXT literal" [1] forall a. unstream (S.map safe (S.streamList (GHC.unpackCString# a))) - = unpackCString# a #-} + = unpackCStringAscii# a #-} {-# RULES "TEXT literal UTF8" [1] forall a. unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a))) @@ -70,7 +128,7 @@ unstream (S.map safe (S.streamList [a])) = singleton_ a #-} --- | /O(1)/ Convert a character into a Text. Subject to fusion. +-- | /O(1)/ Convert a character into a Text. -- Performs replacement on invalid scalar values. singleton :: #if defined(ASSERTS) @@ -95,7 +153,6 @@ x = do arr <- A.new len _ <- unsafeWrite arr 0 d return arr - len | d < '\x10000' = 1 - | otherwise = 2 + len = utf8Length d d = safe c {-# NOINLINE singleton_ #-} diff --git a/src/Data/Text/Unsafe.hs b/src/Data/Text/Unsafe.hs --- a/src/Data/Text/Unsafe.hs +++ b/src/Data/Text/Unsafe.hs @@ -15,24 +15,26 @@ , unsafeDupablePerformIO , Iter(..) , iter + , iterArray , iter_ , reverseIter + , reverseIterArray , reverseIter_ , unsafeHead , unsafeTail - , lengthWord16 - , takeWord16 - , dropWord16 + , lengthWord8 + , takeWord8 + , dropWord8 ) where #if defined(ASSERTS) import Control.Exception (assert) import GHC.Stack (HasCallStack) #endif -import Data.Text.Internal.Encoding.Utf16 (chr2) +import Data.Text.Internal.Encoding.Utf8 (chr2, chr3, chr4, utf8LengthByLeader) import Data.Text.Internal (Text(..)) import Data.Text.Internal.Unsafe (inlineInterleaveST, inlinePerformIO) -import Data.Text.Internal.Unsafe.Char (unsafeChr) +import Data.Text.Internal.Unsafe.Char (unsafeChr8) import qualified Data.Text.Array as A import GHC.IO (unsafeDupablePerformIO) @@ -40,11 +42,15 @@ -- omits the check for the empty case, so there is an obligation on -- the programmer to provide a proof that the 'Text' is non-empty. unsafeHead :: Text -> Char -unsafeHead (Text arr off _len) - | m < 0xD800 || m > 0xDBFF = unsafeChr m - | otherwise = chr2 m n - where m = A.unsafeIndex arr off - n = A.unsafeIndex arr (off+1) +unsafeHead (Text arr off _len) = case utf8LengthByLeader m0 of + 1 -> unsafeChr8 m0 + 2 -> chr2 m0 m1 + 3 -> chr3 m0 m1 m2 + _ -> chr4 m0 m1 m2 m3 + where m0 = A.unsafeIndex arr off + m1 = A.unsafeIndex arr (off+1) + m2 = A.unsafeIndex arr (off+2) + m3 = A.unsafeIndex arr (off+3) {-# INLINE unsafeHead #-} -- | /O(1)/ A variant of 'tail' for non-empty 'Text'. 'unsafeTail' @@ -60,8 +66,9 @@ {-# INLINE unsafeTail #-} data Iter = Iter {-# UNPACK #-} !Char {-# UNPACK #-} !Int + deriving (Show) --- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-16 +-- | /O(1)/ Iterate (unsafely) one step forwards through a UTF-8 -- array, returning the current character and the delta to add to give -- the next offset to iterate at. iter :: @@ -69,61 +76,87 @@ HasCallStack => #endif Text -> Int -> Iter -iter (Text arr off _len) i - | m < 0xD800 || m > 0xDBFF = Iter (unsafeChr m) 1 - | otherwise = Iter (chr2 m n) 2 - where m = A.unsafeIndex arr j - n = A.unsafeIndex arr k - j = off + i - k = j + 1 +iter (Text arr off _len) i = iterArray arr (off + i) {-# INLINE iter #-} --- | /O(1)/ Iterate one step through a UTF-16 array, returning the +-- | @since 2.0 +iterArray :: A.Array -> Int -> Iter +iterArray arr j = Iter chr l + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j+1) + m2 = A.unsafeIndex arr (j+2) + m3 = A.unsafeIndex arr (j+3) + l = utf8LengthByLeader m0 + chr = case l of + 1 -> unsafeChr8 m0 + 2 -> chr2 m0 m1 + 3 -> chr3 m0 m1 m2 + _ -> chr4 m0 m1 m2 m3 +{-# INLINE iterArray #-} + +-- | /O(1)/ Iterate one step through a UTF-8 array, returning the -- delta to add to give the next offset to iterate at. iter_ :: Text -> Int -> Int -iter_ (Text arr off _len) i | m < 0xD800 || m > 0xDBFF = 1 - | otherwise = 2 +iter_ (Text arr off _len) i = utf8LengthByLeader m where m = A.unsafeIndex arr (off+i) {-# INLINE iter_ #-} --- | /O(1)/ Iterate one step backwards through a UTF-16 array, +-- | /O(1)/ Iterate one step backwards through a UTF-8 array, -- returning the current character and the delta to add (i.e. a -- negative number) to give the next offset to iterate at. -reverseIter :: Text -> Int -> (Char,Int) -reverseIter (Text arr off _len) i - | m < 0xDC00 || m > 0xDFFF = (unsafeChr m, -1) - | otherwise = (chr2 n m, -2) - where m = A.unsafeIndex arr j - n = A.unsafeIndex arr k - j = off + i - k = j - 1 +reverseIter :: Text -> Int -> Iter +reverseIter (Text arr off _len) i = reverseIterArray arr (off + i) {-# INLINE reverseIter #-} --- | /O(1)/ Iterate one step backwards through a UTF-16 array, +-- | @since 2.0 +reverseIterArray :: A.Array -> Int -> Iter +reverseIterArray arr j + | m0 < 0x80 = Iter (unsafeChr8 m0) (-1) + | m1 >= 0xC0 = Iter (chr2 m1 m0) (-2) + | m2 >= 0xC0 = Iter (chr3 m2 m1 m0) (-3) + | otherwise = Iter (chr4 m3 m2 m1 m0) (-4) + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j-1) + m2 = A.unsafeIndex arr (j-2) + m3 = A.unsafeIndex arr (j-3) +{-# INLINE reverseIterArray #-} + +-- | /O(1)/ Iterate one step backwards through a UTF-8 array, -- returning the delta to add (i.e. a negative number) to give the -- next offset to iterate at. -- -- @since 1.1.1.0 reverseIter_ :: Text -> Int -> Int reverseIter_ (Text arr off _len) i - | m < 0xDC00 || m > 0xDFFF = -1 - | otherwise = -2 - where m = A.unsafeIndex arr (off+i) + | m0 < 0x80 = -1 + | m1 >= 0xC0 = -2 + | m2 >= 0xC0 = -3 + | otherwise = -4 + where m0 = A.unsafeIndex arr j + m1 = A.unsafeIndex arr (j-1) + m2 = A.unsafeIndex arr (j-2) + j = off + i {-# INLINE reverseIter_ #-} --- | /O(1)/ Return the length of a 'Text' in units of 'Word16'. This +-- | /O(1)/ Return the length of a 'Text' in units of 'Word8'. This -- is useful for sizing a target array appropriately before using -- 'unsafeCopyToPtr'. -lengthWord16 :: Text -> Int -lengthWord16 (Text _arr _off len) = len -{-# INLINE lengthWord16 #-} +-- +-- @since 2.0 +lengthWord8 :: Text -> Int +lengthWord8 (Text _arr _off len) = len +{-# INLINE lengthWord8 #-} --- | /O(1)/ Unchecked take of 'k' 'Word16's from the front of a 'Text'. -takeWord16 :: Int -> Text -> Text -takeWord16 k (Text arr off _len) = Text arr off k -{-# INLINE takeWord16 #-} +-- | /O(1)/ Unchecked take of 'k' 'Word8's from the front of a 'Text'. +-- +-- @since 2.0 +takeWord8 :: Int -> Text -> Text +takeWord8 k (Text arr off _len) = Text arr off k +{-# INLINE takeWord8 #-} --- | /O(1)/ Unchecked drop of 'k' 'Word16's from the front of a 'Text'. -dropWord16 :: Int -> Text -> Text -dropWord16 k (Text arr off len) = Text arr (off+k) (len-k) -{-# INLINE dropWord16 #-} +-- | /O(1)/ Unchecked drop of 'k' 'Word8's from the front of a 'Text'. +-- +-- @since 2.0 +dropWord8 :: Int -> Text -> Text +dropWord8 k (Text arr off len) = Text arr (off+k) (len-k) +{-# INLINE dropWord8 #-} diff --git a/tests/Tests.hs b/tests/Tests.hs --- a/tests/Tests.hs +++ b/tests/Tests.hs @@ -1,5 +1,5 @@ --- | Provides a simple main function which runs all the tests --- +{-# LANGUAGE CPP #-} + module Main ( main ) where @@ -10,14 +10,9 @@ import qualified Tests.Properties as Properties import qualified Tests.Regressions as Regressions -import qualified Tests.Inspection.Strict as InspectionStrict -import qualified Tests.Inspection.Lazy as InspectionLazy - main :: IO () main = defaultMain $ testGroup "All" [ Lift.tests , Properties.tests , Regressions.tests - , InspectionStrict.tests - , InspectionLazy.tests ] diff --git a/tests/Tests/Inspection/Lazy.hs b/tests/Tests/Inspection/Lazy.hs deleted file mode 100644 --- a/tests/Tests/Inspection/Lazy.hs +++ /dev/null @@ -1,181 +0,0 @@ -{-# LANGUAGE TemplateHaskell #-} -{-# OPTIONS_GHC -O -fno-warn-unused-top-binds -fno-warn-missing-signatures -fno-warn-name-shadowing -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} - -module Tests.Inspection.Lazy (tests) where - -import Prelude hiding (all, any, drop, dropWhile, filter, foldl, foldl1, foldr, foldr1, head, init, iterate, last, length, map, maximum, minimum, null, replicate, reverse, scanl, scanl1, scanr, scanr1, tail, take, takeWhile) -import Data.Char (isAscii) - -import qualified Data.Text.Lazy as T - -import Test.Tasty -import Test.Tasty.Inspection - -maximum_unfoldr = T.maximum . unfoldr -isPrefixOf_unfoldr = isPrefixOf . unfoldr -any_filter_init_pack = any . filter . T.init . T.pack -isPrefixOf_intersperse_unfoldr = isPrefixOf . intersperse . unfoldr -compareLength_take_pack = compareLength . take . T.pack -index_toCaseFold_singleton = index . T.toCaseFold . T.singleton -unpack_take_pack = T.unpack . take . T.pack -foldl_singleton = foldl . T.singleton -foldr_filter_singleton = foldr . filter . T.singleton -foldr1_stripStart_singleton = foldr1 . T.stripStart . T.singleton -foldl_drop_singleton = foldl . drop . T.singleton -all_toCaseFold_unfoldr = all . T.toCaseFold . unfoldr -foldl_pack = foldl . T.pack -last_toUpper_toLower_unfoldrN = T.last . T.toUpper . T.toLower . unfoldrN -null_map_unfoldr = T.null . map . unfoldr -any_toTitle_tail_unfoldrN = any . T.toTitle . T.tail . unfoldrN -head_filter_pack = T.head . filter . T.pack -foldr1_toTitle_stripStart_singleton = foldr1 . T.toTitle . T.stripStart . T.singleton -compareLength_snoc_unfoldr = compareLength . snoc . unfoldr -compareLength_intersperse_toUpper_unfoldr = compareLength . intersperse . T.toUpper . unfoldr -compareLength_takeWhile_intersperse_unfoldr = compareLength . takeWhile . intersperse . unfoldr -all_init_map_unfoldrN = all . T.init . map . unfoldrN -foldr_scanl_unfoldrN = foldr . scanl . unfoldrN -compareLength_pack = compareLength . T.pack -foldr_cons_singleton = foldr . cons . T.singleton -last_tail_unfoldrN = T.last . T.tail . unfoldrN -foldr1_take_unfoldr = foldr1 . take . unfoldr -null_unfoldrN = T.null . unfoldrN -foldr1_init_singleton = foldr1 . T.init . T.singleton -all_cons_dropWhile_singleton = all . cons . dropWhile . T.singleton -head_append_toTitle_singleton = T.head . append . T.toTitle . T.singleton -foldl1_tail_intersperse_unfoldr = foldl1 . T.tail . intersperse . unfoldr -foldr1_drop_singleton = foldr1 . drop . T.singleton -minimum_tail_pack = T.minimum . T.tail . T.pack -minimum_pack = T.minimum . T.pack -length_takeWhile_singleton = T.length . takeWhile . T.singleton -foldl_take_toCaseFold_unfoldr = foldl . take . T.toCaseFold . unfoldr -foldr_intersperse_singleton = foldr . intersperse . T.singleton -compareLength_cons_snoc_singleton = compareLength . cons . snoc . T.singleton -last_map_append_unfoldrN = T.last . map . append . unfoldrN -find_justifyLeft_takeWhile_pack = find . justifyLeft . takeWhile . T.pack -head_append_unfoldr = T.head . append . unfoldr -minimum_justifyLeft_drop_unfoldrN = T.minimum . justifyLeft . drop . unfoldrN -null_singleton = T.null . T.singleton -last_map_justifyLeft_pack = T.last . map . justifyLeft . T.pack -foldl1'_tail_unfoldrN = foldl1' . T.tail . unfoldrN -maximum_take_unfoldrN = T.maximum . take . unfoldrN -foldl'_unfoldrN = foldl' . unfoldrN -foldl1'_filter_intersperse_singleton = foldl1' . filter . intersperse . T.singleton -length_singleton = T.length . T.singleton -foldr1_pack = foldr1 . T.pack -foldl1'_singleton = foldl1' . T.singleton -foldr1_scanl_singleton = foldr1 . scanl . T.singleton -minimum_tail_singleton = T.minimum . T.tail . T.singleton -any_pack = any . T.pack -length_pack = T.length . T.pack -head_unfoldrN = T.head . unfoldrN -head_map_pack = T.head . map . T.pack -foldr_stripStart_toCaseFold_singleton = foldr . T.stripStart . T.toCaseFold . T.singleton -any_append_pack = any . append . T.pack -all_drop_toUpper_unfoldrN = all . drop . T.toUpper . unfoldrN -foldl1'_filter_filter_unfoldr = foldl1' . filter . filter . unfoldr -any_singleton = any . T.singleton -any_toTitle_scanl_unfoldr = any . T.toTitle . scanl . unfoldr -foldr1_cons_pack = foldr1 . cons . T.pack -foldl1'_toTitle_dropWhile_singleton = foldl1' . T.toTitle . dropWhile . T.singleton -length_justifyLeft_unfoldr = T.length . justifyLeft . unfoldr -foldl1'_justifyLeft_pack = foldl1' . justifyLeft . T.pack -foldr_map_toTitle_unfoldrN = foldr . map . T.toTitle . unfoldrN -head_singleton = T.head . T.singleton -foldl'_singleton = foldl' . T.singleton -foldr1_dropWhile_intersperse_pack = foldr1 . dropWhile . intersperse . T.pack -foldl1'_pack = foldl1' . T.pack -head_replicate_singleton = T.head . replicate . T.singleton -unpack_toUpper_snoc_singleton = T.unpack . T.toUpper . snoc . T.singleton -null_empty = T.null . empty -maximum_singleton = T.maximum . T.singleton -isPrefixOf_init_singleton = isPrefixOf . T.init . T.singleton -minimum_unfoldr = T.minimum . unfoldr -foldl_stripStart_snoc_singleton = foldl . T.stripStart . snoc . T.singleton -any_toUpper_unfoldr = any . T.toUpper . unfoldr -all_unfoldr = all . unfoldr -minimum_toLower_unfoldrN = T.minimum . T.toLower . unfoldrN -null_pack = T.null . T.pack -index_dropWhile_unfoldr = index . dropWhile . unfoldr -minimum_filter_toUpper_singleton = T.minimum . filter . T.toUpper . T.singleton -head_pack = T.head . T.pack -foldl1'_toTitle_singleton = foldl1' . T.toTitle . T.singleton -find_unfoldr = find . unfoldr -isPrefixOf_unfoldrN = isPrefixOf . unfoldrN -unpack_append_pack = T.unpack . append . T.pack -any_unfoldr = any . unfoldr -length_unfoldrN = T.length . unfoldrN -minimum_singleton = T.minimum . T.singleton -head_snoc_toUpper_singleton = T.head . snoc . T.toUpper . T.singleton -maximum_unfoldrN = T.maximum . unfoldrN -all_take_pack = all . take . T.pack -isPrefixOf_pack = isPrefixOf . T.pack -foldr_init_pack = foldr . T.init . T.pack -foldl1'_filter_pack = foldl1' . filter . T.pack - -tests :: TestTree -tests = testGroup "Lazy fusion" [$(inspectNames (`hasNoType` ''T.Text) - ['maximum_unfoldr, 'isPrefixOf_unfoldr, 'any_filter_init_pack, 'isPrefixOf_intersperse_unfoldr, 'compareLength_take_pack, 'index_toCaseFold_singleton, 'unpack_take_pack, 'foldl_singleton, 'foldr_filter_singleton, 'foldr1_stripStart_singleton, 'foldl_drop_singleton, 'all_toCaseFold_unfoldr, 'foldl_pack, 'last_toUpper_toLower_unfoldrN, 'null_map_unfoldr, 'any_toTitle_tail_unfoldrN, 'head_filter_pack, 'foldr1_toTitle_stripStart_singleton, 'compareLength_snoc_unfoldr, 'compareLength_intersperse_toUpper_unfoldr, 'compareLength_takeWhile_intersperse_unfoldr, 'all_init_map_unfoldrN, 'foldr_scanl_unfoldrN, 'compareLength_pack, 'foldr_cons_singleton, 'last_tail_unfoldrN, 'foldr1_take_unfoldr, 'null_unfoldrN, 'foldr1_init_singleton, 'all_cons_dropWhile_singleton, 'head_append_toTitle_singleton, 'foldl1_tail_intersperse_unfoldr, 'foldr1_drop_singleton, 'minimum_tail_pack, 'minimum_pack, 'length_takeWhile_singleton, 'foldl_take_toCaseFold_unfoldr, 'foldr_intersperse_singleton, 'compareLength_cons_snoc_singleton, 'last_map_append_unfoldrN, 'find_justifyLeft_takeWhile_pack, 'head_append_unfoldr, 'minimum_justifyLeft_drop_unfoldrN, 'null_singleton, 'last_map_justifyLeft_pack, 'foldl1'_tail_unfoldrN, 'maximum_take_unfoldrN, 'foldl'_unfoldrN, 'foldl1'_filter_intersperse_singleton, 'length_singleton, 'foldr1_pack, 'foldl1'_singleton, 'foldr1_scanl_singleton, 'minimum_tail_singleton, 'any_pack, 'length_pack, 'head_unfoldrN, 'head_map_pack, 'foldr_stripStart_toCaseFold_singleton, 'any_append_pack, 'all_drop_toUpper_unfoldrN, 'foldl1'_filter_filter_unfoldr, 'any_singleton, 'any_toTitle_scanl_unfoldr, 'foldr1_cons_pack, 'foldl1'_toTitle_dropWhile_singleton, 'length_justifyLeft_unfoldr, 'foldl1'_justifyLeft_pack, 'foldr_map_toTitle_unfoldrN, 'head_singleton, 'foldl'_singleton, 'foldr1_dropWhile_intersperse_pack, 'foldl1'_pack, 'head_replicate_singleton, 'unpack_toUpper_snoc_singleton, 'null_empty, 'maximum_singleton, 'isPrefixOf_init_singleton, 'minimum_unfoldr, 'foldl_stripStart_snoc_singleton, 'any_toUpper_unfoldr, 'all_unfoldr, 'minimum_toLower_unfoldrN, 'null_pack, 'index_dropWhile_unfoldr, 'minimum_filter_toUpper_singleton, 'head_pack, 'foldl1'_toTitle_singleton, 'find_unfoldr, 'isPrefixOf_unfoldrN, 'unpack_append_pack, 'any_unfoldr, 'length_unfoldrN, 'minimum_singleton, 'head_snoc_toUpper_singleton, 'maximum_unfoldrN, 'all_take_pack, 'isPrefixOf_pack, 'foldr_init_pack, 'foldl1'_filter_pack])] - ---------------------------------------------------------------------------------- --- Definitions below are from inspection-testing package by Joachim Breitner. --- - -i = 42 -{-# NOINLINE i #-} - -empty _ = T.empty -{-# INLINE empty #-} -take x = T.take i x -{-# INLINE take #-} -drop x = T.drop i x -{-# INLINE drop #-} -cons x = 'x' `T.cons` x -{-# INLINE cons #-} -snoc x = x `T.snoc` 'x' -{-# INLINE snoc #-} -map x = T.map succ x -{-# INLINE map #-} -justifyLeft x = T.justifyLeft 42 'x' x -{-# INLINE justifyLeft #-} -intersperse x = T.intersperse 'x' x -{-# INLINE intersperse #-} -append x = unfoldrN 'y' `T.append` x -{-# INLINE append #-} -isPrefixOf x = unfoldrN 'a' `T.isPrefixOf` x -{-# INLINE isPrefixOf #-} -compareLength x = x `T.compareLength` i -{-# INLINE compareLength #-} -foldl x = T.foldl (\x c -> x + fromEnum c) 0 x -{-# INLINE foldl #-} -foldl' x = T.foldl' (\x c -> x + fromEnum c) 0 x -{-# INLINE foldl' #-} -foldl1 x = T.foldl1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldl1 #-} -foldl1' x = T.foldl1' (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldl1' #-} -foldr x = T.foldr (\c x -> x + fromEnum c) 0 x -{-# INLINE foldr #-} -foldr1 x = T.foldr1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldr1 #-} -any x = T.any isAscii x -{-# INLINE any #-} -all x = T.all isAscii x -{-# INLINE all #-} -scanl x = T.scanl (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) 'x' x -{-# INLINE scanl #-} -unfoldr x = T.unfoldr (\c -> if c == 'z' then Nothing else Just (c, succ c)) x -{-# INLINE unfoldr #-} -unfoldrN x = T.unfoldrN i (\c -> if c == 'z' then Nothing else Just (c, succ c)) x -{-# INLINE unfoldrN #-} -takeWhile x = T.takeWhile isAscii x -{-# INLINE takeWhile #-} -dropWhile x = T.dropWhile isAscii x -{-# INLINE dropWhile #-} -filter x = T.filter isAscii x -{-# INLINE filter #-} -find x = T.find isAscii x -{-# INLINE find #-} -replicate x = T.replicate i x -{-# INLINE replicate #-} -index x = x `T.index` i -{-# INLINE index #-} diff --git a/tests/Tests/Inspection/Strict.hs b/tests/Tests/Inspection/Strict.hs deleted file mode 100644 --- a/tests/Tests/Inspection/Strict.hs +++ /dev/null @@ -1,210 +0,0 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE TemplateHaskell #-} -{-# OPTIONS_GHC -O -fno-warn-unused-top-binds -fno-warn-missing-signatures -fno-warn-name-shadowing -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} - -module Tests.Inspection.Strict (tests) where - -import Prelude hiding (all, any, drop, dropWhile, filter, foldl, foldl1, foldr, foldr1, head, init, iterate, last, length, map, maximum, minimum, null, reverse, scanl, scanl1, scanr, scanr1, tail, take, takeWhile) -import Data.Char (isAscii) - -import qualified Data.Text as T -import qualified Data.Text.Encoding as T - -import Test.Tasty -import Test.Tasty.Inspection - -all_cons_pack = all . cons . T.pack -all_drop_pack = all . drop . T.pack -all_dropWhileEnd_singleton = all . dropWhileEnd . T.singleton -all_justifyRight_singleton = all . justifyRight . T.singleton -all_scanl_init_unfoldrN = all . scanl . T.init . unfoldrN -all_stripEnd_stripStart_singleton = all . T.stripEnd . T.stripStart . T.singleton -any_filter_init_pack = any . filter . T.init . T.pack -any_justifyRight_dropEnd_singleton = any . justifyRight . dropEnd . T.singleton -any_singleton = any . T.singleton -any_stripStart_singleton = any . T.stripStart . T.singleton -any_toCaseFold_unfoldr = any . T.toCaseFold . unfoldr -any_unfoldrN = any . unfoldrN -compareLength_cons_snoc_singleton = compareLength . cons . snoc . T.singleton -compareLength_drop_unfoldr = compareLength . drop . unfoldr -compareLength_empty = compareLength . empty -compareLength_pack = compareLength . T.pack -compareLength_singleton = compareLength . T.singleton -compareLength_snoc_unfoldr = compareLength . snoc . unfoldr -compareLength_takeWhile_intersperse_unfoldr = compareLength . takeWhile . intersperse . unfoldr -compareLength_toTitle_singleton = compareLength . T.toTitle . T.singleton -compareLength_toTitle_unfoldrN = compareLength . T.toTitle . unfoldrN -compareLength_unfoldr = compareLength . unfoldr -find_decodeUtf8 = find . T.decodeUtf8 -find_take_unfoldrN = find . take . unfoldrN -foldl'_cons_pack = foldl' . cons . T.pack -foldl'_scanr1_singleton = foldl' . scanr1 . T.singleton -foldl'_toCaseFold_intersperse_unfoldrN = foldl' . T.toCaseFold . intersperse . unfoldrN -foldl_center_empty = foldl . center . empty -foldl_justifyLeft_cons_empty = foldl . justifyLeft . cons . empty -foldl_justifyLeft_decodeUtf8 = foldl . justifyLeft . T.decodeUtf8 -foldl_pack = foldl . T.pack -foldl_scanl_dropWhile_empty = foldl . scanl . dropWhile . empty -foldl1'_append_append_decodeUtf8 = foldl1' . append . append . T.decodeUtf8 -foldl1'_dropWhile_dropWhileEnd_singleton = foldl1' . dropWhile . dropWhileEnd . T.singleton -foldl1'_scanl_decodeUtf8 = foldl1' . scanl . T.decodeUtf8 -foldl1'_scanl_justifyLeft_unfoldr = foldl1' . scanl . justifyLeft . unfoldr -foldl1'_singleton = foldl1' . T.singleton -foldl1'_take_unfoldr = foldl1' . take . unfoldr -foldl1_intersperse_empty = foldl1 . intersperse . empty -foldl1_scanr_singleton = foldl1 . scanr . T.singleton -foldl1_tail_singleton = foldl1 . T.tail . T.singleton -foldr_append_singleton = foldr . append . T.singleton -foldr_empty = foldr . empty -foldr_intersperse_dropWhile_unfoldr = foldr . intersperse . dropWhile . unfoldr -foldr_intersperse_singleton = foldr . intersperse . T.singleton -foldr_scanl_unfoldr = foldr . scanl . unfoldr -foldr1_dropWhile_intersperse_pack = foldr1 . dropWhile . intersperse . T.pack -foldr1_justifyLeft_scanl1_empty = foldr1 . justifyLeft . scanl1 . empty -foldr1_reverse_unfoldrN = foldr1 . T.reverse . unfoldrN -foldr1_singleton = foldr1 . T.singleton -foldr1_take_drop_pack = foldr1 . take . drop . T.pack -foldr1_unfoldrN = foldr1 . unfoldrN -head_append_toTitle_singleton = T.head . append . T.toTitle . T.singleton -head_cons_unfoldr = T.head . cons . unfoldr -head_drop_decodeUtf8 = T.head . drop . T.decodeUtf8 -head_singleton = T.head . T.singleton -head_strip_take_empty = T.head . T.strip . take . empty -head_takeEnd_take_singleton = T.head . takeEnd . take . T.singleton -index_dropWhile_unfoldr = index . dropWhile . unfoldr -index_dropWhileEnd_empty = index . dropWhileEnd . empty -index_justifyLeft_stripEnd_singleton = index . justifyLeft . T.stripEnd . T.singleton -isPrefixOf_dropWhile_dropWhile_pack = isPrefixOf . dropWhile . dropWhile . T.pack -isPrefixOf_init_take_unfoldrN = isPrefixOf . T.init . take . unfoldrN -isPrefixOf_snoc_stripStart_pack = isPrefixOf . snoc . T.stripStart . T.pack -isPrefixOf_take_empty = isPrefixOf . take . empty -isPrefixOf_take_singleton = isPrefixOf . take . T.singleton -last_dropWhile_unfoldrN = T.last . dropWhile . unfoldrN -last_map_take_pack = T.last . map . take . T.pack -last_tail_unfoldrN = T.last . T.tail . unfoldrN -last_toUpper_stripStart_singleton = T.last . T.toUpper . T.stripStart . T.singleton -last_toUpper_toLower_unfoldrN = T.last . T.toUpper . T.toLower . unfoldrN -length_empty = T.length . empty -length_intersperse_center_singleton = T.length . intersperse . center . T.singleton -length_justifyLeft_decodeUtf8 = T.length . justifyLeft . T.decodeUtf8 -length_pack = T.length . T.pack -length_reverse_singleton = T.length . T.reverse . T.singleton -length_takeWhile_intersperse_singleton = T.length . takeWhile . intersperse . T.singleton -length_takeWhile_singleton = T.length . takeWhile . T.singleton -length_toTitle_empty = T.length . T.toTitle . empty -maximum_justifyLeft_filter_singleton = T.maximum . justifyLeft . filter . T.singleton -maximum_justifyRight_singleton = T.maximum . justifyRight . T.singleton -maximum_take_unfoldrN = T.maximum . take . unfoldrN -maximum_toLower_empty = T.maximum . T.toLower . empty -minimum_init_singleton = T.minimum . T.init . T.singleton -minimum_intersperse_toTitle_singleton = T.minimum . intersperse . T.toTitle . T.singleton -minimum_map_singleton = T.minimum . map . T.singleton -minimum_scanl1_takeWhile_singleton = T.minimum . scanl1 . takeWhile . T.singleton -minimum_tail_map_singleton = T.minimum . T.tail . map . T.singleton -minimum_unfoldrN = T.minimum . unfoldrN -null_cons_singleton = T.null . cons . T.singleton -null_init_drop_decodeUtf8 = T.null . T.init . drop . T.decodeUtf8 -null_map_empty = T.null . map . empty -null_toCaseFold_dropAround_singleton = T.null . T.toCaseFold . dropAround . T.singleton -unpack_empty = T.unpack . empty -unpack_justifyLeft_take_empty = T.unpack . justifyLeft . take . empty -unpack_map_pack = T.unpack . map . T.pack -unpack_stripEnd_takeWhileEnd_singleton = T.unpack . T.stripEnd . takeWhileEnd . T.singleton -unpack_toCaseFold_scanr_singleton = T.unpack . T.toCaseFold . scanr . T.singleton -unpack_toUpper_snoc_singleton = T.unpack . T.toUpper . snoc . T.singleton -unpack_unfoldr = T.unpack . unfoldr - -tests :: TestTree -tests = testGroup "Strict fusion" [$(inspectNames (`hasNoType` ''T.Text) - [ -#if __GLASGOW_HASKELL__ >= 806 - 'all_cons_pack, 'all_drop_pack, 'all_scanl_init_unfoldrN, 'any_filter_init_pack, 'any_toCaseFold_unfoldr, 'any_unfoldrN, 'compareLength_drop_unfoldr, 'compareLength_empty, 'compareLength_pack, 'compareLength_snoc_unfoldr, 'compareLength_takeWhile_intersperse_unfoldr, 'compareLength_toTitle_unfoldrN, 'compareLength_unfoldr, 'find_decodeUtf8, 'find_take_unfoldrN, 'foldl'_cons_pack, 'foldl'_toCaseFold_intersperse_unfoldrN, 'foldl_justifyLeft_cons_empty, 'foldl_justifyLeft_decodeUtf8, 'foldl_pack, 'foldl_scanl_dropWhile_empty, 'foldl1'_append_append_decodeUtf8, 'foldl1'_scanl_decodeUtf8, 'foldl1'_scanl_justifyLeft_unfoldr, 'foldl1'_take_unfoldr, 'foldl1_intersperse_empty, 'foldr_empty, 'foldr_intersperse_dropWhile_unfoldr, 'foldr_scanl_unfoldr, 'foldr1_dropWhile_intersperse_pack, 'foldr1_justifyLeft_scanl1_empty, 'foldr1_reverse_unfoldrN, 'foldr1_take_drop_pack, 'foldr1_unfoldrN, 'head_cons_unfoldr, 'head_drop_decodeUtf8, 'head_strip_take_empty, 'index_dropWhile_unfoldr, 'index_dropWhileEnd_empty, 'isPrefixOf_dropWhile_dropWhile_pack, 'isPrefixOf_init_take_unfoldrN, 'isPrefixOf_snoc_stripStart_pack, 'isPrefixOf_take_empty, 'last_dropWhile_unfoldrN, 'last_map_take_pack, 'last_tail_unfoldrN, 'last_toUpper_toLower_unfoldrN, 'length_empty, 'length_justifyLeft_decodeUtf8, 'length_pack, 'length_toTitle_empty, 'maximum_take_unfoldrN, 'maximum_toLower_empty, 'minimum_unfoldrN, 'null_init_drop_decodeUtf8, 'null_map_empty, 'unpack_empty, 'unpack_justifyLeft_take_empty, 'unpack_map_pack, 'unpack_unfoldr -#endif --- https://gitlab.haskell.org/ghc/ghc/-/issues/19822 -#if __GLASGOW_HASKELL__ >= 806 && __GLASGOW_HASKELL__ < 900 - , 'foldl_center_empty -- the only non-singleton-related regression - , 'all_dropWhileEnd_singleton, 'all_justifyRight_singleton, 'all_stripEnd_stripStart_singleton, 'any_justifyRight_dropEnd_singleton, 'any_singleton, 'any_stripStart_singleton, 'compareLength_cons_snoc_singleton, 'compareLength_singleton, 'compareLength_toTitle_singleton, 'foldl'_scanr1_singleton, 'foldl1'_dropWhile_dropWhileEnd_singleton, 'foldl1_scanr_singleton, 'foldl1_tail_singleton, 'foldr_append_singleton, 'foldl1'_singleton, 'foldr_intersperse_singleton, 'foldr1_singleton, 'head_append_toTitle_singleton, 'head_singleton, 'head_takeEnd_take_singleton, 'isPrefixOf_take_singleton, 'index_justifyLeft_stripEnd_singleton, 'last_toUpper_stripStart_singleton, 'length_intersperse_center_singleton, 'length_reverse_singleton, 'length_takeWhile_intersperse_singleton, 'length_takeWhile_singleton, 'maximum_justifyLeft_filter_singleton, 'maximum_justifyRight_singleton, 'minimum_init_singleton, 'minimum_intersperse_toTitle_singleton, 'minimum_map_singleton, 'minimum_scanl1_takeWhile_singleton, 'minimum_tail_map_singleton, 'null_cons_singleton, 'null_toCaseFold_dropAround_singleton, 'unpack_stripEnd_takeWhileEnd_singleton, 'unpack_toCaseFold_scanr_singleton, 'unpack_toUpper_snoc_singleton -#endif - ])] - ---------------------------------------------------------------------------------- --- Definitions below are from inspection-testing package by Joachim Breitner. --- - -i = 42 -{-# NOINLINE i #-} - -empty _ = T.empty -{-# INLINE empty #-} -take x = T.take i x -{-# INLINE take #-} -takeEnd x = T.takeEnd i x -{-# INLINE takeEnd #-} -drop x = T.drop i x -{-# INLINE drop #-} -dropEnd x = T.dropEnd i x -{-# INLINE dropEnd #-} -cons x = 'x' `T.cons` x -{-# INLINE cons #-} -snoc x = x `T.snoc` 'x' -{-# INLINE snoc #-} -map x = T.map succ x -{-# INLINE map #-} -justifyLeft x = T.justifyLeft 42 'x' x -{-# INLINE justifyLeft #-} -justifyRight x = T.justifyRight 42 'x' x -{-# INLINE justifyRight #-} -center x = T.center i 'x' x -{-# INLINE center #-} -intersperse x = T.intersperse 'x' x -{-# INLINE intersperse #-} -append x = unfoldrN 'y' `T.append` x -{-# INLINE append #-} -isPrefixOf x = unfoldrN 'a' `T.isPrefixOf` x -{-# INLINE isPrefixOf #-} -compareLength x = x `T.compareLength` i -{-# INLINE compareLength #-} -foldl x = T.foldl (\x c -> x + fromEnum c) 0 x -{-# INLINE foldl #-} -foldl' x = T.foldl' (\x c -> x + fromEnum c) 0 x -{-# INLINE foldl' #-} -foldl1 x = T.foldl1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldl1 #-} -foldl1' x = T.foldl1' (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldl1' #-} -foldr x = T.foldr (\c x -> x + fromEnum c) 0 x -{-# INLINE foldr #-} -foldr1 x = T.foldr1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE foldr1 #-} -any x = T.any isAscii x -{-# INLINE any #-} -all x = T.all isAscii x -{-# INLINE all #-} -scanl x = T.scanl (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) 'x' x -{-# INLINE scanl #-} -scanl1 x = T.scanl1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE scanl1 #-} -scanr x = T.scanr (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) 'x' x -{-# INLINE scanr #-} -scanr1 x = T.scanr1 (\c1 c2 -> toEnum $ fromEnum c1 + fromEnum c2) x -{-# INLINE scanr1 #-} -unfoldr x = T.unfoldr (\c -> if c == 'z' then Nothing else Just (c, succ c)) x -{-# INLINE unfoldr #-} -unfoldrN x = T.unfoldrN i (\c -> if c == 'z' then Nothing else Just (c, succ c)) x -{-# INLINE unfoldrN #-} -takeWhile x = T.takeWhile isAscii x -{-# INLINE takeWhile #-} -dropWhile x = T.dropWhile isAscii x -{-# INLINE dropWhile #-} -takeWhileEnd x = T.takeWhileEnd isAscii x -{-# INLINE takeWhileEnd #-} -dropWhileEnd x = T.dropWhileEnd isAscii x -{-# INLINE dropWhileEnd #-} -dropAround x = T.dropAround isAscii x -{-# INLINE dropAround #-} -filter x = T.filter isAscii x -{-# INLINE filter #-} -find x = T.find isAscii x -{-# INLINE find #-} -index x = x `T.index` i -{-# INLINE index #-} diff --git a/tests/Tests/Lift.hs b/tests/Tests/Lift.hs --- a/tests/Tests/Lift.hs +++ b/tests/Tests/Lift.hs @@ -16,7 +16,13 @@ [ testCase "strict" $ assertEqual "strict" $(lift ("foo" :: S.Text)) ("foo" :: S.Text) + , testCase "strict0" $ assertEqual "strict0" + $(lift ("f\0o\1o\2" :: S.Text)) + ("f\0o\1o\2" :: S.Text) , testCase "lazy" $ assertEqual "lazy" $(lift ("foo" :: L.Text)) ("foo" :: L.Text) + , testCase "lazy0" $ assertEqual "lazy0" + $(lift ("f\0o\1o\2" :: L.Text)) + ("f\0o\1o\2" :: L.Text) ] diff --git a/tests/Tests/Properties.hs b/tests/Tests/Properties.hs --- a/tests/Tests/Properties.hs +++ b/tests/Tests/Properties.hs @@ -1,6 +1,6 @@ -- | QuickCheck properties for the text library. -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties ( tests diff --git a/tests/Tests/Properties/Basics.hs b/tests/Tests/Properties/Basics.hs --- a/tests/Tests/Properties/Basics.hs +++ b/tests/Tests/Properties/Basics.hs @@ -1,15 +1,16 @@ -- | Test basic text functions -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Basics ( testBasics ) where import Control.Arrow (first, second) import Test.Tasty (TestTree, testGroup) -import Test.Tasty.QuickCheck (testProperty) +import Test.Tasty.QuickCheck (testProperty, applyFun) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Internal.Fusion as S @@ -19,16 +20,24 @@ s_cons x = (x:) `eqP` (unpackS . S.cons x) s_cons_s x = (x:) `eqP` (unpackS . S.unstream . S.cons x) -sf_cons p x = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p) +sf_cons (applyFun -> p) x + = ((x:) . L.filter p) `eqP` (unpackS . S.cons x . S.filter p) t_cons x = (x:) `eqP` (unpackS . T.cons x) tl_cons x = (x:) `eqP` (unpackS . TL.cons x) -s_snoc x = (++ [x]) `eqP` (unpackS . (flip S.snoc) x) -t_snoc x = (++ [x]) `eqP` (unpackS . (flip T.snoc) x) -tl_snoc x = (++ [x]) `eqP` (unpackS . (flip TL.snoc) x) +t_length_cons x = (L.length . (x:)) `eqP` (T.length . T.cons x) +tl_length_cons x = (L.genericLength . (x:)) `eqP` (TL.length . TL.cons x) + +s_snoc x = (++ [x]) `eqP` (unpackS . flip S.snoc x) +t_snoc x = (++ [x]) `eqP` (unpackS . flip T.snoc x) +tl_snoc x = (++ [x]) `eqP` (unpackS . flip TL.snoc x) +t_length_snoc x = (L.length . (++ [x])) `eqP` (T.length . flip T.snoc x) +tl_length_snoc x = (L.genericLength . (++ [x])) `eqP` (TL.length . flip TL.snoc x) + s_append s = (s++) `eqP` (unpackS . S.append (S.streamList s)) s_append_s s = (s++) `eqP` (unpackS . S.unstream . S.append (S.streamList s)) -sf_append p s = (L.filter p s++) `eqP` +sf_append (applyFun -> p) s + = (L.filter p s++) `eqP` (unpackS . S.append (S.filter p $ S.streamList s)) t_append s = (s++) `eqP` (unpackS . T.append (packS s)) @@ -36,7 +45,8 @@ uncons _ = Nothing s_uncons = uncons `eqP` (fmap (second unpackS) . S.uncons) -sf_uncons p = (uncons . L.filter p) `eqP` +sf_uncons (applyFun -> p) + = (uncons . L.filter p) `eqP` (fmap (second unpackS) . S.uncons . S.filter p) t_uncons = uncons `eqP` (fmap (second unpackS) . T.uncons) tl_uncons = uncons `eqP` (fmap (second unpackS) . TL.uncons) @@ -48,29 +58,29 @@ tl_unsnoc = unsnoc `eqP` (fmap (first unpackS) . TL.unsnoc) s_head = head `eqP` S.head -sf_head p = (head . L.filter p) `eqP` (S.head . S.filter p) +sf_head (applyFun -> p) = (head . L.filter p) `eqP` (S.head . S.filter p) t_head = head `eqP` T.head tl_head = head `eqP` TL.head s_last = last `eqP` S.last -sf_last p = (last . L.filter p) `eqP` (S.last . S.filter p) +sf_last (applyFun -> p) = (last . L.filter p) `eqP` (S.last . S.filter p) t_last = last `eqP` T.last tl_last = last `eqP` TL.last s_tail = tail `eqP` (unpackS . S.tail) s_tail_s = tail `eqP` (unpackS . S.unstream . S.tail) -sf_tail p = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p) +sf_tail (applyFun -> p) = (tail . L.filter p) `eqP` (unpackS . S.tail . S.filter p) t_tail = tail `eqP` (unpackS . T.tail) tl_tail = tail `eqP` (unpackS . TL.tail) s_init = init `eqP` (unpackS . S.init) s_init_s = init `eqP` (unpackS . S.unstream . S.init) -sf_init p = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p) +sf_init (applyFun -> p) = (init . L.filter p) `eqP` (unpackS . S.init . S.filter p) t_init = init `eqP` (unpackS . T.init) tl_init = init `eqP` (unpackS . TL.init) s_null = null `eqP` S.null -sf_null p = (null . L.filter p) `eqP` (S.null . S.filter p) +sf_null (applyFun -> p) = (null . L.filter p) `eqP` (S.null . S.filter p) t_null = null `eqP` T.null tl_null = null `eqP` TL.null s_length = length `eqP` S.length -sf_length p = (length . L.filter p) `eqP` (S.length . S.filter p) +sf_length (applyFun -> p) = (length . L.filter p) `eqP` (S.length . S.filter p) sl_length = (fromIntegral . length) `eqP` SL.length t_length = length `eqP` T.length tl_length = L.genericLength `eqP` TL.length @@ -90,9 +100,13 @@ testProperty "sf_cons" sf_cons, testProperty "t_cons" t_cons, testProperty "tl_cons" tl_cons, + testProperty "t_length_cons" t_length_cons, + testProperty "tl_length_cons" tl_length_cons, testProperty "s_snoc" s_snoc, testProperty "t_snoc" t_snoc, testProperty "tl_snoc" tl_snoc, + testProperty "t_length_snoc" t_length_snoc, + testProperty "tl_length_snoc" tl_length_snoc, testProperty "s_append" s_append, testProperty "s_append_s" s_append_s, testProperty "sf_append" sf_append, diff --git a/tests/Tests/Properties/Builder.hs b/tests/Tests/Properties/Builder.hs --- a/tests/Tests/Properties/Builder.hs +++ b/tests/Tests/Properties/Builder.hs @@ -2,7 +2,7 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Builder ( testBuilder ) where @@ -14,7 +14,6 @@ import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.List as L import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB @@ -23,13 +22,14 @@ -- Builder. -tb_singleton = id `eqP` - (unpackS . TB.toLazyText . mconcat . map TB.singleton) -tb_fromText = L.concat `eq` (unpackS . TB.toLazyText . mconcat . - map (TB.fromText . packS)) +tb_singleton = id `eqP` (unpackS . TB.toLazyText . mconcat . map TB.singleton) +tb_fromString = id `eq` (TL.unpack . TB.toLazyText . TB.fromString) +tb_fromText = id `eqP` (unpackS . TB.toLazyText . TB.fromText) +tb_fromStrings = L.concat `eq` (TL.unpack . TB.toLazyText . mconcat . map TB.fromString) +tb_fromTexts = L.concat `eq` (unpackS . TB.toLazyText . mconcat . map (TB.fromText . packS)) tb_associative s1 s2 s3 = - TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) == + TB.toLazyText (b1 `mappend` (b2 `mappend` b3)) === TB.toLazyText ((b1 `mappend` b2) `mappend` b3) where b1 = TB.fromText (packS s1) b2 = TB.fromText (packS s2) @@ -37,7 +37,7 @@ -- Numeric builder stuff. -tb_decimal :: (Integral a, Show a) => a -> Bool +tb_decimal :: (Integral a, Show a) => a -> Property tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show) tb_decimal_integer (a::Integer) = tb_decimal a @@ -53,12 +53,12 @@ tb_decimal_word32 (a::Word32) = tb_decimal a tb_decimal_word64 (a::Word64) = tb_decimal a -tb_decimal_big_int (BigBounded (a::Int)) = tb_decimal a -tb_decimal_big_int64 (BigBounded (a::Int64)) = tb_decimal a -tb_decimal_big_word (BigBounded (a::Word)) = tb_decimal a -tb_decimal_big_word64 (BigBounded (a::Word64)) = tb_decimal a +tb_decimal_big_int (Large (a::Int)) = tb_decimal a +tb_decimal_big_int64 (Large (a::Int64)) = tb_decimal a +tb_decimal_big_word (Large (a::Word)) = tb_decimal a +tb_decimal_big_word64 (Large (a::Word64)) = tb_decimal a -tb_hex :: (Integral a, Show a) => a -> Bool +tb_hex :: (Integral a, Show a) => a -> Property tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "") tb_hexadecimal_integer (a::Integer) = tb_hex a @@ -73,7 +73,7 @@ tb_hexadecimal_word32 (a::Word32) = tb_hex a tb_hexadecimal_word64 (a::Word64) = tb_hex a -tb_realfloat :: (RealFloat a, Show a) => a -> Bool +tb_realfloat :: (RealFloat a, Show a) => a -> Property tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show) tb_realfloat_float (a::Float) = tb_realfloat a @@ -103,8 +103,11 @@ testBuilder :: TestTree testBuilder = testGroup "builder" [ - testProperty "tb_fromText" tb_fromText, testProperty "tb_singleton" tb_singleton, + testProperty "tb_fromString" tb_fromString, + testProperty "tb_fromText" tb_fromText, + testProperty "tb_fromStrings" tb_fromStrings, + testProperty "tb_fromTexts" tb_fromTexts, testProperty "tb_associative" tb_associative, testGroup "decimal" [ testProperty "tb_decimal_int" tb_decimal_int, diff --git a/tests/Tests/Properties/Folds.hs b/tests/Tests/Properties/Folds.hs --- a/tests/Tests/Properties/Folds.hs +++ b/tests/Tests/Properties/Folds.hs @@ -1,6 +1,8 @@ -- | Test folds, scans, and unfolds -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Folds ( testFolds ) where @@ -8,9 +10,8 @@ import Control.Arrow (second) import Data.Word (Word8, Word16) import Test.Tasty (TestTree, testGroup) -import Test.Tasty.QuickCheck (testProperty) +import Test.Tasty.QuickCheck (testProperty, Small(..), (===), applyFun, applyFun2) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Internal.Fusion as S @@ -19,112 +20,135 @@ -- Folds -sf_foldl p f z = (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) +sf_foldl (applyFun -> p) (applyFun2 -> f) z = + (L.foldl f z . L.filter p) `eqP` (S.foldl f z . S.filter p) where _types = f :: Char -> Char -> Char -t_foldl f z = L.foldl f z `eqP` (T.foldl f z) +t_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (T.foldl f z) where _types = f :: Char -> Char -> Char -tl_foldl f z = L.foldl f z `eqP` (TL.foldl f z) +tl_foldl (applyFun2 -> f) z = L.foldl f z `eqP` (TL.foldl f z) where _types = f :: Char -> Char -> Char -sf_foldl' p f z = (L.foldl' f z . L.filter p) `eqP` - (S.foldl' f z . S.filter p) +sf_foldl' (applyFun -> p) (applyFun2 -> f) z = + (L.foldl' f z . L.filter p) `eqP` (S.foldl' f z . S.filter p) where _types = f :: Char -> Char -> Char -t_foldl' f z = L.foldl' f z `eqP` T.foldl' f z +t_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` T.foldl' f z where _types = f :: Char -> Char -> Char -tl_foldl' f z = L.foldl' f z `eqP` TL.foldl' f z +tl_foldl' (applyFun2 -> f) z = L.foldl' f z `eqP` TL.foldl' f z where _types = f :: Char -> Char -> Char -sf_foldl1 p f = (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) -t_foldl1 f = L.foldl1 f `eqP` T.foldl1 f -tl_foldl1 f = L.foldl1 f `eqP` TL.foldl1 f -sf_foldl1' p f = (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) -t_foldl1' f = L.foldl1' f `eqP` T.foldl1' f -tl_foldl1' f = L.foldl1' f `eqP` TL.foldl1' f -sf_foldr p f z = (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) +sf_foldl1 (applyFun -> p) (applyFun2 -> f) = + (L.foldl1 f . L.filter p) `eqP` (S.foldl1 f . S.filter p) +t_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` T.foldl1 f +tl_foldl1 (applyFun2 -> f) = L.foldl1 f `eqP` TL.foldl1 f +sf_foldl1' (applyFun -> p) (applyFun2 -> f) = + (L.foldl1' f . L.filter p) `eqP` (S.foldl1' f . S.filter p) +t_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` T.foldl1' f +tl_foldl1' (applyFun2 -> f) = L.foldl1' f `eqP` TL.foldl1' f +sf_foldr (applyFun -> p) (applyFun2 -> f) z = + (L.foldr f z . L.filter p) `eqP` (S.foldr f z . S.filter p) where _types = f :: Char -> Char -> Char -t_foldr f z = L.foldr f z `eqP` T.foldr f z +t_foldr (applyFun2 -> f) z = L.foldr f z `eqP` T.foldr f z where _types = f :: Char -> Char -> Char -tl_foldr f z = unsquare $ - L.foldr f z `eqP` TL.foldr f z +tl_foldr (applyFun2 -> f) z = L.foldr f z `eqPSqrt` TL.foldr f z where _types = f :: Char -> Char -> Char -sf_foldr1 p f = unsquare $ - (L.foldr1 f . L.filter p) `eqP` (S.foldr1 f . S.filter p) -t_foldr1 f = L.foldr1 f `eqP` T.foldr1 f -tl_foldr1 f = unsquare $ - L.foldr1 f `eqP` TL.foldr1 f +sf_foldr1 (applyFun -> p) (applyFun2 -> f) = + (L.foldr1 f . L.filter p) `eqPSqrt` (S.foldr1 f . S.filter p) +t_foldr1 (applyFun2 -> f) = L.foldr1 f `eqP` T.foldr1 f +tl_foldr1 (applyFun2 -> f) = L.foldr1 f `eqPSqrt` TL.foldr1 f -- Special folds -s_concat_s = unsquare $ - L.concat `eq` (unpackS . S.unstream . S.concat . map packS) -sf_concat p = unsquare $ - (L.concat . map (L.filter p)) `eq` - (unpackS . S.concat . map (S.filter p . packS)) -t_concat = unsquare $ - L.concat `eq` (unpackS . T.concat . map packS) -tl_concat = unsquare $ - L.concat `eq` (unpackS . TL.concat . map TL.pack) -sf_concatMap p f = unsquare $ (L.concatMap f . L.filter p) `eqP` - (unpackS . S.concatMap (packS . f) . S.filter p) -t_concatMap f = unsquare $ - L.concatMap f `eqP` (unpackS . T.concatMap (packS . f)) -tl_concatMap f = unsquare $ - L.concatMap f `eqP` (unpackS . TL.concatMap (TL.pack . f)) -sf_any q p = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) -t_any p = L.any p `eqP` T.any p -tl_any p = L.any p `eqP` TL.any p -sf_all q p = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) -t_all p = L.all p `eqP` T.all p -tl_all p = L.all p `eqP` TL.all p -sf_maximum p = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) +s_concat_s = (L.concat . unSqrt) `eq` (unpackS . S.unstream . S.concat . map packS . unSqrt) +sf_concat (applyFun -> p) + = (L.concat . map (L.filter p) . unSqrt) `eq` + (unpackS . S.concat . map (S.filter p . packS) . unSqrt) +t_concat = (L.concat . unSqrt) `eq` (unpackS . T.concat . map packS . unSqrt) +tl_concat = (L.concat . unSqrt) `eq` (unpackS . TL.concat . map TL.pack . unSqrt) +sf_concatMap (applyFun -> p) (applyFun -> f) = + (L.concatMap f . L.filter p) `eqPSqrt` (unpackS . S.concatMap (packS . f) . S.filter p) +t_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . T.concatMap (packS . f)) +tl_concatMap (applyFun -> f) + = L.concatMap f `eqPSqrt` (unpackS . TL.concatMap (TL.pack . f)) +sf_any (applyFun -> q) (applyFun -> p) + = (L.any p . L.filter q) `eqP` (S.any p . S.filter q) +t_any (applyFun -> p) + = L.any p `eqP` T.any p +tl_any (applyFun -> p) + = L.any p `eqP` TL.any p +sf_all (applyFun -> q) (applyFun -> p) + = (L.all p . L.filter q) `eqP` (S.all p . S.filter q) +t_all (applyFun -> p) + = L.all p `eqP` T.all p +tl_all (applyFun -> p) + = L.all p `eqP` TL.all p +sf_maximum (applyFun -> p) + = (L.maximum . L.filter p) `eqP` (S.maximum . S.filter p) t_maximum = L.maximum `eqP` T.maximum tl_maximum = L.maximum `eqP` TL.maximum -sf_minimum p = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) +sf_minimum (applyFun -> p) + = (L.minimum . L.filter p) `eqP` (S.minimum . S.filter p) t_minimum = L.minimum `eqP` T.minimum tl_minimum = L.minimum `eqP` TL.minimum -- Scans -sf_scanl p f z = (L.scanl f z . L.filter p) `eqP` - (unpackS . S.scanl f z . S.filter p) -t_scanl f z = L.scanl f z `eqP` (unpackS . T.scanl f z) -tl_scanl f z = L.scanl f z `eqP` (unpackS . TL.scanl f z) -t_scanl1 f = L.scanl1 f `eqP` (unpackS . T.scanl1 f) -tl_scanl1 f = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) -t_scanr f z = L.scanr f z `eqP` (unpackS . T.scanr f z) -tl_scanr f z = L.scanr f z `eqP` (unpackS . TL.scanr f z) -t_scanr1 f = L.scanr1 f `eqP` (unpackS . T.scanr1 f) -tl_scanr1 f = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) +sf_scanl (applyFun -> p) (applyFun2 -> f) z = + (L.scanl f z . L.filter p) `eqP` (unpackS . S.scanl f z . S.filter p) +t_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . T.scanl f z) +tl_scanl (applyFun2 -> f) z = L.scanl f z `eqP` (unpackS . TL.scanl f z) +t_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . T.scanl1 f) +tl_scanl1 (applyFun2 -> f) = L.scanl1 f `eqP` (unpackS . TL.scanl1 f) +t_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . T.scanr f z) +tl_scanr (applyFun2 -> f) z = L.scanr f z `eqP` (unpackS . TL.scanr f z) +t_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . T.scanr1 f) +tl_scanr1 (applyFun2 -> f) = L.scanr1 f `eqP` (unpackS . TL.scanr1 f) -t_mapAccumL f z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) +t_mapAccumL_char c t = + snd (T.mapAccumL (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . T.mapAccumL f z) where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumL f z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) +tl_mapAccumL_char c t = + snd (TL.mapAccumL (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumL (applyFun2 -> f) z = L.mapAccumL f z `eqP` (second unpackS . TL.mapAccumL f z) where _types = f :: Int -> Char -> (Int,Char) -t_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) +t_mapAccumR_char c t = + snd (T.mapAccumR (const (const (0 :: Int, c))) 0 t) === T.replicate (T.length t) (T.singleton c) +t_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z) where _types = f :: Int -> Char -> (Int,Char) -tl_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) +tl_mapAccumR_char c t = + snd (TL.mapAccumR (const (const (0 :: Int, c))) 0 t) === TL.replicate (TL.length t) (TL.singleton c) +tl_mapAccumR (applyFun2 -> f) z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z) where _types = f :: Int -> Char -> (Int,Char) -- Unfolds -tl_repeat n = (L.take m . L.repeat) `eq` - (unpackS . TL.take (fromIntegral m) . TL.repeat) - where m = fromIntegral (n :: Word8) +tl_repeat (Small n) = L.replicate n `eq` (unpackS . TL.take (fromIntegral n) . TL.repeat) -any_replicate n l = concat (L.replicate n l) +s_replicate (Small n) = (L.concat . L.replicate n) `eq` (unpackS . S.replicateI (fromIntegral n) . packS) -s_replicate n = any_replicate m `eq` - (unpackS . S.replicateI (fromIntegral m) . packS) - where m = fromIntegral (n :: Word8) -t_replicate n = any_replicate m `eq` (unpackS . T.replicate m . packS) - where m = fromIntegral (n :: Word8) -tl_replicate n = any_replicate m `eq` - (unpackS . TL.replicate (fromIntegral m) . packS) - where m = fromIntegral (n :: Word8) +t_replicate_char (Small n) c = + L.replicate n c === T.unpack (T.replicate n (T.singleton c)) +tl_replicate_char (Small n) c = + L.replicate n c === TL.unpack (TL.replicate (fromIntegral n) (TL.singleton c)) +t_length_replicate_char (Small n) c = + L.length (L.replicate n c) === T.length (T.replicate n (T.singleton c)) +tl_length_replicate_char (Small n) c = + L.genericLength (L.replicate n c) === TL.length (TL.replicate (fromIntegral n) (TL.singleton c)) +t_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . T.replicate n) +tl_replicate (Small n) = + (L.concat . L.replicate n) `eqPSqrt` (unpackS . TL.replicate (fromIntegral n)) +t_length_replicate (Small n) = + (L.length . L.concat . L.replicate n) `eqPSqrt` (T.length . T.replicate n) +tl_length_replicate (Small n) = + (L.genericLength . L.concat . L.replicate n) `eqPSqrt` (TL.length . TL.replicate (fromIntegral n)) + tl_cycle n = (L.take m . L.cycle) `eq` (unpackS . TL.take (fromIntegral m) . TL.cycle . packS) where m = fromIntegral (n :: Word8) -tl_iterate f n = (L.take m . L.iterate f) `eq` +tl_iterate (applyFun -> f) n + = (L.take m . L.iterate f) `eq` (unpackS . TL.take (fromIntegral m) . TL.iterate f) where m = fromIntegral (n :: Word8) @@ -204,22 +228,36 @@ ], testGroup "mapAccum" [ + testProperty "t_mapAccumL_char" t_mapAccumL_char, testProperty "t_mapAccumL" t_mapAccumL, + testProperty "tl_mapAccumL_char" tl_mapAccumL_char, testProperty "tl_mapAccumL" tl_mapAccumL, + testProperty "t_mapAccumR_char" t_mapAccumR_char, testProperty "t_mapAccumR" t_mapAccumR, + testProperty "tl_mapAccumR_char" tl_mapAccumR_char, testProperty "tl_mapAccumR" tl_mapAccumR ], testGroup "unfolds" [ - testProperty "tl_repeat" tl_repeat, - testProperty "s_replicate" s_replicate, - testProperty "t_replicate" t_replicate, - testProperty "tl_replicate" tl_replicate, testProperty "tl_cycle" tl_cycle, testProperty "tl_iterate" tl_iterate, testProperty "t_unfoldr" t_unfoldr, testProperty "tl_unfoldr" tl_unfoldr, testProperty "t_unfoldrN" t_unfoldrN, testProperty "tl_unfoldrN" tl_unfoldrN + ], + + testGroup "replicate" [ + testProperty "tl_repeat" tl_repeat, + testProperty "s_replicate" s_replicate, + testProperty "t_replicate_char" t_replicate_char, + testProperty "tl_replicate_char" tl_replicate_char, + testProperty "t_length_replicate_char" t_length_replicate_char, + testProperty "tl_length_replicate_char" tl_length_replicate_char, + testProperty "t_replicate" t_replicate, + testProperty "tl_replicate" tl_replicate, + testProperty "t_length_replicate" t_length_replicate, + testProperty "tl_length_replicate" tl_length_replicate ] + ] diff --git a/tests/Tests/Properties/Instances.hs b/tests/Tests/Properties/Instances.hs --- a/tests/Tests/Properties/Instances.hs +++ b/tests/Tests/Properties/Instances.hs @@ -1,7 +1,7 @@ -- | Test instances -{-# LANGUAGE CPP #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# LANGUAGE ViewPatterns #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Instances ( testInstances ) where @@ -11,7 +11,6 @@ import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Internal.Fusion.Common as S @@ -19,14 +18,14 @@ s_Eq s = (s==) `eq` ((S.streamList s==) . S.streamList) where _types = s :: String -sf_Eq p s = +sf_Eq (applyFun -> p) s = ((L.filter p s==) . L.filter p) `eq` (((S.filter p $ S.streamList s)==) . S.filter p . S.streamList) t_Eq s = (s==) `eq` ((T.pack s==) . T.pack) tl_Eq s = (s==) `eq` ((TL.pack s==) . TL.pack) s_Ord s = (compare s) `eq` (compare (S.streamList s) . S.streamList) where _types = s :: String -sf_Ord p s = +sf_Ord (applyFun -> p) s = ((compare $ L.filter p s) . L.filter p) `eq` (compare (S.filter p $ S.streamList s) . S.filter p . S.streamList) t_Ord s = (compare s) `eq` (compare (T.pack s) . T.pack) @@ -37,10 +36,8 @@ tl_Show = show `eq` (show . TL.pack) t_mappend s = mappend s`eqP` (unpackS . mappend (T.pack s)) tl_mappend s = mappend s`eqP` (unpackS . mappend (TL.pack s)) -t_mconcat = unsquare $ - mconcat `eq` (unpackS . mconcat . L.map T.pack) -tl_mconcat = unsquare $ - mconcat `eq` (unpackS . mconcat . L.map TL.pack) +t_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map T.pack . unSqrt) +tl_mconcat = (mconcat . unSqrt) `eq` (unpackS . mconcat . L.map TL.pack . unSqrt) t_mempty = mempty === (unpackS (mempty :: T.Text)) tl_mempty = mempty === (unpackS (mempty :: TL.Text)) t_IsString = fromString `eqP` (T.unpack . fromString) diff --git a/tests/Tests/Properties/LowLevel.hs b/tests/Tests/Properties/LowLevel.hs --- a/tests/Tests/Properties/LowLevel.hs +++ b/tests/Tests/Properties/LowLevel.hs @@ -1,31 +1,44 @@ -- | Test low-level operations +{-# LANGUAGE CPP #-} +{-# LANGUAGE MagicHash #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures -fno-warn-unused-imports #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures -fno-warn-unused-imports #-} + +#ifdef MIN_VERSION_tasty_inspection_testing +{-# LANGUAGE TemplateHaskell #-} +{-# OPTIONS_GHC -O -dsuppress-all -dno-suppress-type-signatures -fplugin=Test.Tasty.Inspection.Plugin #-} +#endif + module Tests.Properties.LowLevel (testLowLevel) where import Control.Applicative ((<$>), pure) import Control.Exception as E (SomeException, catch, evaluate) import Data.Int (Int32, Int64) import Data.Text.Foreign -import Data.Text.Internal (mul, mul32, mul64) -import Data.Word (Word16, Word32) +import Data.Text.Internal (Text(..), mul, mul32, mul64, safe) +import Data.Word (Word8, Word16, Word32) import System.IO.Unsafe (unsafePerformIO) -import Test.QuickCheck.Monadic import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit (testCase, assertEqual) import Test.Tasty.QuickCheck (testProperty) import Test.QuickCheck hiding ((.&.)) import Tests.QuickCheckUtils import Tests.Utils -import qualified Data.Bits as Bits (shiftL, shiftR) import qualified Data.Text as T -import qualified Data.Text.Internal.Unsafe.Shift as U import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL import qualified System.IO as IO +#ifdef MIN_VERSION_tasty_inspection_testing +import Test.Tasty.Inspection (inspectObligations, hasNoTypes, doesNotUseAnyOf) +import qualified Data.Text.Internal.Fusion as S +import qualified Data.Text.Internal.Fusion.Common as S +import qualified GHC.CString as GHC +#endif + mulRef :: (Integral a, Bounded a) => a -> a -> Maybe a mulRef a b | ab < bot || ab > top = Nothing @@ -47,31 +60,33 @@ t_mul :: Int -> Int -> Property t_mul a b = mulRef a b === eval mul a b --- Bit shifts. -shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k - where width = round (log (fromIntegral m) / log 2 :: Double) - (m,_) = (maxBound, m == w) -shiftR w = forAll (choose (0,width-1)) $ \k -> Bits.shiftR w k == U.shiftR w k - where width = round (log (fromIntegral m) / log 2 :: Double) - (m,_) = (maxBound, m == w) - -shiftL_Int = shiftL :: Int -> Property -shiftL_Word16 = shiftL :: Word16 -> Property -shiftL_Word32 = shiftL :: Word32 -> Property -shiftR_Int = shiftR :: Int -> Property -shiftR_Word16 = shiftR :: Word16 -> Property -shiftR_Word32 = shiftR :: Word32 -> Property - -- Misc. -t_dropWord16 m t = dropWord16 m t `T.isSuffixOf` t -t_takeWord16 m t = takeWord16 m t `T.isPrefixOf` t -t_take_drop_16 m t = T.append (takeWord16 n t) (dropWord16 n t) === t - where n = small m -t_use_from t = monadicIO $ assert . (==t) =<< run (useAsPtr t fromPtr) +t_dropWord8 m t = dropWord8 m t `T.isSuffixOf` t +t_takeWord8 m t = takeWord8 m t `T.isPrefixOf` t +t_take_drop_8 (Small n) t = T.append (takeWord8 n t) (dropWord8 n t) === t +t_use_from t = ioProperty $ (==t) <$> useAsPtr t fromPtr t_copy t = T.copy t === t +t_literal_length1 = assertEqual xs (length xs) byteLen + where + xs = "\0\1\0\1\0" + Text _ _ byteLen = T.pack xs +t_literal_length2 = assertEqual xs (length xs) byteLen + where + xs = "\1\2\3\4\5" + Text _ _ byteLen = T.pack xs +t_literal_surrogates = assertEqual xs (T.pack xs) (T.pack ys) + where + ys = "\xd7ff \xd800 \xdbff \xdc00 \xdfff \xe000" + xs = map safe ys + +#ifdef MIN_VERSION_tasty_inspection_testing +t_literal_foo :: Text +t_literal_foo = T.pack "foo" +#endif + -- Input and output. -- t_put_get = write_read T.unlines T.filter put get @@ -83,10 +98,10 @@ t_write_read = write_read T.unlines T.filter T.hPutStr T.hGetContents tl_write_read = write_read TL.unlines TL.filter TL.hPutStr TL.hGetContents -t_write_read_line e m b t = write_read head T.filter T.hPutStrLn - T.hGetLine e m b [t] -tl_write_read_line e m b t = write_read head TL.filter TL.hPutStrLn - TL.hGetLine e m b [t] +t_write_read_line m b t = write_read head T.filter T.hPutStrLn + T.hGetLine m b [t] +tl_write_read_line m b t = write_read head TL.filter TL.hPutStrLn + TL.hGetLine m b [t] testLowLevel :: TestTree @@ -98,21 +113,24 @@ testProperty "t_mul64" t_mul64 ], - testGroup "shifts" [ - testProperty "shiftL_Int" shiftL_Int, - testProperty "shiftL_Word16" shiftL_Word16, - testProperty "shiftL_Word32" shiftL_Word32, - testProperty "shiftR_Int" shiftR_Int, - testProperty "shiftR_Word16" shiftR_Word16, - testProperty "shiftR_Word32" shiftR_Word32 - ], - testGroup "misc" [ - testProperty "t_dropWord16" t_dropWord16, - testProperty "t_takeWord16" t_takeWord16, - testProperty "t_take_drop_16" t_take_drop_16, + testProperty "t_dropWord8" t_dropWord8, + testProperty "t_takeWord8" t_takeWord8, + testProperty "t_take_drop_8" t_take_drop_8, testProperty "t_use_from" t_use_from, - testProperty "t_copy" t_copy + testProperty "t_copy" t_copy, + testCase "t_literal_length1" t_literal_length1, + testCase "t_literal_length2" t_literal_length2, + testCase "t_literal_surrogates" t_literal_surrogates +#ifdef MIN_VERSION_tasty_inspection_testing + , $(inspectObligations + [ (`hasNoTypes` [''Char, ''[]]) + , (`doesNotUseAnyOf` ['T.pack, 'S.unstream, 'T.map, 'safe, 'S.streamList]) + , (`doesNotUseAnyOf` ['GHC.unpackCString#, 'GHC.unpackCStringUtf8#]) + , (`doesNotUseAnyOf` ['T.unpackCString#, 'T.unpackCStringAscii#]) + ] + 't_literal_foo) +#endif ], testGroup "input-output" [ diff --git a/tests/Tests/Properties/Read.hs b/tests/Tests/Properties/Read.hs --- a/tests/Tests/Properties/Read.hs +++ b/tests/Tests/Properties/Read.hs @@ -1,7 +1,7 @@ -- | Tests for readers {-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Read ( testRead ) where @@ -12,7 +12,6 @@ import Test.Tasty.QuickCheck (testProperty) import Test.QuickCheck import Tests.QuickCheckUtils () -import Text.Show.Functions () import qualified Data.Text as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Read as TL @@ -41,14 +40,14 @@ t_read_rational p tol (n::Double) s = case p (T.pack (show n) `T.append` t) of - Left _err -> False - Right (n',t') -> t == t' && abs (n-n') <= tol + Left err -> counterexample err $ property False + Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol) where t = T.dropWhile isFloaty s tl_read_rational p tol (n::Double) s = case p (TL.pack (show n) `TL.append` t) of - Left _err -> False - Right (n',t') -> t == t' && abs (n-n') <= tol + Left err -> counterexample err $ property False + Right (n',t') -> t === t' .&&. property (abs (n-n') <= tol) where t = TL.dropWhile isFloaty s t_double = t_read_rational T.double 1e-13 diff --git a/tests/Tests/Properties/Substrings.hs b/tests/Tests/Properties/Substrings.hs --- a/tests/Tests/Properties/Substrings.hs +++ b/tests/Tests/Properties/Substrings.hs @@ -1,6 +1,8 @@ -- | Tests for substring functions (@take@, @split@, @isInfixOf@, etc.) -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# LANGUAGE ViewPatterns #-} + +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Substrings ( testSubstrings ) where @@ -9,75 +11,104 @@ import Test.QuickCheck import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) -import Test.QuickCheck.Unicode (char) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.List as L import qualified Data.Text as T import qualified Data.Text.Internal.Fusion as S import qualified Data.Text.Internal.Fusion.Common as S +import qualified Data.Text.Internal.Lazy as TL (Text(..)) import qualified Data.Text.Internal.Lazy.Fusion as SL import qualified Data.Text.Lazy as TL import qualified Tests.SlowFunctions as Slow s_take n = L.take n `eqP` (unpackS . S.take n) -s_take_s m = L.take n `eqP` (unpackS . S.unstream . S.take n) - where n = small m -sf_take p n = (L.take n . L.filter p) `eqP` +s_take_s (Small n) = L.take n `eqP` (unpackS . S.unstream . S.take n) +sf_take (applyFun -> p) n + = (L.take n . L.filter p) `eqP` (unpackS . S.take n . S.filter p) t_take n = L.take n `eqP` (unpackS . T.take n) t_takeEnd n = (L.reverse . L.take n . L.reverse) `eqP` (unpackS . T.takeEnd n) -tl_take n = L.take n `eqP` (unpackS . TL.take (fromIntegral n)) -tl_takeEnd n = (L.reverse . L.take (fromIntegral n) . L.reverse) `eqP` +tl_take n = L.genericTake n `eqP` (unpackS . TL.take n) +tl_take_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + L.genericTake n `eqP` (unpackS . TL.take n) +tl_takeEnd n = (L.reverse . L.genericTake n . L.reverse) `eqP` (unpackS . TL.takeEnd n) +tl_takeEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + (L.reverse . L.genericTake n . L.reverse) `eqP` + (unpackS . TL.takeEnd n) + s_drop n = L.drop n `eqP` (unpackS . S.drop n) -s_drop_s m = L.drop n `eqP` (unpackS . S.unstream . S.drop n) - where n = small m -sf_drop p n = (L.drop n . L.filter p) `eqP` - (unpackS . S.drop n . S.filter p) +s_drop_s (Small n) = L.drop n `eqP` (unpackS . S.unstream . S.drop n) +sf_drop (applyFun -> p) n + = (L.drop n . L.filter p) `eqP` (unpackS . S.drop n . S.filter p) t_drop n = L.drop n `eqP` (unpackS . T.drop n) t_dropEnd n = (L.reverse . L.drop n . L.reverse) `eqP` (unpackS . T.dropEnd n) -tl_drop n = L.drop n `eqP` (unpackS . TL.drop (fromIntegral n)) -tl_dropEnd n = (L.reverse . L.drop n . L.reverse) `eqP` - (unpackS . TL.dropEnd (fromIntegral n)) -s_take_drop m = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n) - where n = small m -s_take_drop_s m = (L.take n . L.drop n) `eqP` +tl_drop n = L.genericDrop n `eqP` (unpackS . TL.drop n) +tl_drop_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in L.genericDrop n `eqP` (unpackS . TL.drop n) +tl_dropEnd n = (L.reverse . L.genericDrop n . L.reverse) `eqP` + (unpackS . TL.dropEnd n) +tl_dropEnd_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in (L.reverse . L.genericDrop n . L.reverse) `eqP` + (unpackS . TL.dropEnd n) + +s_take_drop (Small n) = (L.take n . L.drop n) `eqP` (unpackS . S.take n . S.drop n) +s_take_drop_s (Small n) = (L.take n . L.drop n) `eqP` (unpackS . S.unstream . S.take n . S.drop n) - where n = small m -s_takeWhile p = L.takeWhile p `eqP` (unpackS . S.takeWhile p) -s_takeWhile_s p = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p) -sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP` - (unpackS . S.takeWhile p . S.filter q) -noMatch = do - c <- char - d <- suchThat char (/= c) - return (c,d) -t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p) -tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p) -t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP` +s_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . S.takeWhile p) +s_takeWhile_s (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . S.unstream . S.takeWhile p) +sf_takeWhile (applyFun -> q) (applyFun -> p) + = (L.takeWhile p . L.filter q) `eqP` (unpackS . S.takeWhile p . S.filter q) + +data NoMatch = NoMatch Char Char + deriving (Eq, Show) + +instance Arbitrary NoMatch where + arbitrary = do + c <- arbitraryUnicodeChar + d <- suchThat arbitraryUnicodeChar (/= c) + return $ NoMatch c d + shrink (NoMatch c d) = fmap (NoMatch c) (filter (/= c) (shrink d)) + ++ fmap (`NoMatch` d) (filter (/= d) (shrink c)) + +t_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . T.takeWhile p) +tl_takeWhile (applyFun -> p) + = L.takeWhile p `eqP` (unpackS . TL.takeWhile p) +t_takeWhileEnd (applyFun -> p) + = (L.reverse . L.takeWhile p . L.reverse) `eqP` (unpackS . T.takeWhileEnd p) -t_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> T.null $ - T.takeWhileEnd (==d) (T.snoc t c) -tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP` +t_takeWhileEnd_null t (NoMatch c d) + = T.null $ T.takeWhileEnd (==d) (T.snoc t c) +tl_takeWhileEnd (applyFun -> p) + = (L.reverse . L.takeWhile p . L.reverse) `eqP` (unpackS . TL.takeWhileEnd p) -tl_takeWhileEnd_null t = forAll noMatch $ \(c,d) -> TL.null $ - TL.takeWhileEnd (==d) (TL.snoc t c) -s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) -s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p) -sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP` +tl_takeWhileEnd_null t (NoMatch c d) + = TL.null $ TL.takeWhileEnd (==d) (TL.snoc t c) +s_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +s_dropWhile_s (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p) +sf_dropWhile (applyFun -> q) (applyFun -> p) + = (L.dropWhile p . L.filter q) `eqP` (unpackS . S.dropWhile p . S.filter q) -t_dropWhile p = L.dropWhile p `eqP` (unpackS . T.dropWhile p) -tl_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p) -t_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` +t_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . T.dropWhile p) +tl_dropWhile (applyFun -> p) + = L.dropWhile p `eqP` (unpackS . S.dropWhile p) +t_dropWhileEnd (applyFun -> p) + = (L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . T.dropWhileEnd p) -tl_dropWhileEnd p = (L.reverse . L.dropWhile p . L.reverse) `eqP` +tl_dropWhileEnd (applyFun -> p) + = (L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . TL.dropWhileEnd p) -t_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) +t_dropAround (applyFun -> p) + = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . T.dropAround p) -tl_dropAround p = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) +tl_dropAround (applyFun -> p) + = (L.dropWhile p . L.reverse . L.dropWhile p . L.reverse) `eqP` (unpackS . TL.dropAround p) t_stripStart = T.dropWhile isSpace `eq` T.stripStart tl_stripStart = TL.dropWhile isSpace `eq` TL.stripStart @@ -85,11 +116,15 @@ tl_stripEnd = TL.dropWhileEnd isSpace `eq` TL.stripEnd t_strip = T.dropAround isSpace `eq` T.strip tl_strip = TL.dropAround isSpace `eq` TL.strip + t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n) -tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n)) -t_span p = L.span p `eqP` (unpack2 . T.span p) -tl_span p = L.span p `eqP` (unpack2 . TL.span p) +tl_splitAt n = L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n) +tl_splitAt_maxBound m = let n = fromIntegral (m :: Int) + fromIntegral (maxBound :: Int) in + L.genericSplitAt n `eqP` (unpack2 . TL.splitAt n) +t_span (applyFun -> p) = L.span p `eqP` (unpack2 . T.span p) +tl_span (applyFun -> p) = L.span p `eqP` (unpack2 . TL.span p) + t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s) where squid t | T.null s = error "empty" | otherwise = t @@ -108,44 +143,45 @@ tl_breakOnEnd_end (NotEmpty s) t = let (m,k) = TL.breakOnEnd s t in k `TL.isSuffixOf` t && (TL.null m || s `TL.isSuffixOf` m) -t_break p = L.break p `eqP` (unpack2 . T.break p) -tl_break p = L.break p `eqP` (unpack2 . TL.break p) +t_break (applyFun -> p) + = L.break p `eqP` (unpack2 . T.break p) +tl_break (applyFun -> p) + = L.break p `eqP` (unpack2 . TL.break p) t_group = L.group `eqP` (map unpackS . T.group) tl_group = L.group `eqP` (map unpackS . TL.group) -t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p) -tl_groupBy p = L.groupBy p `eqP` (map unpackS . TL.groupBy p) +t_groupBy (applyFun2 -> p) + = L.groupBy p `eqP` (map unpackS . T.groupBy p) +tl_groupBy (applyFun2 -> p) + = L.groupBy p `eqP` (map unpackS . TL.groupBy p) t_inits = L.inits `eqP` (map unpackS . T.inits) tl_inits = L.inits `eqP` (map unpackS . TL.inits) t_tails = L.tails `eqP` (map unpackS . T.tails) -tl_tails = unsquare $ - L.tails `eqP` (map unpackS . TL.tails) -t_findAppendId = unsquare $ \(NotEmpty s) ts -> +tl_tails = L.tails `eqPSqrt` (map unpackS . TL.tails) +t_findAppendId = \(Sqrt (NotEmpty s)) ts -> let t = T.intercalate s ts - in all (==t) $ map (uncurry T.append) (T.breakOnAll s t) -tl_findAppendId = unsquare $ \(NotEmpty s) ts -> + in conjoin $ map (=== t) $ map (uncurry T.append) (T.breakOnAll s t) +tl_findAppendId = \(Sqrt (NotEmpty s)) ts -> let t = TL.intercalate s ts - in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t) -t_findContains = unsquare $ \(NotEmpty s) -> + in conjoin $ map (=== t) $ map (uncurry TL.append) (TL.breakOnAll s t) +t_findContains = \(Sqrt (NotEmpty s)) -> all (T.isPrefixOf s . snd) . T.breakOnAll s . T.intercalate s -tl_findContains = unsquare $ \(NotEmpty s) -> all (TL.isPrefixOf s . snd) . - TL.breakOnAll s . TL.intercalate s +tl_findContains = \(Sqrt (NotEmpty s)) -> all (TL.isPrefixOf s . snd) . + TL.breakOnAll s . TL.intercalate s sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s -t_splitOn_split s = unsquare $ - (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s -tl_splitOn_split s = unsquare $ - ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq` - (map TL.fromStrict . T.splitOn s)) . T.intercalate s +t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s . unSqrt +tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq` + (map TL.fromStrict . T.splitOn s)) . T.intercalate s . unSqrt t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t) tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t) -t_split p = split p `eqP` (map unpackS . T.split p) +t_split (applyFun -> p) = split p `eqP` (map unpackS . T.split p) t_split_count c = (L.length . T.split (==c)) `eq` ((1+) . T.count (T.singleton c)) t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c) -tl_split p = split p `eqP` (map unpackS . TL.split p) +tl_split (applyFun -> p) = split p `eqP` (map unpackS . TL.split p) split :: (a -> Bool) -> [a] -> [[a]] split _ [] = [[]] @@ -154,11 +190,11 @@ | otherwise = l : loop (tail s') where (l, s') = break p s -t_chunksOf_same_lengths k = all ((==k) . T.length) . ini . T.chunksOf k +t_chunksOf_same_lengths k = conjoin . map ((===k) . T.length) . ini . T.chunksOf k where ini [] = [] ini xs = init xs -t_chunksOf_length k t = len == T.length t || (k <= 0 && len == 0) +t_chunksOf_length k t = len === T.length t .||. property (k <= 0 && len == 0) where len = L.sum . L.map T.length $ T.chunksOf k t tl_chunksOf k = T.chunksOf k `eq` (map (T.concat . TL.toChunks) . @@ -166,31 +202,27 @@ t_lines = L.lines `eqP` (map unpackS . T.lines) tl_lines = L.lines `eqP` (map unpackS . TL.lines) -{- -t_lines' = lines' `eqP` (map unpackS . T.lines') - where lines' "" = [] - lines' s = let (l, s') = break eol s - in l : case s' of - [] -> [] - ('\r':'\n':s'') -> lines' s'' - (_:s'') -> lines' s'' - eol c = c == '\r' || c == '\n' --} -t_words = L.words `eqP` (map unpackS . T.words) +t_lines_spacy = (L.lines `eqP` (map unpackS . T.lines)) . getSpacyString +tl_lines_spacy = (L.lines `eqP` (map unpackS . TL.lines)) . getSpacyString +tl_lines_laziness = TL.head (head (TL.lines (TL.replicate 1000000000000000 (TL.singleton 'a')))) === 'a' + +tl_lines_specialCase = TL.lines (TL.Chunk (T.pack "foo") $ TL.Chunk (T.pack "bar\nbaz\n") $ TL.Empty) === [TL.pack "foobar", TL.pack "baz"] + +t_words = L.words `eqP` (map unpackS . T.words) tl_words = L.words `eqP` (map unpackS . TL.words) -t_unlines = unsquare $ - L.unlines `eq` (unpackS . T.unlines . map packS) -tl_unlines = unsquare $ - L.unlines `eq` (unpackS . TL.unlines . map packS) -t_unwords = unsquare $ - L.unwords `eq` (unpackS . T.unwords . map packS) -tl_unwords = unsquare $ - L.unwords `eq` (unpackS . TL.unwords . map packS) +t_words_spacy = (L.words `eqP` (map unpackS . T.words)) . getSpacyString +tl_words_spacy = (L.words `eqP` (map unpackS . TL.words)) . getSpacyString +t_unlines = (L.unlines . unSqrt) `eq` (unpackS . T.unlines . map packS . unSqrt) +tl_unlines = (L.unlines . unSqrt) `eq` (unpackS . TL.unlines . map packS . unSqrt) +t_unwords = (L.unwords . unSqrt) `eq` (unpackS . T.unwords . map packS . unSqrt) +tl_unwords = (L.unwords . unSqrt) `eq` (unpackS . TL.unwords . map packS . unSqrt) + s_isPrefixOf s = L.isPrefixOf s `eqP` (S.isPrefixOf (S.stream $ packS s) . S.stream) -sf_isPrefixOf p s = (L.isPrefixOf s . L.filter p) `eqP` +sf_isPrefixOf (applyFun -> p) s + = (L.isPrefixOf s . L.filter p) `eqP` (S.isPrefixOf (S.stream $ packS s) . S.filter p . S.stream) t_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (packS s) tl_isPrefixOf s = L.isPrefixOf s`eqP` TL.isPrefixOf (packS s) @@ -214,14 +246,14 @@ commonPrefixes _ _ = Nothing t_commonPrefixes a b (NonEmpty p) - = commonPrefixes pa pb == + = commonPrefixes pa pb === repack `fmap` T.commonPrefixes (packS pa) (packS pb) where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) pa = p ++ a pb = p ++ b tl_commonPrefixes a b (NonEmpty p) - = commonPrefixes pa pb == + = commonPrefixes pa pb === repack `fmap` TL.commonPrefixes (packS pa) (packS pb) where repack (x,y,z) = (unpackS x,unpackS y,unpackS z) pa = p ++ a @@ -237,14 +269,18 @@ testProperty "t_take" t_take, testProperty "t_takeEnd" t_takeEnd, testProperty "tl_take" tl_take, + testProperty "tl_take_maxBound" tl_take_maxBound, testProperty "tl_takeEnd" tl_takeEnd, + testProperty "tl_takeEnd_maxBound" tl_takeEnd_maxBound, testProperty "s_drop" s_drop, testProperty "s_drop_s" s_drop_s, testProperty "sf_drop" sf_drop, testProperty "t_drop" t_drop, testProperty "t_dropEnd" t_dropEnd, testProperty "tl_drop" tl_drop, + testProperty "tl_drop_maxBound" tl_drop_maxBound, testProperty "tl_dropEnd" tl_dropEnd, + testProperty "tl_dropEnd_maxBound" tl_dropEnd_maxBound, testProperty "s_take_drop" s_take_drop, testProperty "s_take_drop_s" s_take_drop_s, testProperty "s_takeWhile" s_takeWhile, @@ -273,6 +309,7 @@ testProperty "tl_strip" tl_strip, testProperty "t_splitAt" t_splitAt, testProperty "tl_splitAt" tl_splitAt, + testProperty "tl_splitAt_maxBound" tl_splitAt_maxBound, testProperty "t_span" t_span, testProperty "tl_span" tl_span, testProperty "t_breakOn_id" t_breakOn_id, @@ -317,9 +354,14 @@ testGroup "lines and words" [ testProperty "t_lines" t_lines, testProperty "tl_lines" tl_lines, - --testProperty "t_lines'" t_lines', + testProperty "t_lines_spacy" t_lines_spacy, + testProperty "tl_lines_spacy" tl_lines_spacy, + testProperty "tl_lines_laziness" tl_lines_laziness, + testProperty "tl_lines_specialCase" tl_lines_specialCase, testProperty "t_words" t_words, testProperty "tl_words" tl_words, + testProperty "t_words_spacy" t_words_spacy, + testProperty "tl_words_spacy" tl_words_spacy, testProperty "t_unlines" t_unlines, testProperty "tl_unlines" tl_unlines, testProperty "t_unwords" t_unwords, diff --git a/tests/Tests/Properties/Text.hs b/tests/Tests/Properties/Text.hs --- a/tests/Tests/Properties/Text.hs +++ b/tests/Tests/Properties/Text.hs @@ -1,7 +1,8 @@ -- | Tests for operations that don't fit in the other @Test.Properties.*@ modules. {-# LANGUAGE BangPatterns #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-} +{-# LANGUAGE ViewPatterns #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Text ( testText ) where @@ -14,7 +15,6 @@ import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Data.Char as C import qualified Data.List as L import qualified Data.Text as T @@ -38,35 +38,40 @@ tl_chunk_unchunk = (TL.fromChunks . TL.toChunks) `eq` id tl_from_to_strict = (TL.fromStrict . TL.toStrict) `eq` id +s_map (applyFun -> f) = map f `eqP` (unpackS . S.map f) +s_map_s (applyFun -> f) = map f `eqP` (unpackS . S.unstream . S.map f) +sf_map (applyFun -> p) (applyFun -> f) = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) -s_map f = map f `eqP` (unpackS . S.map f) -s_map_s f = map f `eqP` (unpackS . S.unstream . S.map f) -sf_map p f = (map f . L.filter p) `eqP` (unpackS . S.map f . S.filter p) -t_map f = map f `eqP` (unpackS . T.map f) -tl_map f = map f `eqP` (unpackS . TL.map f) -s_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . S.intercalate (packS c) . map packS) -t_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . T.intercalate (packS c) . map packS) -tl_intercalate c = unsquare $ - L.intercalate c `eq` - (unpackS . TL.intercalate (TL.pack c) . map TL.pack) +t_map (applyFun -> f) = map f `eqP` (unpackS . T.map f) +tl_map (applyFun -> f) = map f `eqP` (unpackS . TL.map f) +t_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . T.map f . T.map g) +tl_map_map (applyFun -> f) (applyFun -> g) = (map f . map g) `eqP` (unpackS . TL.map f . TL.map g) +t_length_map (applyFun -> f) = (L.length . map f) `eqP` (T.length . T.map f) +tl_length_map (applyFun -> f) = (L.genericLength . map f) `eqP` (TL.length . TL.map f) + +s_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . S.intercalate (packS c) . map packS . unSqrt) +t_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . T.intercalate (packS c) . map packS . unSqrt) +tl_intercalate c = (L.intercalate c . unSqrt) `eq` + (unpackS . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) +t_length_intercalate c = (L.length . L.intercalate c . unSqrt) `eq` + (T.length . T.intercalate (packS c) . map packS . unSqrt) +tl_length_intercalate c = (L.genericLength . L.intercalate c . unSqrt) `eq` + (TL.length . TL.intercalate (TL.pack c) . map TL.pack . unSqrt) s_intersperse c = L.intersperse c `eqP` (unpackS . S.intersperse c) s_intersperse_s c = L.intersperse c `eqP` (unpackS . S.unstream . S.intersperse c) -sf_intersperse p c= (L.intersperse c . L.filter p) `eqP` +sf_intersperse (applyFun -> p) c + = (L.intersperse c . L.filter p) `eqP` (unpackS . S.intersperse c . S.filter p) -t_intersperse c = unsquare $ - L.intersperse c `eqP` (unpackS . T.intersperse c) -tl_intersperse c = unsquare $ - L.intersperse c `eqP` (unpackS . TL.intersperse c) -t_transpose = unsquare $ - L.transpose `eq` (map unpackS . T.transpose . map packS) -tl_transpose = unsquare $ - L.transpose `eq` (map unpackS . TL.transpose . map TL.pack) +t_intersperse c = L.intersperse c `eqPSqrt` (unpackS . T.intersperse c) +tl_intersperse c = L.intersperse c `eqPSqrt` (unpackS . TL.intersperse c) +t_length_intersperse c = (L.length . L.intersperse c) `eqPSqrt` (T.length . T.intersperse c) +tl_length_intersperse c = (L.genericLength . L.intersperse c) `eqPSqrt` (TL.length . TL.intersperse c) +t_transpose = (L.transpose . unSqrt) `eq` (map unpackS . T.transpose . map packS . unSqrt) +tl_transpose = (L.transpose . unSqrt) `eq` (map unpackS . TL.transpose . map TL.pack . unSqrt) t_reverse = L.reverse `eqP` (unpackS . T.reverse) tl_reverse = L.reverse `eqP` (unpackS . TL.reverse) t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream) @@ -91,7 +96,7 @@ s_toCaseFold_length xs = S.length (S.toCaseFold s) >= length xs where s = S.streamList xs -sf_toCaseFold_length p xs = +sf_toCaseFold_length (applyFun -> p) xs = (S.length . S.toCaseFold . S.filter p $ s) >= (length . L.filter p $ xs) where s = S.streamList xs t_toCaseFold_length t = T.length (T.toCaseFold t) >= T.length t @@ -123,6 +128,20 @@ -- https://en.wikipedia.org/wiki/Georgian_Extended isGeorgian c = c >= '\4256' && c < '\4352' +ascii_toLower (ASCIIString xs) = map C.toLower xs === T.unpack (T.toLower (T.pack xs)) +ascii_toUpper (ASCIIString xs) = map C.toUpper xs === T.unpack (T.toUpper (T.pack xs)) +ascii_toCaseFold (ASCIIString xs) = map C.toLower xs === T.unpack (T.toCaseFold (T.pack xs)) + +ascii_toTitle (ASCIIString xs) = referenceToTitle False xs === T.unpack (T.toTitle (T.pack xs)) + where + referenceToTitle _ [] = [] + referenceToTitle False (y : ys) + | C.isLetter y = C.toUpper y : referenceToTitle True ys + | otherwise = y : referenceToTitle False ys + referenceToTitle True (y : ys) + | C.isLetter y = C.toLower y : referenceToTitle True ys + | otherwise = y : referenceToTitle (not (C.isSpace y)) ys + justifyLeft k c xs = xs ++ L.replicate (k - length xs) c justifyRight m n xs = L.replicate (m - length xs) n ++ xs center k c xs @@ -138,7 +157,8 @@ s_justifyLeft_s k c = justifyLeft j c `eqP` (unpackS . S.unstream . S.justifyLeftI j c) where j = fromIntegral (k :: Word8) -sf_justifyLeft p k c = (justifyLeft j c . L.filter p) `eqP` +sf_justifyLeft (applyFun -> p) k c + = (justifyLeft j c . L.filter p) `eqP` (unpackS . S.justifyLeftI j c . S.filter p) where j = fromIntegral (k :: Word8) t_justifyLeft k c = justifyLeft j c `eqP` (unpackS . T.justifyLeft j c) @@ -158,46 +178,90 @@ t_elem c = L.elem c `eqP` T.elem c tl_elem c = L.elem c `eqP` TL.elem c -sf_elem p c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) -sf_filter q p = (L.filter p . L.filter q) `eqP` - (unpackS . S.filter p . S.filter q) -t_filter p = L.filter p `eqP` (unpackS . T.filter p) -tl_filter p = L.filter p `eqP` (unpackS . TL.filter p) -sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) -t_find p = L.find p `eqP` T.find p -tl_find p = L.find p `eqP` TL.find p -t_partition p = L.partition p `eqP` (unpack2 . T.partition p) -tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p) +sf_elem (applyFun -> p) c = (L.elem c . L.filter p) `eqP` (S.elem c . S.filter p) +sf_filter (applyFun -> q) (applyFun -> p) + = (L.filter p . L.filter q) `eqP` (unpackS . S.filter p . S.filter q) -sf_index p s = forAll (choose (-l,l*2)) - ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) +t_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . T.filter p) +tl_filter (applyFun -> p) + = L.filter p `eqP` (unpackS . TL.filter p) +t_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . T.filter p . T.filter q) +tl_filter_filter (applyFun -> p) (applyFun -> q) + = (L.filter p . L.filter q) `eqP` (unpackS . TL.filter p . TL.filter q) +t_length_filter (applyFun -> p) + = (L.length . L.filter p) `eqP` (T.length . T.filter p) +tl_length_filter (applyFun -> p) + = (L.genericLength . L.filter p) `eqP` (TL.length . TL.filter p) + +sf_findBy (applyFun -> q) (applyFun -> p) + = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q) +t_find (applyFun -> p) = L.find p `eqP` T.find p +tl_find (applyFun -> p) = L.find p `eqP` TL.find p +t_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . T.partition p) +tl_partition (applyFun -> p) = L.partition p `eqP` (unpack2 . TL.partition p) + +sf_index (applyFun -> p) s i = ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s)) j where l = L.length s -t_index s = forAll (choose (-l,l*2)) ((s L.!!) `eq` T.index (packS s)) + j = if l == 0 then 0 else i `mod` (3 * l) - l +t_index s i = ((s L.!!) `eq` T.index (packS s)) j where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l -tl_index s = forAll (choose (-l,l*2)) - ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) +tl_index s i = ((s L.!!) `eq` (TL.index (packS s) . fromIntegral)) j where l = L.length s + j = if l == 0 then 0 else i `mod` (3 * l) - l -t_findIndex p = L.findIndex p `eqP` T.findIndex p +t_findIndex (applyFun -> p) = L.findIndex p `eqP` T.findIndex p t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq` TL.count t t_zip s = L.zip s `eqP` T.zip (packS s) tl_zip s = L.zip s `eqP` TL.zip (packS s) -sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` +sf_zipWith (applyFun -> p) (applyFun2 -> c) s + = (L.zipWith c (L.filter p s) . L.filter p) `eqP` (unpackS . S.zipWith c (S.filter p $ packS s) . S.filter p) -t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) -tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s)) +tl_zipWith (applyFun2 -> c) s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s)) +t_length_zipWith (applyFun2 -> c) s = (L.length . L.zipWith c s) `eqP` (T.length . T.zipWith c (packS s)) +tl_length_zipWith (applyFun2 -> c) s = (L.genericLength . L.zipWith c s) `eqP` (TL.length . TL.zipWith c (packS s)) t_indices (NotEmpty s) = Slow.indices s `eq` T.indices s tl_indices (NotEmpty s) = lazyIndices s `eq` S.indices s where lazyIndices ss t = map fromIntegral $ Slow.indices (conc ss) (conc t) conc = T.concat . TL.toChunks -t_indices_occurs = unsquare $ \(NotEmpty t) ts -> +t_indices_occurs = \(Sqrt (NotEmpty t)) ts -> let s = T.intercalate t ts in Slow.indices t s === T.indices t s +t_indices_drop5 = T.indices (T.pack "no") (T.drop 5 (T.pack "abcdefghijklmno")) === [8] +tl_indices_drop5 = S.indices (TL.pack "no") (TL.drop 5 (TL.pack "abcdefghijklmno")) === [8] + +t_indices_drop n s pref suff = T.indices s t === Slow.indices s t + where + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_drop n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + t = TL.drop n $ pref `TL.append` s `TL.append` suff + +tl_indices_chunked = S.indices (TL.pack "1234") (TL.pack "1" `TL.append` TL.pack "234" `TL.append` TL.pack "567") === [0] +tl_indices_drop_chunked n s pref suff = + map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + -- constructing a pathologically chunked haystack + t = TL.concatMap TL.singleton $ TL.drop n $ pref `TL.append` s `TL.append` suff + +t_indices_char_drop n c pref suff = T.indices s t === Slow.indices s t + where + s = T.singleton c + t = T.drop n $ pref `T.append` s `T.append` suff +tl_indices_char_drop n c pref suff = map fromIntegral (S.indices s t) === Slow.indices (TL.toStrict s) (TL.toStrict t) + where + s = TL.singleton c + t = TL.drop n $ pref `TL.append` s `TL.append` suff + -- Make a stream appear shorter than it really is, to ensure that -- functions that consume inaccurately sized streams behave -- themselves. @@ -226,16 +290,26 @@ testProperty "s_map" s_map, testProperty "s_map_s" s_map_s, testProperty "sf_map" sf_map, + testProperty "t_map" t_map, testProperty "tl_map" tl_map, + testProperty "t_map_map" t_map_map, + testProperty "tl_map_map" tl_map_map, + testProperty "t_length_map" t_length_map, + testProperty "tl_length_map" tl_length_map, + testProperty "s_intercalate" s_intercalate, testProperty "t_intercalate" t_intercalate, testProperty "tl_intercalate" tl_intercalate, + testProperty "t_length_intercalate" t_length_intercalate, + testProperty "tl_length_intercalate" tl_length_intercalate, testProperty "s_intersperse" s_intersperse, testProperty "s_intersperse_s" s_intersperse_s, testProperty "sf_intersperse" sf_intersperse, testProperty "t_intersperse" t_intersperse, testProperty "tl_intersperse" tl_intersperse, + testProperty "t_length_intersperse" t_length_intersperse, + testProperty "tl_length_intersperse" tl_length_intersperse, testProperty "t_transpose" t_transpose, testProperty "tl_transpose" tl_transpose, testProperty "t_reverse" t_reverse, @@ -256,7 +330,11 @@ testProperty "t_toUpper_upper" t_toUpper_upper, testProperty "tl_toUpper_upper" tl_toUpper_upper, testProperty "t_toTitle_title" t_toTitle_title, - testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower + testProperty "t_toTitle_1stNotLower" t_toTitle_1stNotLower, + testProperty "ascii_toLower" ascii_toLower, + testProperty "ascii_toUpper" ascii_toUpper, + testProperty "ascii_toTitle" ascii_toTitle, + testProperty "ascii_toCaseFold" ascii_toCaseFold ], testGroup "justification" [ @@ -279,6 +357,10 @@ testProperty "sf_filter" sf_filter, testProperty "t_filter" t_filter, testProperty "tl_filter" tl_filter, + testProperty "t_filter_filter" t_filter_filter, + testProperty "tl_filter_filter" tl_filter_filter, + testProperty "t_length_filter" t_length_filter, + testProperty "tl_length_filter" tl_length_filter, testProperty "sf_findBy" sf_findBy, testProperty "t_find" t_find, testProperty "tl_find" tl_find, @@ -295,7 +377,16 @@ testProperty "tl_count" tl_count, testProperty "t_indices" t_indices, testProperty "tl_indices" tl_indices, - testProperty "t_indices_occurs" t_indices_occurs + testProperty "t_indices_occurs" t_indices_occurs, + + testProperty "t_indices_drop5" t_indices_drop5, + testProperty "tl_indices_drop5" tl_indices_drop5, + testProperty "t_indices_drop" t_indices_drop, + testProperty "tl_indices_drop" tl_indices_drop, + testProperty "tl_indices_chunked" tl_indices_chunked, + testProperty "tl_indices_drop_chunked" tl_indices_drop_chunked, + testProperty "t_indices_char_drop" t_indices_char_drop, + testProperty "tl_indices_char_drop" tl_indices_char_drop ], testGroup "zips" [ @@ -303,6 +394,8 @@ testProperty "tl_zip" tl_zip, testProperty "sf_zipWith" sf_zipWith, testProperty "t_zipWith" t_zipWith, - testProperty "tl_zipWith" tl_zipWith + testProperty "tl_zipWith" tl_zipWith, + testProperty "t_length_zipWith" t_length_zipWith, + testProperty "tl_length_zipWith" tl_length_zipWith ] ] diff --git a/tests/Tests/Properties/Transcoding.hs b/tests/Tests/Properties/Transcoding.hs --- a/tests/Tests/Properties/Transcoding.hs +++ b/tests/Tests/Properties/Transcoding.hs @@ -1,64 +1,76 @@ -- | Tests for encoding and decoding {-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures -fno-warn-unused-imports -fno-warn-deprecations #-} +{-# OPTIONS_GHC -fno-warn-missing-signatures #-} module Tests.Properties.Transcoding ( testTranscoding ) where -import Control.Applicative ((<$>), (<*>)) -import Data.Bits ((.&.)) +import Data.Bits ((.&.), shiftR) import Data.Char (chr, ord) -import Data.Text.Encoding.Error (UnicodeException) -import Data.Text.Internal.Encoding.Utf8 (ord2, ord3, ord4) import Test.QuickCheck hiding ((.&.)) -import Test.QuickCheck.Property (Property(..)) -import Test.QuickCheck.Monadic import Test.Tasty (TestTree, testGroup) import Test.Tasty.QuickCheck (testProperty) import Tests.QuickCheckUtils -import Text.Show.Functions () import qualified Control.Exception as Exception import qualified Data.Bits as Bits (shiftL, shiftR) import qualified Data.ByteString as B +import qualified Data.ByteString.Builder as B +import qualified Data.ByteString.Builder.Extra as B +import qualified Data.ByteString.Builder.Prim as BP +import qualified Data.ByteString.Char8 as BC import qualified Data.ByteString.Lazy as BL +import qualified Data.ByteString.Lazy.Char8 as BLC import qualified Data.Text as T import qualified Data.Text.Encoding as E import qualified Data.Text.Encoding.Error as E import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Encoding as EL --- Note: this silently truncates code-points > 255 to 8-bit due to 'B.pack' -encodeL1 :: T.Text -> B.ByteString -encodeL1 = B.pack . map (fromIntegral . fromEnum) . T.unpack -encodeLazyL1 :: TL.Text -> BL.ByteString -encodeLazyL1 = BL.fromChunks . map encodeL1 . TL.toChunks - t_ascii t = E.decodeASCII (E.encodeUtf8 a) === a where a = T.map (\c -> chr (ord c `mod` 128)) t tl_ascii t = EL.decodeASCII (EL.encodeUtf8 a) === a where a = TL.map (\c -> chr (ord c `mod` 128)) t -t_latin1 t = E.decodeLatin1 (encodeL1 a) === a - where a = T.map (\c -> chr (ord c `mod` 256)) t -tl_latin1 t = EL.decodeLatin1 (encodeLazyL1 a) === a - where a = TL.map (\c -> chr (ord c `mod` 256)) t -t_utf8 = forAll genUnicode $ (E.decodeUtf8 . E.encodeUtf8) `eq` id -t_utf8' = forAll genUnicode $ (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) -tl_utf8 = forAll genUnicode $ (EL.decodeUtf8 . EL.encodeUtf8) `eq` id -tl_utf8' = forAll genUnicode $ (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right) -t_utf16LE = forAll genUnicode $ (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id -tl_utf16LE = forAll genUnicode $ (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id -t_utf16BE = forAll genUnicode $ (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id -tl_utf16BE = forAll genUnicode $ (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id -t_utf32LE = forAll genUnicode $ (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id -tl_utf32LE = forAll genUnicode $ (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id -t_utf32BE = forAll genUnicode $ (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id -tl_utf32BE = forAll genUnicode $ (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id -t_utf8_incr = forAll genUnicode $ \s (Positive n) -> (recode n `eq` id) s - where recode n = T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 . - E.encodeUtf8 +t_latin1 = E.decodeLatin1 `eq` (T.pack . BC.unpack) +tl_latin1 = EL.decodeLatin1 `eq` (TL.pack . BLC.unpack) +t_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id +t_utf8' = (E.decodeUtf8' . E.encodeUtf8) `eq` (id . Right) +tl_utf8 = (EL.decodeUtf8 . EL.encodeUtf8) `eq` id +tl_utf8' = (EL.decodeUtf8' . EL.encodeUtf8) `eq` (id . Right) +t_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id +tl_utf16LE = (EL.decodeUtf16LE . EL.encodeUtf16LE) `eq` id +t_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id +tl_utf16BE = (EL.decodeUtf16BE . EL.encodeUtf16BE) `eq` id +t_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id +tl_utf32LE = (EL.decodeUtf32LE . EL.encodeUtf32LE) `eq` id +t_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id +tl_utf32BE = (EL.decodeUtf32BE . EL.encodeUtf32BE) `eq` id + +runBuilder :: B.Builder -> B.ByteString +runBuilder = + -- Use smallish buffers to exercise bufferFull case as well + BL.toStrict . B.toLazyByteStringWith (B.safeStrategy 5 5) "" + +t_encodeUtf8Builder_ toBuilder = (runBuilder . toBuilder) `eq` E.encodeUtf8 + +t_encodeUtf8Builder_nonZeroOffset_ toBuilder (Positive n) = + (runBuilder . toBuilder . T.drop n) `eq` (E.encodeUtf8 . T.drop n) + +t_encodeUtf8Builder = t_encodeUtf8Builder_ E.encodeUtf8Builder +t_encodeUtf8Builder_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ E.encodeUtf8Builder + +t_encodeUtf8BuilderEscaped = t_encodeUtf8Builder_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) +t_encodeUtf8BuilderEscaped_nonZeroOffset = t_encodeUtf8Builder_nonZeroOffset_ (E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) + +t_encodeUtf8Builder_sanity t = + (runBuilder . E.encodeUtf8Builder) t === + (runBuilder . E.encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)) t + +t_utf8_incr (Positive n) = + (T.concat . map fst . feedChunksOf n E.streamDecodeUtf8 . E.encodeUtf8) `eq` id + feedChunksOf :: Int -> (B.ByteString -> E.Decoding) -> B.ByteString -> [(T.Text, B.ByteString)] feedChunksOf n f bs @@ -67,56 +79,56 @@ E.Some t b f' = f x in (t,b) : feedChunksOf n f' y -t_utf8_undecoded = forAll genUnicode $ \t -> +t_utf8_undecoded t = let b = E.encodeUtf8 t ls = concatMap (leftover . E.encodeUtf8 . T.singleton) . T.unpack $ t leftover = (++ [B.empty]) . init . tail . B.inits in (map snd . feedChunksOf 1 E.streamDecodeUtf8) b === ls -data Badness = Solo | Leading | Trailing - deriving (Eq, Show) +data InvalidUtf8 = InvalidUtf8 + { iu8Prefix :: T.Text + , iu8Invalid :: B.ByteString + , iu8Suffix :: T.Text + } deriving (Eq) -instance Arbitrary Badness where - arbitrary = elements [Solo, Leading, Trailing] +instance Show InvalidUtf8 where + show i = "InvalidUtf8 {prefix = " ++ show (iu8Prefix i) + ++ ", invalid = " ++ show (iu8Invalid i) + ++ ", suffix = " ++ show (iu8Suffix i) + ++ ", asBS = " ++ show (toByteString i) + ++ ", length = " ++ show (B.length (toByteString i)) + ++ "}" -t_utf8_err :: Badness -> Maybe DecodeErr -> Property -t_utf8_err bad mde = do - let gen = case bad of - Solo -> genInvalidUTF8 - Leading -> B.append <$> genInvalidUTF8 <*> genUTF8 - Trailing -> B.append <$> genUTF8 <*> genInvalidUTF8 - genUTF8 = E.encodeUtf8 <$> genUnicode - forAll gen $ \bs -> MkProperty $ - case mde of - -- generate an invalid character - Nothing -> do - c <- choose ('\x10000', maxBound) - let onErr _ _ = Just c - unProperty . monadicIO $ do - l <- run $ let len = T.length (E.decodeUtf8With onErr bs) - in (len `seq` return (Right len)) `Exception.catch` - (\(e::Exception.SomeException) -> return (Left e)) - assert $ case l of - Left err -> - "non-BMP replacement characters not supported" `T.isInfixOf` T.pack (show err) - Right _ -> False +toByteString :: InvalidUtf8 -> B.ByteString +toByteString (InvalidUtf8 a b c) = + E.encodeUtf8 a `B.append` b `B.append` E.encodeUtf8 c - -- generate a valid onErr - Just de -> do - onErr <- genDecodeErr de - unProperty . monadicIO $ do - l <- run $ let len = T.length (E.decodeUtf8With onErr bs) - in (len `seq` return (Right len)) `Exception.catch` - (\(e::UnicodeException) -> return (Left e)) - assert $ case l of - Left err -> length (show err) >= 0 - Right _ -> de /= Strict +instance Arbitrary InvalidUtf8 where + arbitrary = oneof + [ InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> pure mempty + , InvalidUtf8 <$> pure mempty <*> genInvalidUTF8 <*> arbitrary + , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> pure mempty + , InvalidUtf8 <$> arbitrary <*> genInvalidUTF8 <*> arbitrary + ] + shrink (InvalidUtf8 a b c) + = map (\c' -> InvalidUtf8 a b c') (shrink c) + ++ map (\a' -> InvalidUtf8 a' b c) (shrink a) -t_utf8_err' :: B.ByteString -> Property -t_utf8_err' bs = monadicIO . assert $ case E.decodeUtf8' bs of - Left err -> length (show err) >= 0 - Right t -> T.length t >= 0 +t_utf8_err :: InvalidUtf8 -> DecodeErr -> Property +t_utf8_err bad de = forAll (Blind <$> genDecodeErr de) $ \(Blind onErr) -> ioProperty $ do + let decoded = E.decodeUtf8With onErr (toByteString bad) + len = T.length (E.decodeUtf8With onErr (toByteString bad)) + l <- Exception.try (Exception.evaluate len) + pure $ case l of + Left (err :: Exception.SomeException) -> counterexample (show err) $ + length (show err) >= 0 + Right _ -> counterexample (show (decoded, l)) $ de /= Strict +t_utf8_err' :: B.ByteString -> Bool +t_utf8_err' bs = case E.decodeUtf8' bs of + Left err -> length (show err) >= 0 + Right t -> T.length t >= 0 + genInvalidUTF8 :: Gen B.ByteString genInvalidUTF8 = B.pack <$> oneof [ -- invalid leading byte of a 2-byte sequence @@ -137,21 +149,21 @@ -- short 4-byte sequence , (:) <$> choose (0xF0, 0xF4) <*> upTo 2 contByte -- overlong encoding - , do k <- choose (0,0xFFFF) - let c = chr k + , do k <- choose (0 :: Int, 0xFFFF) case k of - _ | k < 0x80 -> oneof [ let (w,x) = ord2 c in return [w,x] - , let (w,x,y) = ord3 c in return [w,x,y] - , let (w,x,y,z) = ord4 c in return [w,x,y,z] ] - | k < 0x7FF -> oneof [ let (w,x,y) = ord3 c in return [w,x,y] - , let (w,x,y,z) = ord4 c in return [w,x,y,z] ] - | otherwise -> let (w,x,y,z) = ord4 c in return [w,x,y,z] + _ | k < 0x80 -> elements [ord2_ k, ord3_ k, ord4_ k] + | k < 0x7FF -> elements [ord3_ k, ord4_ k] + | otherwise -> return (ord4_ k) ] where contByte = (0x80 +) <$> choose (0, 0x3f) upTo n gen = do k <- choose (0,n) vectorOf k gen + -- Data.Text.Internal.Encoding.Utf8.ord{2,3,4} withous sanity checks + ord2_ n = map fromIntegral [(n `shiftR` 6) + 0xC0, (n .&. 0x3F) + 0x80] + ord3_ n = map fromIntegral [(n `shiftR` 12) + 0xE0, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80] + ord4_ n = map fromIntegral [(n `shiftR` 18) + 0xF0, ((n `shiftR` 12) .&. 0x3F) + 0x80, ((n `shiftR` 6) .&. 0x3F) + 0x80, (n .&. 0x3F) + 0x80] decodeLL :: BL.ByteString -> TL.Text decodeLL = EL.decodeUtf8With E.lenientDecode @@ -187,9 +199,14 @@ t_decode_with_error4' = case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of E.Some x _ _ -> x === "xaaa" +t_decode_with_error5' = ioProperty $ do + ret <- Exception.try $ Exception.evaluate $ E.streamDecodeUtf8 (B.pack [0x81]) + pure $ case ret of + Left (_ :: E.UnicodeException) -> True + Right{} -> False t_infix_concat bs1 text bs2 = - forAll (genDecodeErr Replace) $ \onErr -> + forAll (Blind <$> genDecodeErr Replace) $ \(Blind onErr) -> text `T.isInfixOf` E.decodeUtf8With onErr (B.concat [bs1, E.encodeUtf8 text, bs2]) @@ -214,6 +231,13 @@ testProperty "tl_utf32LE" tl_utf32LE, testProperty "t_utf32BE" t_utf32BE, testProperty "tl_utf32BE" tl_utf32BE, + testGroup "builder" [ + testProperty "t_encodeUtf8Builder" t_encodeUtf8Builder, + testProperty "t_encodeUtf8Builder_nonZeroOffset" t_encodeUtf8Builder_nonZeroOffset, + testProperty "t_encodeUtf8BuilderEscaped" t_encodeUtf8BuilderEscaped, + testProperty "t_encodeUtf8BuilderEscaped_nonZeroOffset" t_encodeUtf8BuilderEscaped_nonZeroOffset, + testProperty "t_encodeUtf8Builder_sanity" t_encodeUtf8Builder_sanity + ], testGroup "errors" [ testProperty "t_utf8_err" t_utf8_err, testProperty "t_utf8_err'" t_utf8_err' @@ -226,6 +250,7 @@ testProperty "t_decode_with_error2'" t_decode_with_error2', testProperty "t_decode_with_error3'" t_decode_with_error3', testProperty "t_decode_with_error4'" t_decode_with_error4', + testProperty "t_decode_with_error5'" t_decode_with_error5', testProperty "t_infix_concat" t_infix_concat ] ] diff --git a/tests/Tests/QuickCheckUtils.hs b/tests/Tests/QuickCheckUtils.hs --- a/tests/Tests/QuickCheckUtils.hs +++ b/tests/Tests/QuickCheckUtils.hs @@ -2,26 +2,20 @@ -- instances, and comparison functions, so we can focus on the actual properties -- in the 'Tests.Properties' module. -- -{-# LANGUAGE CPP, FlexibleInstances, TypeSynonymInstances #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE DeriveFunctor #-} + {-# OPTIONS_GHC -fno-warn-orphans #-} -module Tests.QuickCheckUtils - ( - genUnicode - , unsquare - , smallArbitrary - , BigBounded(..) - , BigInt(..) +module Tests.QuickCheckUtils + ( BigInt(..) , NotEmpty(..) - - , Small(..) - , small + , Sqrt(..) + , SpacyString(..) , Precision(..) , precision - , integralRandomR - , DecodeErr(..) , genDecodeErr @@ -29,24 +23,20 @@ , unpack2 , eq , eqP - - , Encoding(..) + , eqPSqrt , write_read ) where -import Control.Arrow (first, (***)) +import Control.Arrow ((***)) import Control.DeepSeq (NFData (..), deepseq) import Control.Exception (bracket) -import Data.String (IsString, fromString) -import Data.Text.Foreign (I16) +import Data.Char (isSpace) +import Data.Text.Foreign (I8) import Data.Text.Lazy.Builder.RealFloat (FPFormat(..)) import Data.Word (Word8, Word16) -import Debug.Trace (trace) -import System.Random (Random(..), RandomGen) -import Test.QuickCheck hiding (Fixed(..), Small (..), (.&.)) -import Test.QuickCheck.Monadic (assert, monadicIO, run) -import Test.QuickCheck.Unicode (string) +import Test.QuickCheck (Arbitrary(..), arbitraryUnicodeChar, arbitraryBoundedEnum, getUnicodeString, arbitrarySizedIntegral, shrinkIntegral, Property, ioProperty, discard, counterexample, scale, (===), (.&&.), NonEmptyList(..)) +import Test.QuickCheck.Gen (Gen, choose, chooseAny, elements, frequency, listOf, oneof, resize, sized) import Tests.Utils import qualified Data.ByteString as B import qualified Data.ByteString.Lazy as BL @@ -59,17 +49,10 @@ import qualified Data.Text.Lazy as TL import qualified System.IO as IO -genUnicode :: IsString a => Gen a -genUnicode = fromString <$> string - genWord8 :: Gen Word8 genWord8 = chooseAny -instance Random I16 where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary I16 where +instance Arbitrary I8 where arbitrary = arbitrarySizedIntegral shrink = shrinkIntegral @@ -91,21 +74,26 @@ ] shrink xs = BL.fromChunks <$> shrink (BL.toChunks xs) --- For tests that have O(n^2) running times or input sizes, resize +-- | For tests that have O(n^2) running times or input sizes, resize -- their inputs to the square root of the originals. -unsquare :: (Arbitrary a, Show a, Testable b) => (a -> b) -> Property -unsquare = forAll smallArbitrary +newtype Sqrt a = Sqrt { unSqrt :: a } + deriving (Eq, Show) -smallArbitrary :: (Arbitrary a, Show a) => Gen a -smallArbitrary = sized $ \n -> resize (smallish n) arbitrary - where smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs +instance Arbitrary a => Arbitrary (Sqrt a) where + arbitrary = fmap Sqrt $ sized $ \n -> resize (smallish n) arbitrary + where + smallish = round . (sqrt :: Double -> Double) . fromIntegral . abs + shrink = map Sqrt . shrink . unSqrt instance Arbitrary T.Text where - arbitrary = T.pack `fmap` string + arbitrary = do + t <- (T.pack . getUnicodeString) `fmap` scale (* 2) arbitrary + -- Generate chunks that start in the middle of their buffers. + (\i -> T.drop i t) <$> choose (0, T.length t) shrink = map T.pack . shrink . T.unpack instance Arbitrary TL.Text where - arbitrary = (TL.fromChunks . map notEmpty) `fmap` smallArbitrary + arbitrary = (TL.fromChunks . map notEmpty . unSqrt) `fmap` arbitrary shrink = map TL.pack . shrink . TL.unpack newtype BigInt = Big Integer @@ -116,90 +104,21 @@ shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l] where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer -newtype BigBounded a = BigBounded a - deriving (Eq, Show) - -instance (Bounded a, Random a, Arbitrary a) => Arbitrary (BigBounded a) where - arbitrary = BigBounded <$> choose (minBound, maxBound) - newtype NotEmpty a = NotEmpty { notEmpty :: a } - deriving (Eq, Ord) - -instance Show a => Show (NotEmpty a) where - show (NotEmpty a) = show a - -instance Functor NotEmpty where - fmap f (NotEmpty a) = NotEmpty (f a) - -instance Arbitrary a => Arbitrary (NotEmpty [a]) where - arbitrary = sized (\n -> NotEmpty `fmap` (choose (1,n+1) >>= vector)) - shrink = shrinkNotEmpty null + deriving (Eq, Ord, Show) instance Arbitrary (NotEmpty T.Text) where - arbitrary = (fmap T.pack) `fmap` arbitrary - shrink = shrinkNotEmpty T.null + arbitrary = fmap (NotEmpty . T.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . T.pack . getNonEmpty) + . shrink . NonEmpty . T.unpack . notEmpty instance Arbitrary (NotEmpty TL.Text) where - arbitrary = (fmap TL.pack) `fmap` arbitrary - shrink = shrinkNotEmpty TL.null - -instance Arbitrary (NotEmpty B.ByteString) where - arbitrary = (fmap B.pack) `fmap` arbitrary - shrink = shrinkNotEmpty B.null - -shrinkNotEmpty :: Arbitrary a => (a -> Bool) -> NotEmpty a -> [NotEmpty a] -shrinkNotEmpty isNull (NotEmpty xs) = - [ NotEmpty xs' | xs' <- shrink xs, not (isNull xs') ] - -data Small = S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 - | S8 | S9 | S10 | S11 | S12 | S13 | S14 | S15 - | S16 | S17 | S18 | S19 | S20 | S21 | S22 | S23 - | S24 | S25 | S26 | S27 | S28 | S29 | S30 | S31 - deriving (Eq, Ord, Enum, Bounded) - -small :: Integral a => Small -> a -small = fromIntegral . fromEnum - -intf :: (Int -> Int -> Int) -> Small -> Small -> Small -intf f a b = toEnum ((fromEnum a `f` fromEnum b) `mod` 32) - -instance Show Small where - show = show . fromEnum - -instance Read Small where - readsPrec n = map (first toEnum) . readsPrec n - -instance Num Small where - fromInteger = toEnum . fromIntegral - signum _ = 1 - abs = id - (+) = intf (+) - (-) = intf (-) - (*) = intf (*) - -instance Real Small where - toRational = toRational . fromEnum - -instance Integral Small where - toInteger = toInteger . fromEnum - quotRem a b = (toEnum x, toEnum y) - where (x, y) = fromEnum a `quotRem` fromEnum b - -instance Random Small where - randomR = integralRandomR - random = randomR (minBound,maxBound) - -instance Arbitrary Small where - arbitrary = choose (minBound, maxBound) - shrink = shrinkIntegral - -integralRandomR :: (Integral a, RandomGen g) => (a,a) -> g -> (a,g) -integralRandomR (a,b) g = case randomR (fromIntegral a :: Integer, - fromIntegral b :: Integer) g of - (x,h) -> (fromIntegral x, h) + arbitrary = fmap (NotEmpty . TL.pack . getNonEmpty) arbitrary + shrink = fmap (NotEmpty . TL.pack . getNonEmpty) + . shrink . NonEmpty . TL.unpack . notEmpty data DecodeErr = Lenient | Ignore | Strict | Replace - deriving (Show, Eq) + deriving (Show, Eq, Bounded, Enum) genDecodeErr :: DecodeErr -> Gen T.OnDecodeError genDecodeErr Lenient = return T.lenientDecode @@ -207,11 +126,11 @@ genDecodeErr Strict = return T.strictDecode genDecodeErr Replace = (\c _ _ -> c) <$> frequency [ (1, return Nothing) - , (50, Just <$> choose ('\x1', '\xffff')) + , (50, Just <$> arbitraryUnicodeChar) ] instance Arbitrary DecodeErr where - arbitrary = elements [Lenient, Ignore, Strict, Replace] + arbitrary = arbitraryBoundedEnum class Stringy s where packS :: String -> s @@ -246,16 +165,16 @@ unpack2 = unpackS *** unpackS -- Do two functions give the same answer? -eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Bool +eq :: (Eq a, Show a) => (t -> a) -> (t -> a) -> t -> Property eq a b s = a s =^= b s -- What about with the RHS packed? eqP :: (Eq a, Show a, Stringy s) => - (String -> a) -> (s -> a) -> String -> Word8 -> Bool -eqP f g s w = eql "orig" (f s) (g t) && - eql "mini" (f s) (g mini) && - eql "head" (f sa) (g ta) && - eql "tail" (f sb) (g tb) + (String -> a) -> (s -> a) -> String -> Word8 -> Property +eqP f g s w = counterexample "orig" (f s =^= g t) .&&. + counterexample "mini" (f s =^= g mini) .&&. + counterexample "head" (f sa =^= g ta) .&&. + counterexample "tail" (f sb =^= g tb) where t = packS s mini = packSChunkSize 10 s (sa,sb) = splitAt m s @@ -264,12 +183,13 @@ m | l == 0 = n | otherwise = n `mod` l n = fromIntegral w - eql d a b - | a =^= b = True - | otherwise = trace (d ++ ": " ++ show a ++ " /= " ++ show b) False +eqPSqrt :: (Eq a, Show a, Stringy s) => + (String -> a) -> (s -> a) -> Sqrt String -> Word8 -> Property +eqPSqrt f g s = eqP f g (unSqrt s) + instance Arbitrary FPFormat where - arbitrary = elements [Exponent, Fixed, Generic] + arbitrary = arbitraryBoundedEnum newtype Precision a = Precision (Maybe Int) deriving (Eq, Show) @@ -292,28 +212,11 @@ arbitrary = arbitraryPrecision 22 shrink = map Precision . shrink . precision undefined --- Work around lack of Show instance for TextEncoding. -data Encoding = E String IO.TextEncoding - -instance Show Encoding where show (E n _) = "utf" ++ n - -instance Arbitrary Encoding where - arbitrary = oneof . map return $ - [ E "8" IO.utf8, E "8_bom" IO.utf8_bom, E "16" IO.utf16 - , E "16le" IO.utf16le, E "16be" IO.utf16be, E "32" IO.utf32 - , E "32le" IO.utf32le, E "32be" IO.utf32be - ] - -windowsNewlineMode :: IO.NewlineMode -windowsNewlineMode = IO.NewlineMode - { IO.inputNL = IO.CRLF, IO.outputNL = IO.CRLF - } +instance Arbitrary IO.Newline where + arbitrary = oneof [return IO.LF, return IO.CRLF] instance Arbitrary IO.NewlineMode where - arbitrary = oneof . map return $ - [ IO.noNewlineTranslation, IO.universalNewlineMode, IO.nativeNewlineMode - , windowsNewlineMode - ] + arbitrary = IO.NewlineMode <$> arbitrary <*> arbitrary instance Arbitrary IO.BufferMode where arbitrary = oneof [ return IO.NoBuffering, @@ -331,31 +234,46 @@ -- * The lines themselves, scrubbed to contain neither CR nor LF. (By -- working with a list of lines, we ensure that the data will -- sometimes contain line endings.) --- * Encoding. -- * Newline translation mode. -- * Buffering. -write_read :: (NFData a, Eq a) +write_read :: (NFData a, Eq a, Show a) => ([b] -> a) -> ((Char -> Bool) -> a -> b) -> (IO.Handle -> a -> IO ()) -> (IO.Handle -> IO a) - -> Encoding -> IO.NewlineMode -> IO.BufferMode -> [a] -> Property -write_read unline filt writer reader (E _ _) nl buf ts = - monadicIO $ assert . (==t) =<< run act - where t = unline . map (filt (not . (`elem` "\r\n"))) $ ts - act = withTempFile $ \path h -> do - -- hSetEncoding h enc - IO.hSetNewlineMode h nl - IO.hSetBuffering h buf - () <- writer h t - IO.hClose h - bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do - -- hSetEncoding h' enc - IO.hSetNewlineMode h' nl - IO.hSetBuffering h' buf - r <- reader h' - r `deepseq` return r +write_read _ _ _ _ (IO.NewlineMode IO.LF IO.CRLF) _ _ = discard +write_read unline filt writer reader nl buf ts = ioProperty $ + (===t) <$> act + where + t = unline . map (filt (not . (`elem` "\r\n"))) $ ts + + act = withTempFile $ \path h -> do + IO.hSetEncoding h IO.utf8 + IO.hSetNewlineMode h nl + IO.hSetBuffering h buf + () <- writer h t + IO.hClose h + bracket (IO.openFile path IO.ReadMode) IO.hClose $ \h' -> do + IO.hSetEncoding h' IO.utf8 + IO.hSetNewlineMode h' nl + IO.hSetBuffering h' buf + r <- reader h' + r `deepseq` return r + +-- Generate various Unicode space characters with high probability +arbitrarySpacyChar :: Gen Char +arbitrarySpacyChar = oneof + [ arbitraryUnicodeChar + , elements $ filter isSpace [minBound..maxBound] + ] + +newtype SpacyString = SpacyString { getSpacyString :: String } + deriving (Eq, Ord, Show, Read) + +instance Arbitrary SpacyString where + arbitrary = SpacyString `fmap` listOf arbitrarySpacyChar + shrink (SpacyString xs) = SpacyString `fmap` shrink xs diff --git a/tests/Tests/Regressions.hs b/tests/Tests/Regressions.hs --- a/tests/Tests/Regressions.hs +++ b/tests/Tests/Regressions.hs @@ -1,5 +1,6 @@ -- | Regression tests for specific bugs. -- +{-# LANGUAGE BangPatterns #-} {-# LANGUAGE MagicHash #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -80,10 +81,10 @@ let f a _ = (a, '\65536') count = 5 val = T.mapAccumL f (0::Int) (T.replicate count "a") - assertEqual "mapAccumL should correctly fill buffers for two-word results" + assertEqual "mapAccumL should correctly fill buffers for four-byte results" (0, T.replicate count "\65536") val - assertEqual "mapAccumL should correctly size buffers for two-word results" - (count * 2) (T.lengthWord16 (snd val)) + assertEqual "mapAccumL should correctly size buffers for four-byte results" + (count * 4) (T.lengthWord8 (snd val)) -- See GitHub #197 t197 :: IO () @@ -127,15 +128,15 @@ t301 :: IO () t301 = do assertEqual "The length of the array remains the same despite slicing" - (I# (sizeofByteArray# (TA.aBA originalArr))) - (I# (sizeofByteArray# (TA.aBA newArr))) + (I# (sizeofByteArray# originalArr)) + (I# (sizeofByteArray# newArr)) assertEqual "The new array still contains the original value" - (T.Text newArr originalOff originalLen) + (T.Text (TA.ByteArray newArr) originalOff originalLen) original where - original@(T.Text originalArr originalOff originalLen) = T.pack "1234567890" - T.Text newArr _off _len = T.take 1 $ T.drop 1 original + !original@(T.Text (TA.ByteArray originalArr) originalOff originalLen) = T.pack "1234567890" + !(T.Text (TA.ByteArray newArr) _off _len) = T.take 1 $ T.drop 1 original t330 :: IO () t330 = do diff --git a/tests/Tests/Utils.hs b/tests/Tests/Utils.hs --- a/tests/Tests/Utils.hs +++ b/tests/Tests/Utils.hs @@ -10,24 +10,23 @@ import Control.Exception (SomeException, bracket, bracket_, evaluate, try) import Control.Monad (when) -import Debug.Trace (trace) import GHC.IO.Handle.Internals (withHandle) import System.Directory (removeFile) import System.IO (Handle, hClose, hFlush, hIsOpen, hIsWritable, openTempFile) -import System.IO.Unsafe (unsafePerformIO) +import Test.QuickCheck (Property, ioProperty, property, (===), counterexample) -- Ensure that two potentially bottom values (in the sense of crashing -- for some inputs, not looping infinitely) either both crash, or both -- give comparable results for some input. -(=^=) :: (Eq a, Show a) => a -> a -> Bool -i =^= j = unsafePerformIO $ do +(=^=) :: (Eq a, Show a) => a -> a -> Property +i =^= j = ioProperty $ do x <- try (evaluate i) y <- try (evaluate j) - case (x,y) of + return $ case (x, y) of (Left (_ :: SomeException), Left (_ :: SomeException)) - -> return True - (Right a, Right b) -> return (a == b) - e -> trace ("*** Divergence: " ++ show e) return False + -> property True + (Right a, Right b) -> a === b + e -> counterexample ("Divergence: " ++ show e) $ property False infix 4 =^= {-# NOINLINE (=^=) #-} diff --git a/text.cabal b/text.cabal --- a/text.cabal +++ b/text.cabal @@ -1,6 +1,6 @@ -cabal-version: >= 1.10 +cabal-version: 2.2 name: text -version: 1.2.5.0 +version: 2.0 homepage: https://github.com/haskell/text bug-reports: https://github.com/haskell/text/issues @@ -8,7 +8,7 @@ description: . An efficient packed, immutable Unicode text type (both strict and - lazy), with a powerful loop fusion optimization framework. + lazy). . The 'Text' type represents Unicode character strings, in a time and space-efficient manner. This package provides text processing @@ -37,35 +37,33 @@ the [text-icu package](https://hackage.haskell.org/package/text-icu) based on the well-respected and liberally licensed [ICU library](http://site.icu-project.org/). - . - == Internal Representation: UTF-16 vs. UTF-8 - . - Currently the @text@ library uses UTF-16 as its internal representation - which is [neither a fixed-width nor always the most dense representation](http://utf8everywhere.org/) - for Unicode text. We're currently investigating the feasibility - of [changing Text's internal representation to UTF-8](https://github.com/text-utf8) - and if you need such a 'Text' type right now you might be interested in using the spin-off - packages and - . - -license: BSD2 +license: BSD-2-Clause license-file: LICENSE author: Bryan O'Sullivan maintainer: Haskell Text Team , Core Libraries Committee -copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper +copyright: 2009-2011 Bryan O'Sullivan, 2008-2009 Tom Harper, 2021 Andrew Lelechenko category: Data, Text build-type: Simple -tested-with: GHC==9.0.1, - GHC==8.10.4, GHC==8.8.4, GHC==8.6.5, GHC==8.4.4, - GHC==8.2.2, GHC==8.0.2 +tested-with: + GHC == 8.0.2 + GHC == 8.2.2 + GHC == 8.4.4 + GHC == 8.6.5 + GHC == 8.8.4 + GHC == 8.10.7 + GHC == 9.0.1 + GHC == 9.2.1 + extra-source-files: -- scripts/CaseFolding.txt -- scripts/SpecialCasing.txt README.markdown changelog.md - include/*.h scripts/*.hs + simdutf/LICENSE-APACHE + simdutf/LICENSE-MIT + simdutf/simdutf.h tests/literal-rule-test.sh tests/LiteralRuleTest.hs @@ -74,11 +72,35 @@ default: False manual: True +flag simdutf + description: use simdutf library + default: True + manual: True + library - c-sources: cbits/cbits.c - include-dirs: include + c-sources: cbits/is_ascii.c + cbits/measure_off.c + cbits/reverse.c + cbits/utils.c hs-source-dirs: src + if flag(simdutf) + include-dirs: simdutf + cxx-sources: simdutf/simdutf.cpp + cbits/validate_utf8.cpp + cxx-options: -std=c++17 + cpp-options: -DSIMDUTF + if os(windows) + if arch(x86_64) + extra-libraries: stdc++-6 gcc_s_seh-1 + else + extra-libraries: stdc++-6 gcc_s_dw2-1 + else + if os(darwin) + extra-libraries: c++ + else + extra-libraries: stdc++ + exposed-modules: Data.Text Data.Text.Array @@ -98,7 +120,6 @@ Data.Text.Internal.Encoding.Utf16 Data.Text.Internal.Encoding.Utf32 Data.Text.Internal.Encoding.Utf8 - Data.Text.Internal.Functions Data.Text.Internal.Fusion Data.Text.Internal.Fusion.CaseMapping Data.Text.Internal.Fusion.Common @@ -114,7 +135,6 @@ Data.Text.Internal.Search Data.Text.Internal.Unsafe Data.Text.Internal.Unsafe.Char - Data.Text.Internal.Unsafe.Shift Data.Text.Lazy Data.Text.Lazy.Builder Data.Text.Lazy.Builder.Int @@ -143,6 +163,10 @@ ghc-options: -fno-ignore-asserts cpp-options: -DASSERTS + -- https://gitlab.haskell.org/ghc/ghc/-/issues/19900 + if os(windows) + extra-libraries: gcc_s + default-language: Haskell2010 default-extensions: NondecreasingIndentation @@ -178,8 +202,6 @@ hs-source-dirs: tests main-is: Tests.hs other-modules: - Tests.Inspection.Lazy - Tests.Inspection.Strict Tests.Lift Tests.Properties Tests.Properties.Basics @@ -197,20 +219,23 @@ Tests.Utils build-depends: - QuickCheck >= 2.14.1 && < 2.15, + QuickCheck >= 2.12.6 && < 2.15, base <5, bytestring, deepseq, directory, - quickcheck-unicode >= 1.0.1.0, - random, + ghc-prim, tasty, tasty-hunit, - tasty-inspection-testing, tasty-quickcheck, template-haskell, text + -- Starting from 9.2 ghc library depends on parsec, + -- which causes circular dependency. + if impl(ghc < 9.2) + build-depends: tasty-inspection-testing + default-language: Haskell2010 default-extensions: NondecreasingIndentation @@ -224,19 +249,15 @@ ghc-options: "-with-rtsopts=-A32m" build-depends: base, - binary, bytestring >= 0.10.4, - bytestring-lexing >= 0.5.0, containers, deepseq, + directory, filepath, - stringsearch, tasty-bench >= 0.2, text, - transformers, - vector + transformers - c-sources: benchmarks/cbits-bench/time_iconv.c hs-source-dirs: benchmarks/haskell main-is: Benchmarks.hs other-modules: @@ -247,7 +268,6 @@ Benchmarks.Equality Benchmarks.FileRead Benchmarks.FoldLines - Benchmarks.Mul Benchmarks.Multilang Benchmarks.Programs.BigTable Benchmarks.Programs.Cut