diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,16 @@ # Changelog +## 0.3.0 (July 2024) + +- Improve performance. +- Added opional support for `ByteString` API. + Use the package flag `has-bytestring` to enable it. +- Added opional support for `Text` API. + Use the package flag `has-text` to enable it. +- Add `unicodeVersion` to `Unicode.Char.General.Names`. +- Fix the inlining of `Addr#` literals and reduce their size. This results in + a sensible decrease of the executable size. + ## 0.2.0 (September 2022) - Update to [Unicode 15.0.0](https://www.unicode.org/versions/Unicode15.0.0/). diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ character names and aliases from the [Unicode character database](https://www.unicode.org/ucd/). +There are 3 APIs: +- `String` API: enabled by default. +- `ByteString` API: enabled via the package flag `has-bytestring`. +- `Text` API: enabled via the package flag `has-text`. + The Haskell data structures are generated programmatically from the Unicode character database (UCD) files. The latest Unicode version supported by this library is @@ -13,6 +18,17 @@ [Haddock documentation](https://hackage.haskell.org/package/unicode-data-names) for reference documentation. +## Comparing with ICU + +We can compare the implementation against ICU. This requires working with the +source repository, as we need the _internal_ package `icu`. + +__Warning:__ An ICU version with the _exact same Unicode version_ is required. + +```bash +cabal run -O2 --flag dev-has-icu unicode-data-names:tests -- -m ICU +``` + ## Comparing with Python In order to check Unicode implementation in Haskell, we compare the results obtained @@ -21,7 +37,7 @@ __Warning:__ A Python version with the _exact same Unicode version_ is required. ```bash -cabal run -f "export-all-chars" -v0 export-all-chars > ./test/all_chars.csv +cabal run -O2 -f "export-all-chars" -v0 export-all-chars > ./test/all_chars.csv python3 ./test/check.py -v ./test/all_chars.csv ``` diff --git a/bench/Main.hs b/bench/Main.hs --- a/bench/Main.hs +++ b/bench/Main.hs @@ -1,60 +1,268 @@ -{-# LANGUAGE ExistentialQuantification #-} +{-# LANGUAGE CPP, ExistentialQuantification #-} +{-# OPTIONS_GHC -Wno-orphans #-} -import Control.DeepSeq (NFData, deepseq) +import Control.DeepSeq (NFData, deepseq, force) +import Control.Exception (evaluate) import Data.Ix (Ix(..)) -import Test.Tasty.Bench (Benchmark, bgroup, bcompare, bench, nf, defaultMain) +import Data.Proxy (Proxy(..)) +import GHC.Exts (Char(..), indexCharOffAddr#) +import Test.Tasty (askOption, includingOptions) +import Test.Tasty.Bench (Benchmark, bgroup, bcompare, bench, benchIngredients, nf, env) +import Test.Tasty.Options + ( IsOption(defaultValue, optionHelp, optionName, parseValue) + , OptionDescription(..) ) +import Test.Tasty.Runners (TestTree, defaultMainWithIngredients) -import qualified Unicode.Char.General.Names as Names +import qualified Unicode.Char as UChar +import qualified Unicode.Char.General.Names as String import qualified Unicode.Internal.Char.UnicodeData.DerivedName as DerivedName import qualified Unicode.Internal.Char.UnicodeData.NameAliases as NameAliases +#ifdef HAS_BYTESTRING +import qualified Unicode.Char.General.Names.ByteString as ByteString +import Data.ByteString () +#endif +#ifdef HAS_TEXT +import qualified Unicode.Char.General.Names.Text as Text +import Data.Text () +#endif +#ifdef HAS_ICU +import qualified ICU.Names as ICUString +#ifdef HAS_TEXT +import qualified ICU.Names.Text as ICUText +#endif +#endif +-------------------------------------------------------------------------------- +-- CLI options +-------------------------------------------------------------------------------- + +data CharRange = CharRange !Char !Char + +instance IsOption CharRange where + defaultValue = CharRange minBound maxBound + parseValue = \case + "ascii" -> Just (CharRange minBound '\x7f') + "bmp" -> Just (CharRange minBound '\xffff') + "planes0To3" -> Just (CharRange minBound '\x3FFFF') + -- [TODO] handle errors + s -> + let (l, u) = drop 1 <$> break (== '-') s + in Just (CharRange (UChar.chr (read l)) (UChar.chr (read u))) + optionName = pure "chars" + optionHelp = pure "Range of chars to test" + +data Filter + = NoFilter -- ^ No condition + | WithName -- ^ Char has a name + | WithNameAlias -- ^ Char has a name alias + | WithNameOrAlias -- ^ Char has a name or an alias + +instance IsOption Filter where + defaultValue = WithNameOrAlias + parseValue = \case + "name" -> Just WithName + "alias" -> Just WithNameAlias + "nameOrAlias" -> Just WithNameOrAlias + "none" -> Just NoFilter + _ -> Nothing + optionName = pure "chars-filter" + optionHelp = pure "Filter the chars to test" + +-------------------------------------------------------------------------------- +-- Benchmark utils +-------------------------------------------------------------------------------- + +-- Orphan instance +instance NFData String.NameAliasType + -- | A unit benchmark data Bench = forall a. (NFData a) => Bench - { _title :: !String -- ^ Name - , _func :: Char -> a -- ^ Function to benchmark - } + { -- | Name + _title :: !String + -- | Function to benchmark + , _func :: Char -> a } +hasName :: Char -> Bool +hasName (C# c#) = case DerivedName.name c# of + (# _, 0# #) -> False + _ -> True + +hasNameAlias :: Char -> Bool +hasNameAlias (C# c#) = + let addr# = NameAliases.nameAliases c# + in case indexCharOffAddr# addr# 0# of + '\xff'# -> False + _ -> True + +-------------------------------------------------------------------------------- +-- Benchmark +-------------------------------------------------------------------------------- + main :: IO () -main = defaultMain +main = do + let customOpts = [ Option (Proxy :: Proxy CharRange) + , Option (Proxy :: Proxy Filter)] + ingredients = includingOptions customOpts : benchIngredients + defaultMainWithIngredients ingredients + (askOption (askOption . benchmarks)) + +benchmarks :: CharRange -> Filter -> TestTree +benchmarks charRange charFilter = bgroup "All" [ bgroup "Unicode.Char.General.Names" - -- Character classification - [ bgroup' "name" - [ Bench "CString" DerivedName.name - , Bench "String" Names.name + [ bgroup "name" + [ bgroup' "name" "String" + [ Bench "unicode-data" String.name +#ifdef HAS_ICU + , Bench "icu" ICUString.name +#endif + ] +#ifdef HAS_BYTESTRING + , bgroup' "name" "ByteString" + [ Bench "unicode-data" ByteString.name +-- #ifdef HAS_ICU +-- , Bench "icu" ICUByteString.name +-- #endif + ] +#endif +#ifdef HAS_TEXT + , bgroup' "name" "Text" + [ Bench "unicode-data" Text.name +#ifdef HAS_ICU + , Bench "icu" ICUText.name +#endif + ] +#endif ] - , bgroup' "correctedName" - [ Bench "String" Names.correctedName + , bgroup "correctedName" + [ bgroup' "correctedName" "String" + [ Bench "unicode-data" String.correctedName +#ifdef HAS_ICU + , Bench "icu" ICUString.correctedName +#endif + ] +#ifdef HAS_BYTESTRING + , bgroup' "name" "ByteString" + [ Bench "unicode-data" ByteString.correctedName +-- #ifdef HAS_ICU +-- , Bench "icu" ICUByteString.correctedName +-- #endif + ] +#endif +#ifdef HAS_TEXT + , bgroup' "correctedName" "Text" + [ Bench "unicode-data" Text.correctedName +#ifdef HAS_ICU + , Bench "icu" ICUText.correctedName +#endif + ] +#endif ] - , bgroup' "nameOrAlias" - [ Bench "String" Names.name + , bgroup "nameOrAlias" + [ bgroup' "nameOrAlias" "String" + [ Bench "unicode-data" String.nameOrAlias + ] +#ifdef HAS_BYTESTRING + , bgroup' "nameOrAlias" "ByteString" + [ Bench "unicode-data" ByteString.nameOrAlias + ] +#endif +#ifdef HAS_TEXT + , bgroup' "nameOrAlias" "Text" + [ Bench "unicode-data" Text.nameOrAlias + ] +#endif ] - , bgroup' "nameAliasesByType" - [ Bench "CString" - (\c -> (`NameAliases.nameAliasesByType` c) <$> [minBound..maxBound]) - , Bench "String" - (\c -> (`Names.nameAliasesByType` c) <$> [minBound..maxBound]) + , bgroup "nameAliasesByType" + [ bgroup' "nameAliasesByType" "String" + [ Bench "unicode-data" + (\c -> fold_ (`String.nameAliasesByType` c)) + ] +#ifdef HAS_BYTESTRING + , bgroup' "nameAliasesByType" "ByteString" + [ Bench "unicode-data" + (\c -> fold_ (`String.nameAliasesByType` c)) + ] +#endif +#ifdef HAS_TEXT + , bgroup' "nameAliasesByType" "Text" + [ Bench "unicode-data" + (\c -> fold_ (`String.nameAliasesByType` c)) + ] +#endif ] - , bgroup' "nameAliasesWithTypes" - [ Bench "CString" (show . NameAliases.nameAliasesWithTypes) - , Bench "String" (show . Names.nameAliasesWithTypes) + , bgroup "nameAliasesWithTypes" + [ bgroup' "nameAliasesWithTypes" "String" + [ Bench "unicode-data" String.nameAliasesWithTypes + ] +#ifdef HAS_BYTESTRING + , bgroup' "nameAliasesWithTypes" "ByteString" + [ Bench "unicode-data" ByteString.nameAliasesWithTypes + ] +#endif +#ifdef HAS_TEXT + , bgroup' "nameAliasesWithTypes" "Text" + [ Bench "unicode-data" Text.nameAliasesWithTypes + ] +#endif ] + , bgroup "nameAliases" + [ bgroup' "nameAliases" "String" + [ Bench "unicode-data" String.nameAliases + ] +#ifdef HAS_BYTESTRING + , bgroup' "nameAliases" "ByteString" + [ Bench "unicode-data" ByteString.nameAliases + ] +#endif +#ifdef HAS_TEXT + , bgroup' "nameAliases" "Text" + [ Bench "unicode-data" Text.nameAliases + ] +#endif + ] ] ] where - bgroup' groupTitle bs = bgroup groupTitle - [ benchNF' groupTitle title f + bgroup' superGroupTitle groupTitle bs = bgroup groupTitle + [ benchNF' superGroupTitle groupTitle title f | Bench title f <- bs ] -- [NOTE] Works if groupTitle uniquely identifies the benchmark group. - benchNF' groupTitle title = case title of - "CString" -> benchNF title - _ -> - bcompare ("$NF == \"CString\" && $(NF-1) == \"" ++ groupTitle ++ "\"") - . benchNF title + benchNF' superGroupTitle groupTitle title = case title of + "unicode-data" -> benchCharsNF title + _ -> + bcompare ( mconcat + [ "$NF == \"unicode-data\" && $(NF-1) == \"" + , groupTitle + , "\" && $(NF-2) == \"" + , superGroupTitle + , "\"" ] ) + . benchCharsNF title - benchNF :: forall a. (NFData a) => String -> (Char -> a) -> Benchmark - benchNF t f = bench t $ nf (fold_ f) (minBound, maxBound) + {-# INLINE benchCharsNF #-} + benchCharsNF + :: forall a. (NFData a) + => String + -> (Char -> a) + -> Benchmark + benchCharsNF t f = + -- Avoid side-effects with garbage collection (see tasty-bench doc) + env + (evaluate (force chars')) -- initialize + (bench t . nf (foldr (deepseq . f) ())) -- benchmark + where + CharRange l u = charRange + extraFilter = case charFilter of + NoFilter -> const True + WithName -> hasName + WithNameAlias -> hasNameAlias + WithNameOrAlias -> \c -> hasName c || hasNameAlias c + chars = filter isValid [l..u] + -- Ensure to have sufficiently chars + n = 0x10FFFF `div` length chars + chars' = mconcat (replicate n chars) + isValid c = UChar.generalCategory c < UChar.Surrogate && extraFilter c - fold_ :: forall a. (NFData a) => (Char -> a) -> (Char, Char) -> () - fold_ f = foldr (deepseq . f) () . range + fold_ :: forall a. (NFData a) => (String.NameAliasType -> a) -> () + fold_ f = foldr (deepseq . f) () (range (minBound, maxBound)) diff --git a/lib/Unicode/Char/General/Names.hs b/lib/Unicode/Char/General/Names.hs --- a/lib/Unicode/Char/General/Names.hs +++ b/lib/Unicode/Char/General/Names.hs @@ -5,14 +5,21 @@ -- Maintainer : streamly@composewell.com -- Stability : experimental -- --- Unicode character names and name aliases. +-- Unicode character names and name aliases with 'String' API. -- See Unicode standard 15.0.0, section 4.8. -- +-- There are also two optional APIs: +-- +-- * @ByteString@ API, which requires using the package flag @has-bytestring@. +-- * @Text@ API, which requires using the package flag @has-text@. +-- -- @since 0.1.0 module Unicode.Char.General.Names - ( -- * Name - name + ( -- Unicode version + unicodeVersion + -- * Name + , name , nameOrAlias , correctedName -- * Name Aliases @@ -23,37 +30,96 @@ ) where import Control.Applicative ((<|>)) -import Data.Maybe (listToMaybe) -import Foreign.C.String (CString, peekCAString) -import System.IO.Unsafe (unsafePerformIO) +import GHC.Exts + ( Addr#, Char(..), Char#, Int# + , indexCharOffAddr#, plusAddr#, (+#), (-#), (<#), isTrue#, quotRemInt# + , dataToTag#, ord# ) +import Unicode.Internal.Bits.Names (unpackNBytes#) import qualified Unicode.Internal.Char.UnicodeData.DerivedName as DerivedName import qualified Unicode.Internal.Char.UnicodeData.NameAliases as NameAliases +import Unicode.Internal.Char.Names.Version (unicodeVersion) -- | Name of a character, if defined. -- -- @since 0.1.0 -{-# INLINE name #-} name :: Char -> Maybe String -name = fmap unpack . DerivedName.name +name (C# c#) = case DerivedName.name c# of + (# name#, len# #) -> case len# of + DerivedName.NoName -> Nothing + DerivedName.CjkCompatibilityIdeograph -> Just n + where + !hex = showHex c# + !n = 'C':'J':'K':' ':'C':'O':'M':'P':'A':'T':'I':'B':'I':'L':'I':'T':'Y':' ':'I':'D':'E':'O':'G':'R':'A':'P':'H':'-':hex + DerivedName.CjkUnifiedIdeograph -> Just n + where + !hex = showHex c# + !n = 'C':'J':'K':' ':'U':'N':'I':'F':'I':'E':'D':' ':'I':'D':'E':'O':'G':'R':'A':'P':'H':'-':hex + DerivedName.TangutIdeograph -> Just n + where + !hex = showHex c# + !n = 'T':'A':'N':'G':'U':'T':' ':'I':'D':'E':'O':'G':'R':'A':'P':'H':'-':hex + _ + | isTrue# (len# <# DerivedName.HangulSyllable) -> let !n = unpack name# [] len# in Just n + | otherwise -> + let !rest = unpack name# [] (len# -# DerivedName.HangulSyllable) + !n = 'H':'A':'N':'G':'U':'L':' ':'S':'Y':'L':'L':'A':'B':'L':'E':' ':rest + in Just n +{-# INLINE unpack #-} +unpack :: Addr# -> String -> Int# -> String +unpack !addr# !acc = \case + 0# -> acc + !i -> unpack addr# acc' i' + where + !i' = i -# 1# + !c'# = indexCharOffAddr# addr# i' + !c' = C# c'# + !acc' = c' : acc + +-- [NOTE] We assume c# >= '\x1000' to avoid to check for padding +showHex :: Char# -> String +showHex !c# = showIt [] (quotRemInt# (ord# c#) 16#) + where + showIt !acc (# q, r #) = case q of + 0# -> acc' + _ -> showIt acc' (quotRemInt# q 16#) + where + !c = C# (indexCharOffAddr# "0123456789ABCDEF"# r) + !acc' = c : acc + -- | Returns /corrected/ name of a character (see 'NameAliases.Correction'), -- if defined, otherwise returns its original 'name' if defined. -- -- @since 0.1.0 {-# INLINE correctedName #-} correctedName :: Char -> Maybe String -correctedName c = - listToMaybe (nameAliasesByType NameAliases.Correction c) <|> name c +correctedName c@(C# c#) = corrected <|> name c + where + -- Assumption: fromEnum NameAliases.Correction == 0 + !corrected = case indexCharOffAddr# addr# 0# of + '\xff'# -> Nothing -- no aliases + '\x00'# -> Nothing -- no correction + i# -> + let !n = unpackNBytes'# (addr# `plusAddr#` ord# i#) + in Just n + !addr# = NameAliases.nameAliases c# -- | Returns a character’s 'name' if defined, -- otherwise returns its first name alias if defined. -- -- @since 0.1.0 nameOrAlias :: Char -> Maybe String -nameOrAlias c = name c <|> case NameAliases.nameAliasesWithTypes c of - (_, n:_):_ -> Just (unpack n) - _ -> Nothing +nameOrAlias c@(C# c#) = name c <|> case indexCharOffAddr# addr# 0# of + '\xff'# -> Nothing -- no aliases + '\x00'# -> let !n = go 1# in Just n + _ -> let !n = go 0# in Just n + where + !addr# = NameAliases.nameAliases c# + go t# = case ord# (indexCharOffAddr# (addr# `plusAddr#` t#) 0#) of + -- No bound check for t#: there is at least one alias + 0# -> go (t# +# 1#) + i# -> unpackNBytes'# (addr# `plusAddr#` i#) -- | All name aliases of a character, if defined. -- The names are listed in the original order of the UCD. @@ -63,14 +129,34 @@ -- @since 0.1.0 {-# INLINE nameAliases #-} nameAliases :: Char -> [String] -nameAliases = fmap unpack . NameAliases.nameAliases +nameAliases (C# c#) = case indexCharOffAddr# addr0# 0# of + '\xff'# -> [] -- no aliases + _ -> go (addr0# `plusAddr#` (NameAliases.MaxNameAliasType +# 1#)) + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> case ord# (indexCharOffAddr# (addr# `plusAddr#` 1#) 0#) of + -- End of list + 0# -> [] + -- skip empty entry + l# -> + let !s = unpackNBytes# (addr# `plusAddr#` 2#) l# + in s : go (addr# `plusAddr#` (l# +# 2#)) + l# -> + let !s = unpackNBytes# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) + where + addr0# = NameAliases.nameAliases c# -- | Name aliases of a character for a specific name alias type. -- -- @since 0.1.0 {-# INLINE nameAliasesByType #-} nameAliasesByType :: NameAliases.NameAliasType -> Char -> [String] -nameAliasesByType t = fmap unpack . NameAliases.nameAliasesByType t +nameAliasesByType t (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + _ -> nameAliasesByType# addr# t + where + addr# = NameAliases.nameAliases c# -- | Detailed character names aliases. -- The names are listed in the original order of the UCD. @@ -80,11 +166,35 @@ -- @since 0.1.0 {-# INLINE nameAliasesWithTypes #-} nameAliasesWithTypes :: Char -> [(NameAliases.NameAliasType, [String])] -nameAliasesWithTypes - = fmap (fmap (fmap unpack)) - . NameAliases.nameAliasesWithTypes +nameAliasesWithTypes (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + '\x00'# -> foldr mk mempty [succ minBound..maxBound] + _ -> foldr mk mempty [minBound..maxBound] + where + addr# = NameAliases.nameAliases c# + mk t acc = case nameAliasesByType# addr# t of + [] -> acc + !as -> (t, as) : acc --- Note: names are ASCII. See Unicode Standard 15.0.0, section 4.8. -{-# INLINE unpack #-} -unpack :: CString -> String -unpack = unsafePerformIO . peekCAString +{-# INLINE unpackNBytes'# #-} +unpackNBytes'# :: Addr# -> String +unpackNBytes'# addr# = unpackNBytes# + (addr# `plusAddr#` 1#) + (ord# (indexCharOffAddr# addr# 0#)) + +{-# INLINE nameAliasesByType# #-} +nameAliasesByType# :: Addr# -> NameAliases.NameAliasType -> [String] +nameAliasesByType# addr# t = case indexCharOffAddr# (addr# `plusAddr#` t#) 0# of + '\0'# -> [] -- no aliases for this type + i# -> unpackCStrings# (addr# `plusAddr#` ord# i#) + where t# = dataToTag# t + +{-# INLINE unpackCStrings# #-} +unpackCStrings# :: Addr# -> [String] +unpackCStrings# = go + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> [] + l# -> + let !s = unpackNBytes# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) diff --git a/lib/Unicode/Char/General/Names/ByteString.hs b/lib/Unicode/Char/General/Names/ByteString.hs new file mode 100644 --- /dev/null +++ b/lib/Unicode/Char/General/Names/ByteString.hs @@ -0,0 +1,232 @@ +-- | +-- Module : Unicode.Char.General.Names.Text +-- Copyright : (c) 2022 Composewell Technologies and Contributors +-- License : Apache-2.0 +-- Maintainer : streamly@composewell.com +-- Stability : experimental +-- +-- Unicode character names and name aliases with 'Data.ByteString' API. +-- See Unicode standard 15.0.0, section 4.8. +-- +-- This API is /optional/ and requires using the package flag @has-bytestring@. +-- +-- @since 0.3.0 + +module Unicode.Char.General.Names.ByteString + ( -- * Name + name + , nameOrAlias + , correctedName + -- * Name Aliases + , NameAliases.NameAliasType(..) + , nameAliases + , nameAliasesByType + , nameAliasesWithTypes + ) where + +import Control.Applicative ((<|>)) +import qualified Data.ByteString as BS +import qualified Data.ByteString.Internal as BS +import GHC.Exts + ( Addr#, indexCharOffAddr#, indexWord8OffAddr#, plusAddr# + , Char(..), ord# + , Int#, Int(..), (+#), (-#), (<#), isTrue#, andI#, uncheckedIShiftRL# + , dataToTag# + , newPinnedByteArray#, byteArrayContents# + , copyAddrToByteArray#, writeWord8Array# + , unsafeCoerce# ) +import GHC.IO (IO(..)) +import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents (PlainPtr)) +import System.IO.Unsafe (unsafeDupablePerformIO) + +import qualified Unicode.Internal.Char.UnicodeData.DerivedName as DerivedName +import qualified Unicode.Internal.Char.UnicodeData.NameAliases as NameAliases + +-------------------------------------------------------------------------------- +-- Name +-------------------------------------------------------------------------------- + +-- | Name of a character, if defined. +-- +-- @since 0.3.0 +name :: Char -> Maybe BS.ByteString +name (C# c#) = case DerivedName.name c# of + (# name#, len# #) -> case len# of + DerivedName.NoName -> Nothing + DerivedName.CjkCompatibilityIdeograph -> Just n + where + !n = mkNameFromTemplate# "CJK COMPATIBILITY IDEOGRAPH-"# 28# (ord# c#) + DerivedName.CjkUnifiedIdeograph -> Just n + where + !n = mkNameFromTemplate# "CJK UNIFIED IDEOGRAPH-"# 22# (ord# c#) + DerivedName.TangutIdeograph -> Just n + where + !n = mkNameFromTemplate# "TANGUT IDEOGRAPH-"# 17# (ord# c#) + _ + | isTrue# (len# <# DerivedName.HangulSyllable) -> + let !n = unpackAddr# name# len# + in Just n + | otherwise -> + let !n = unsafePack2LenLiteral# + 16# + "HANGUL SYLLABLE "# + (len# -# DerivedName.HangulSyllable) + name# + in Just n + +{-# INLINE unpackAddr# #-} +unpackAddr# :: Addr# -> Int# -> BS.ByteString +unpackAddr# addr# len# = BS.unsafePackLenLiteral (I# len#) addr# + +-- [NOTE] bytestring does not expose enough primitives for working with raw +-- Addr# literals. Therefore we go low level, which performs better than FFI. +-- Inspiration: +-- • Data.ByteString.Internal.unsafeCreate +-- • GHC.ForeignPtr.mallocPlainForeignPtrBytes + +-- | Pack 2 'Addr#' literal into a 'BS.ByteString'. +{-# INLINE unsafePack2LenLiteral# #-} +unsafePack2LenLiteral# :: Int# -> Addr# -> Int# -> Addr# -> BS.ByteString +unsafePack2LenLiteral# l1# addr1# l2# addr2# = unsafeDupablePerformIO ( + IO (\s0 -> let l# = l1# +# l2# in case newPinnedByteArray# l# s0 of + (# s1, marr# #) -> case copyAddrToByteArray# addr1# marr# 0# l1# s1 of + s2 -> case copyAddrToByteArray# addr2# marr# l1# l2# s2 of + s3 -> + (# s3 + -- Note: for compatibility reason we cannot use + -- mutableByteArrayContents# here. + , BS.BS (ForeignPtr (byteArrayContents# (unsafeCoerce# marr#)) + (PlainPtr marr#)) + (I# l#) #))) + +-- | Create a 'BS.ByteString' from a template prefix and then format the given +-- codepoint into hexadecimal string. +mkNameFromTemplate# :: Addr# -> Int# -> Int# -> BS.ByteString +mkNameFromTemplate# template# len# cp# = + let len'# = len# +# if isTrue# (cp# <# 0x10000#) then 4# else 5# + in unsafeDupablePerformIO (IO (\s0 -> case newPinnedByteArray# len'# s0 of + (# s1, marr# #) -> case copyAddrToByteArray# template# marr# 0# len# s1 of + s2 -> case go marr# s2 (len'# -# 1#) cp# of + s3 -> + (# s3 + , BS.BS (ForeignPtr (byteArrayContents# (unsafeCoerce# marr#)) + (PlainPtr marr#)) + (I# len'#) #) + where + -- Write hex in reverse order + -- [NOTE] We assume cp# >= 0x1000 to avoid to check for padding + go ma !s !offset = \case + 0# -> s + n -> go ma + (writeWord8Array# ma offset + (indexWord8OffAddr# "0123456789ABCDEF"# + (andI# 0xf# n)) s) + (offset -# 1#) + (uncheckedIShiftRL# n 4#) + + )) + +-- | Returns /corrected/ name of a character (see 'NameAliases.Correction'), +-- if defined, otherwise returns its original 'name' if defined. +-- +-- @since 0.3.0 +{-# INLINE correctedName #-} +correctedName :: Char -> Maybe BS.ByteString +correctedName c@(C# c#) = corrected <|> name c + where + -- Assumption: fromEnum NameAliases.Correction == 0 + !corrected = case indexCharOffAddr# addr# 0# of + '\xff'# -> Nothing -- no aliases + '\x00'# -> Nothing -- no correction + i# -> + let !n = unpackAddr# + (addr# `plusAddr#` (ord# i# +# 1#)) + (ord# (indexCharOffAddr# (addr# `plusAddr#` ord# i#) 0#)) + in Just n + !addr# = NameAliases.nameAliases c# + +-- | Returns a character’s 'name' if defined, +-- otherwise returns its first name alias if defined. +-- +-- @since 0.3.0 +nameOrAlias :: Char -> Maybe BS.ByteString +nameOrAlias c@(C# c#) = name c <|> case indexCharOffAddr# addr# 0# of + '\xff'# -> Nothing -- no aliases + '\x00'# -> let !n = go 1# in Just n + _ -> let !n = go 0# in Just n + where + !addr# = NameAliases.nameAliases c# + go t# = case ord# (indexCharOffAddr# (addr# `plusAddr#` t#) 0#) of + -- No bound check for t#: there is at least one alias + 0# -> go (t# +# 1#) + i# -> unpackAddr# + (addr# `plusAddr#` (i# +# 1#)) + (ord# (indexCharOffAddr# (addr# `plusAddr#` i#) 0#)) + +-- | All name aliases of a character, if defined. +-- The names are listed in the original order of the UCD. +-- +-- See 'nameAliasesWithTypes' for the detailed list by alias type. +-- +-- @since 0.3.0 +{-# INLINE nameAliases #-} +nameAliases :: Char -> [BS.ByteString] +nameAliases (C# c#) = case indexCharOffAddr# addr0# 0# of + '\xff'# -> [] -- no aliases + _ -> go (addr0# `plusAddr#` (NameAliases.MaxNameAliasType +# 1#)) + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> case ord# (indexCharOffAddr# (addr# `plusAddr#` 1#) 0#) of + -- End of list + 0# -> [] + -- Skip empty entry + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 2#) l# + in s : go (addr# `plusAddr#` (l# +# 2#)) + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) + where + addr0# = NameAliases.nameAliases c# + +-- | Name aliases of a character for a specific name alias type. +-- +-- @since 0.3.0 +{-# INLINE nameAliasesByType #-} +nameAliasesByType :: NameAliases.NameAliasType -> Char -> [BS.ByteString] +nameAliasesByType t (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + _ -> nameAliasesByType# addr# t + where + addr# = NameAliases.nameAliases c# + +-- | Detailed character names aliases. +-- The names are listed in the original order of the UCD. +-- +-- See 'nameAliases' if the alias type is not required. +-- +-- @since 0.3.0 +{-# INLINE nameAliasesWithTypes #-} +nameAliasesWithTypes :: Char -> [(NameAliases.NameAliasType, [BS.ByteString])] +nameAliasesWithTypes (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + '\x00'# -> foldr mk mempty [succ minBound..maxBound] + _ -> foldr mk mempty [minBound..maxBound] + where + addr# = NameAliases.nameAliases c# + mk t acc = case nameAliasesByType# addr# t of + [] -> acc + as -> (t, as) : acc + +{-# INLINE nameAliasesByType# #-} +nameAliasesByType# :: Addr# -> NameAliases.NameAliasType -> [BS.ByteString] +nameAliasesByType# addr0# t = + case ord# (indexCharOffAddr# (addr0# `plusAddr#` dataToTag# t) 0#) of + 0# -> [] -- no aliases for this type + i# -> go (addr0# `plusAddr#` i#) + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> [] + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) diff --git a/lib/Unicode/Char/General/Names/Text.hs b/lib/Unicode/Char/General/Names/Text.hs new file mode 100644 --- /dev/null +++ b/lib/Unicode/Char/General/Names/Text.hs @@ -0,0 +1,206 @@ +-- | +-- Module : Unicode.Char.General.Names.Text +-- Copyright : (c) 2022 Composewell Technologies and Contributors +-- License : Apache-2.0 +-- Maintainer : streamly@composewell.com +-- Stability : experimental +-- +-- Unicode character names and name aliases with 'Data.Text' API. +-- See Unicode standard 15.0.0, section 4.8. +-- +-- This API is /optional/ and requires using the package flag @has-text@. +-- +-- @since 0.3.0 + +module Unicode.Char.General.Names.Text + ( -- * Name + name + , nameOrAlias + , correctedName + -- * Name Aliases + , NameAliases.NameAliasType(..) + , nameAliases + , nameAliasesByType + , nameAliasesWithTypes + ) where + +import Control.Applicative ((<|>)) +import qualified Control.Monad.ST as ST +import qualified Data.Text as T +import qualified Data.Text.Internal as T +import qualified Data.Text.Array as A +import GHC.Exts + ( Addr#, Ptr(..), indexCharOffAddr#, indexWord8OffAddr#, plusAddr# + , Char(..), ord# + , Int#, Int(..), (+#), (-#), (<#), isTrue#, quotRemInt#, dataToTag# ) +import GHC.Word (Word8(..)) + +import qualified Unicode.Internal.Char.UnicodeData.DerivedName as DerivedName +import qualified Unicode.Internal.Char.UnicodeData.NameAliases as NameAliases + +-- | Name of a character, if defined. +-- +-- @since 0.3.0 +name :: Char -> Maybe T.Text +name (C# c#) = case DerivedName.name c# of + (# name#, len# #) -> case len# of + DerivedName.NoName -> Nothing + DerivedName.CjkCompatibilityIdeograph -> Just n + where + !n = mkNameFromTemplate "CJK COMPATIBILITY IDEOGRAPH-"# 28# (ord# c#) + DerivedName.CjkUnifiedIdeograph -> Just n + where + !n = mkNameFromTemplate "CJK UNIFIED IDEOGRAPH-"# 22# (ord# c#) + DerivedName.TangutIdeograph -> Just n + where + !n = mkNameFromTemplate "TANGUT IDEOGRAPH-"# 17# (ord# c#) + _ + | isTrue# (len# <# DerivedName.HangulSyllable) -> + let !n = unpackAddr# name# len# + in Just n + | otherwise -> + let !len = I# (len# -# DerivedName.HangulSyllable +# 16#) + ba = ST.runST (do + marr <- A.new len + A.copyFromPointer marr 0 (Ptr "HANGUL SYLLABLE "#) 16 + A.copyFromPointer marr 16 + (Ptr name#) + (I# (len# -# DerivedName.HangulSyllable)) + A.unsafeFreeze marr) + !n = T.Text ba 0 len + in Just n + +-- See: unpackCStringAscii#. Here we know the length. +unpackAddr# :: Addr# -> Int# -> T.Text +unpackAddr# addr# len# = T.Text ba 0 (I# len#) + where + ba = ST.runST (do + marr <- A.new (I# len#) + A.copyFromPointer marr 0 (Ptr addr#) (I# len#) + A.unsafeFreeze marr) + +mkNameFromTemplate :: Addr# -> Int# -> Int# -> T.Text +mkNameFromTemplate template# len# cp# = T.Text ba 0 (I# len'#) + where + len'# = len# +# if isTrue# (cp# <# 0x10000#) then 4# else 5# + ba = ST.runST (do + marr <- A.new (I# len'#) + A.copyFromPointer marr 0 (Ptr template#) (I# len#) + writeHex cp# (len'# -# 1#) marr + A.unsafeFreeze marr) + +-- [NOTE] We assume c# >= '\x1000' to avoid to check for padding +{-# INLINE writeHex #-} +writeHex :: Int# -> Int# -> A.MArray s -> ST.ST s () +writeHex cp# offset0# !marr = showIt offset0# (quotRemInt# cp# 16#) + where + showIt offset# (# q, r #) + = A.unsafeWrite marr (I# offset#) + (W8# (indexWord8OffAddr# "0123456789ABCDEF"# r)) + *> case q of + 0# -> pure () + _ -> showIt (offset# -# 1#) (quotRemInt# q 16#) + +-- | Returns /corrected/ name of a character (see 'NameAliases.Correction'), +-- if defined, otherwise returns its original 'name' if defined. +-- +-- @since 0.3.0 +{-# INLINE correctedName #-} +correctedName :: Char -> Maybe T.Text +correctedName c@(C# c#) = corrected <|> name c + where + -- Assumption: fromEnum NameAliases.Correction == 0 + !corrected = case ord# (indexCharOffAddr# addr# 0#) of + 0xff# -> Nothing -- no aliases + 0x00# -> Nothing -- no correction + i# -> + let l# = ord# (indexCharOffAddr# (addr# `plusAddr#` i#) 0#) + !n = unpackAddr# (addr# `plusAddr#` (i# +# 1#)) l# + in Just n + !addr# = NameAliases.nameAliases c# + +-- | Returns a character’s 'name' if defined, +-- otherwise returns its first name alias if defined. +-- +-- @since 0.3.0 +nameOrAlias :: Char -> Maybe T.Text +nameOrAlias c@(C# c#) = name c <|> case indexCharOffAddr# addr# 0# of + '\xff'# -> Nothing -- no aliases + '\x00'# -> let !n = go 1# in Just n + _ -> let !n = go 0# in Just n + where + !addr# = NameAliases.nameAliases c# + go t# = case ord# (indexCharOffAddr# (addr# `plusAddr#` t#) 0#) of + -- No bound check for t#: there is at least one alias + 0# -> go (t# +# 1#) + i# -> + let l# = ord# (indexCharOffAddr# (addr# `plusAddr#` i#) 0#) + in unpackAddr# (addr# `plusAddr#` (i# +# 1#)) l# + +-- | All name aliases of a character, if defined. +-- The names are listed in the original order of the UCD. +-- +-- See 'nameAliasesWithTypes' for the detailed list by alias type. +-- +-- @since 0.3.0 +{-# INLINE nameAliases #-} +nameAliases :: Char -> [T.Text] +nameAliases (C# c#) = case indexCharOffAddr# addr0# 0# of + '\xff'# -> [] -- no aliases + _ -> go (addr0# `plusAddr#` (NameAliases.MaxNameAliasType +# 1#)) + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> case ord# (indexCharOffAddr# (addr# `plusAddr#` 1#) 0#) of + -- End of list + 0# -> [] + -- Skip empty entry + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 2#) l# + in s : go (addr# `plusAddr#` (l# +# 2#)) + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) + where + addr0# = NameAliases.nameAliases c# + +-- | Name aliases of a character for a specific name alias type. +-- +-- @since 0.3.0 +{-# INLINE nameAliasesByType #-} +nameAliasesByType :: NameAliases.NameAliasType -> Char -> [T.Text] +nameAliasesByType t (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + _ -> nameAliasesByType# addr# t + where + addr# = NameAliases.nameAliases c# + +-- | Detailed character names aliases. +-- The names are listed in the original order of the UCD. +-- +-- See 'nameAliases' if the alias type is not required. +-- +-- @since 0.3.0 +{-# INLINE nameAliasesWithTypes #-} +nameAliasesWithTypes :: Char -> [(NameAliases.NameAliasType, [T.Text])] +nameAliasesWithTypes (C# c#) = case indexCharOffAddr# addr# 0# of + '\xff'# -> [] -- no aliases + '\x00'# -> foldr mk mempty [succ minBound..maxBound] + _ -> foldr mk mempty [minBound..maxBound] + where + addr# = NameAliases.nameAliases c# + mk t acc = case nameAliasesByType# addr# t of + [] -> acc + as -> (t, as) : acc + +{-# INLINE nameAliasesByType# #-} +nameAliasesByType# :: Addr# -> NameAliases.NameAliasType -> [T.Text] +nameAliasesByType# addr0# t = + case ord# (indexCharOffAddr# (addr0# `plusAddr#` dataToTag# t) 0#) of + 0# -> [] -- no aliases for this type + i# -> go (addr0# `plusAddr#` i#) + where + go addr# = case ord# (indexCharOffAddr# addr# 0#) of + 0# -> [] + l# -> + let !s = unpackAddr# (addr# `plusAddr#` 1#) l# + in s : go (addr# `plusAddr#` (l# +# 1#)) diff --git a/lib/Unicode/Internal/Bits/Names.hs b/lib/Unicode/Internal/Bits/Names.hs new file mode 100644 --- /dev/null +++ b/lib/Unicode/Internal/Bits/Names.hs @@ -0,0 +1,73 @@ +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Use fewer imports" #-} + +-- | +-- Module : Unicode.Internal.Bits +-- Copyright : (c) 2020 Andrew Lelechenko +-- (c) 2020 Composewell Technologies +-- License : BSD-3-Clause +-- Maintainer : streamly@composewell.com +-- Stability : experimental +-- Portability : GHC +-- +-- Fast, static bitmap lookup utilities + +module Unicode.Internal.Bits.Names + ( -- * Bitmap lookup + lookupInt32# + -- * CString + , unpackNBytes# + ) where + +#include "MachDeps.h" + +import GHC.Exts (Addr#, Int#) + +#ifdef WORDS_BIGENDIAN + +import GHC.Exts (narrow32Word#, word2Int#, byteSwap32#, indexWord32OffAddr#) +#if MIN_VERSION_base(4,16,0) +import GHC.Exts (word32ToWord#) +#endif + +#else + +import GHC.Exts (indexInt32OffAddr#) +#if MIN_VERSION_base(4,16,0) +import GHC.Exts (int32ToInt#) +#endif + +#endif + +#if MIN_VERSION_base(4,15,0) +import GHC.Exts (unpackNBytes#) +#else +import GHC.CString (unpackNBytes#) +#endif + +{-| @lookupInt32# addr index@ looks up for the @index@-th 32-bits word in +the bitmap starting at @addr@, then convert it to an 'Int#'. + +The caller must make sure that: + +* @ceiling (addr + (n * 32))@ is legally accessible 'GHC.Exts.Word32#'. + +@since 0.4.1 +-} +lookupInt32# + :: Addr# -- ^ Bitmap address + -> Int# -- ^ Word index + -> Int# -- ^ Resulting int +lookupInt32# +#ifdef WORDS_BIGENDIAN +#if MIN_VERSION_base(4,16,0) + addr# k# = word2Int# (narrow32Word# (byteSwap32# (word32ToWord# (indexWord32OffAddr# addr# k#)))) +#else + addr# k# = word2Int# (narrow32Word# (byteSwap32# (indexWord32OffAddr# addr# k#))) +#endif +#elif MIN_VERSION_base(4,16,0) + addr# k# = int32ToInt# (indexInt32OffAddr# addr# k#) +#else + = indexInt32OffAddr# +#endif diff --git a/lib/Unicode/Internal/Char/Names/Version.hs b/lib/Unicode/Internal/Char/Names/Version.hs new file mode 100644 --- /dev/null +++ b/lib/Unicode/Internal/Char/Names/Version.hs @@ -0,0 +1,20 @@ +-- DO NOT EDIT MANUALLY: autogenerated by ucd2haskell +{-# OPTIONS_HADDOCK hide #-} + +-- | +-- Module : Unicode.Internal.Char.Names.Version +-- Copyright : (c) 2024 Composewell Technologies and Contributors +-- License : Apache-2.0 +-- Maintainer : streamly@composewell.com +-- Stability : experimental + +module Unicode.Internal.Char.Names.Version (unicodeVersion) where + +import Data.Version (Version, makeVersion) + +-- | Version of the Unicode standard used by this package: +-- [15.0.0](https://www.unicode.org/versions/Unicode15.0.0/). +-- +-- @since 0.3.0 +unicodeVersion :: Version +unicodeVersion = makeVersion [15,0,0] diff --git a/lib/Unicode/Internal/Char/UnicodeData/DerivedName.hs b/lib/Unicode/Internal/Char/UnicodeData/DerivedName.hs # file too large to diff: lib/Unicode/Internal/Char/UnicodeData/DerivedName.hs diff --git a/lib/Unicode/Internal/Char/UnicodeData/NameAliases.hs b/lib/Unicode/Internal/Char/UnicodeData/NameAliases.hs --- a/lib/Unicode/Internal/Char/UnicodeData/NameAliases.hs +++ b/lib/Unicode/Internal/Char/UnicodeData/NameAliases.hs @@ -6,16 +6,16 @@ -- Maintainer : streamly@composewell.com -- Stability : experimental +{-# LANGUAGE DeriveGeneric, PatternSynonyms #-} {-# OPTIONS_HADDOCK hide #-} module Unicode.Internal.Char.UnicodeData.NameAliases -(NameAliasType(..), nameAliases, nameAliasesByType, nameAliasesWithTypes) +(NameAliasType(..), pattern MaxNameAliasType, nameAliases) where import Data.Ix (Ix) -import Data.Maybe (fromMaybe) -import Foreign.C.String (CString) -import GHC.Exts (Ptr(..)) +import GHC.Exts (Addr#, Char#, Int#) +import GHC.Generics (Generic) -- | Type of name alias. See Unicode Standard 15.0.0, section 4.8. -- @@ -34,412 +34,413 @@ | Abbreviation -- ^ Commonly occurring abbreviations (or acronyms) for control codes, -- format characters, spaces, and variation selectors. - deriving (Enum, Bounded, Eq, Ord, Ix, Show) + deriving (Generic, Enum, Bounded, Eq, Ord, Ix, Show) --- | All name aliases of a character. --- The names are listed in the original order of the UCD. --- --- See 'nameAliasesWithTypes' for the detailed list by alias type. --- --- @since 0.1.0 -{-# INLINE nameAliases #-} -nameAliases :: Char -> [CString] -nameAliases = mconcat . fmap snd . nameAliasesWithTypes +-- $setup +-- >>> import GHC.Exts (Int(..)) --- | Name aliases of a character for a specific name alias type. --- --- @since 0.1.0 -{-# INLINE nameAliasesByType #-} -nameAliasesByType :: NameAliasType -> Char -> [CString] -nameAliasesByType t = fromMaybe mempty . lookup t . nameAliasesWithTypes +-- | +-- >>> I# MaxNameAliasType == fromEnum (maxBound :: NameAliasType) +-- True +pattern MaxNameAliasType :: Int# +pattern MaxNameAliasType = 4# -- | Detailed character names aliases. -- The names are listed in the original order of the UCD. -- --- See 'nameAliases' if the alias type is not required. +-- Encoding: -- +-- * If there is no alias, return @"\\xff"#@. +-- * For each type of alias, the aliases are encoded as list of (length, alias). +-- The list terminates with @\\0@. +-- * The list are then concatenated in order of type of alias and +-- terminates with @\\0@. +-- * The first 5 bytes represent each one the index of the first element of the +-- corresponding list of aliases. When the list is empty, then the index is 0. +-- * Example: @\"\\5\\0\\13\\0\\0\\3XXX\\2YY\\0\\4ZZZZ\\0\\0\"#@ +-- represents: @[('Correction',[\"XXX\", \"YY\"]),('Alternate', [\"ZZZZ\"])]@. +-- -- @since 0.1.0 -nameAliasesWithTypes :: Char -> [(NameAliasType, [CString])] -nameAliasesWithTypes = \case - '\x0000' -> [(Control,[Ptr "NULL\0"#]),(Abbreviation,[Ptr "NUL\0"#])] - '\x0001' -> [(Control,[Ptr "START OF HEADING\0"#]),(Abbreviation,[Ptr "SOH\0"#])] - '\x0002' -> [(Control,[Ptr "START OF TEXT\0"#]),(Abbreviation,[Ptr "STX\0"#])] - '\x0003' -> [(Control,[Ptr "END OF TEXT\0"#]),(Abbreviation,[Ptr "ETX\0"#])] - '\x0004' -> [(Control,[Ptr "END OF TRANSMISSION\0"#]),(Abbreviation,[Ptr "EOT\0"#])] - '\x0005' -> [(Control,[Ptr "ENQUIRY\0"#]),(Abbreviation,[Ptr "ENQ\0"#])] - '\x0006' -> [(Control,[Ptr "ACKNOWLEDGE\0"#]),(Abbreviation,[Ptr "ACK\0"#])] - '\x0007' -> [(Control,[Ptr "ALERT\0"#]),(Abbreviation,[Ptr "BEL\0"#])] - '\x0008' -> [(Control,[Ptr "BACKSPACE\0"#]),(Abbreviation,[Ptr "BS\0"#])] - '\x0009' -> [(Control,[Ptr "CHARACTER TABULATION\0"#,Ptr "HORIZONTAL TABULATION\0"#]),(Abbreviation,[Ptr "HT\0"#,Ptr "TAB\0"#])] - '\x000a' -> [(Control,[Ptr "LINE FEED\0"#,Ptr "NEW LINE\0"#,Ptr "END OF LINE\0"#]),(Abbreviation,[Ptr "LF\0"#,Ptr "NL\0"#,Ptr "EOL\0"#])] - '\x000b' -> [(Control,[Ptr "LINE TABULATION\0"#,Ptr "VERTICAL TABULATION\0"#]),(Abbreviation,[Ptr "VT\0"#])] - '\x000c' -> [(Control,[Ptr "FORM FEED\0"#]),(Abbreviation,[Ptr "FF\0"#])] - '\x000d' -> [(Control,[Ptr "CARRIAGE RETURN\0"#]),(Abbreviation,[Ptr "CR\0"#])] - '\x000e' -> [(Control,[Ptr "SHIFT OUT\0"#,Ptr "LOCKING-SHIFT ONE\0"#]),(Abbreviation,[Ptr "SO\0"#])] - '\x000f' -> [(Control,[Ptr "SHIFT IN\0"#,Ptr "LOCKING-SHIFT ZERO\0"#]),(Abbreviation,[Ptr "SI\0"#])] - '\x0010' -> [(Control,[Ptr "DATA LINK ESCAPE\0"#]),(Abbreviation,[Ptr "DLE\0"#])] - '\x0011' -> [(Control,[Ptr "DEVICE CONTROL ONE\0"#]),(Abbreviation,[Ptr "DC1\0"#])] - '\x0012' -> [(Control,[Ptr "DEVICE CONTROL TWO\0"#]),(Abbreviation,[Ptr "DC2\0"#])] - '\x0013' -> [(Control,[Ptr "DEVICE CONTROL THREE\0"#]),(Abbreviation,[Ptr "DC3\0"#])] - '\x0014' -> [(Control,[Ptr "DEVICE CONTROL FOUR\0"#]),(Abbreviation,[Ptr "DC4\0"#])] - '\x0015' -> [(Control,[Ptr "NEGATIVE ACKNOWLEDGE\0"#]),(Abbreviation,[Ptr "NAK\0"#])] - '\x0016' -> [(Control,[Ptr "SYNCHRONOUS IDLE\0"#]),(Abbreviation,[Ptr "SYN\0"#])] - '\x0017' -> [(Control,[Ptr "END OF TRANSMISSION BLOCK\0"#]),(Abbreviation,[Ptr "ETB\0"#])] - '\x0018' -> [(Control,[Ptr "CANCEL\0"#]),(Abbreviation,[Ptr "CAN\0"#])] - '\x0019' -> [(Control,[Ptr "END OF MEDIUM\0"#]),(Abbreviation,[Ptr "EOM\0"#,Ptr "EM\0"#])] - '\x001a' -> [(Control,[Ptr "SUBSTITUTE\0"#]),(Abbreviation,[Ptr "SUB\0"#])] - '\x001b' -> [(Control,[Ptr "ESCAPE\0"#]),(Abbreviation,[Ptr "ESC\0"#])] - '\x001c' -> [(Control,[Ptr "INFORMATION SEPARATOR FOUR\0"#,Ptr "FILE SEPARATOR\0"#]),(Abbreviation,[Ptr "FS\0"#])] - '\x001d' -> [(Control,[Ptr "INFORMATION SEPARATOR THREE\0"#,Ptr "GROUP SEPARATOR\0"#]),(Abbreviation,[Ptr "GS\0"#])] - '\x001e' -> [(Control,[Ptr "INFORMATION SEPARATOR TWO\0"#,Ptr "RECORD SEPARATOR\0"#]),(Abbreviation,[Ptr "RS\0"#])] - '\x001f' -> [(Control,[Ptr "INFORMATION SEPARATOR ONE\0"#,Ptr "UNIT SEPARATOR\0"#]),(Abbreviation,[Ptr "US\0"#])] - '\x0020' -> [(Abbreviation,[Ptr "SP\0"#])] - '\x007f' -> [(Control,[Ptr "DELETE\0"#]),(Abbreviation,[Ptr "DEL\0"#])] - '\x0080' -> [(Figment,[Ptr "PADDING CHARACTER\0"#]),(Abbreviation,[Ptr "PAD\0"#])] - '\x0081' -> [(Figment,[Ptr "HIGH OCTET PRESET\0"#]),(Abbreviation,[Ptr "HOP\0"#])] - '\x0082' -> [(Control,[Ptr "BREAK PERMITTED HERE\0"#]),(Abbreviation,[Ptr "BPH\0"#])] - '\x0083' -> [(Control,[Ptr "NO BREAK HERE\0"#]),(Abbreviation,[Ptr "NBH\0"#])] - '\x0084' -> [(Control,[Ptr "INDEX\0"#]),(Abbreviation,[Ptr "IND\0"#])] - '\x0085' -> [(Control,[Ptr "NEXT LINE\0"#]),(Abbreviation,[Ptr "NEL\0"#])] - '\x0086' -> [(Control,[Ptr "START OF SELECTED AREA\0"#]),(Abbreviation,[Ptr "SSA\0"#])] - '\x0087' -> [(Control,[Ptr "END OF SELECTED AREA\0"#]),(Abbreviation,[Ptr "ESA\0"#])] - '\x0088' -> [(Control,[Ptr "CHARACTER TABULATION SET\0"#,Ptr "HORIZONTAL TABULATION SET\0"#]),(Abbreviation,[Ptr "HTS\0"#])] - '\x0089' -> [(Control,[Ptr "CHARACTER TABULATION WITH JUSTIFICATION\0"#,Ptr "HORIZONTAL TABULATION WITH JUSTIFICATION\0"#]),(Abbreviation,[Ptr "HTJ\0"#])] - '\x008a' -> [(Control,[Ptr "LINE TABULATION SET\0"#,Ptr "VERTICAL TABULATION SET\0"#]),(Abbreviation,[Ptr "VTS\0"#])] - '\x008b' -> [(Control,[Ptr "PARTIAL LINE FORWARD\0"#,Ptr "PARTIAL LINE DOWN\0"#]),(Abbreviation,[Ptr "PLD\0"#])] - '\x008c' -> [(Control,[Ptr "PARTIAL LINE BACKWARD\0"#,Ptr "PARTIAL LINE UP\0"#]),(Abbreviation,[Ptr "PLU\0"#])] - '\x008d' -> [(Control,[Ptr "REVERSE LINE FEED\0"#,Ptr "REVERSE INDEX\0"#]),(Abbreviation,[Ptr "RI\0"#])] - '\x008e' -> [(Control,[Ptr "SINGLE SHIFT TWO\0"#,Ptr "SINGLE-SHIFT-2\0"#]),(Abbreviation,[Ptr "SS2\0"#])] - '\x008f' -> [(Control,[Ptr "SINGLE SHIFT THREE\0"#,Ptr "SINGLE-SHIFT-3\0"#]),(Abbreviation,[Ptr "SS3\0"#])] - '\x0090' -> [(Control,[Ptr "DEVICE CONTROL STRING\0"#]),(Abbreviation,[Ptr "DCS\0"#])] - '\x0091' -> [(Control,[Ptr "PRIVATE USE ONE\0"#,Ptr "PRIVATE USE-1\0"#]),(Abbreviation,[Ptr "PU1\0"#])] - '\x0092' -> [(Control,[Ptr "PRIVATE USE TWO\0"#,Ptr "PRIVATE USE-2\0"#]),(Abbreviation,[Ptr "PU2\0"#])] - '\x0093' -> [(Control,[Ptr "SET TRANSMIT STATE\0"#]),(Abbreviation,[Ptr "STS\0"#])] - '\x0094' -> [(Control,[Ptr "CANCEL CHARACTER\0"#]),(Abbreviation,[Ptr "CCH\0"#])] - '\x0095' -> [(Control,[Ptr "MESSAGE WAITING\0"#]),(Abbreviation,[Ptr "MW\0"#])] - '\x0096' -> [(Control,[Ptr "START OF GUARDED AREA\0"#,Ptr "START OF PROTECTED AREA\0"#]),(Abbreviation,[Ptr "SPA\0"#])] - '\x0097' -> [(Control,[Ptr "END OF GUARDED AREA\0"#,Ptr "END OF PROTECTED AREA\0"#]),(Abbreviation,[Ptr "EPA\0"#])] - '\x0098' -> [(Control,[Ptr "START OF STRING\0"#]),(Abbreviation,[Ptr "SOS\0"#])] - '\x0099' -> [(Figment,[Ptr "SINGLE GRAPHIC CHARACTER INTRODUCER\0"#]),(Abbreviation,[Ptr "SGC\0"#])] - '\x009a' -> [(Control,[Ptr "SINGLE CHARACTER INTRODUCER\0"#]),(Abbreviation,[Ptr "SCI\0"#])] - '\x009b' -> [(Control,[Ptr "CONTROL SEQUENCE INTRODUCER\0"#]),(Abbreviation,[Ptr "CSI\0"#])] - '\x009c' -> [(Control,[Ptr "STRING TERMINATOR\0"#]),(Abbreviation,[Ptr "ST\0"#])] - '\x009d' -> [(Control,[Ptr "OPERATING SYSTEM COMMAND\0"#]),(Abbreviation,[Ptr "OSC\0"#])] - '\x009e' -> [(Control,[Ptr "PRIVACY MESSAGE\0"#]),(Abbreviation,[Ptr "PM\0"#])] - '\x009f' -> [(Control,[Ptr "APPLICATION PROGRAM COMMAND\0"#]),(Abbreviation,[Ptr "APC\0"#])] - '\x00a0' -> [(Abbreviation,[Ptr "NBSP\0"#])] - '\x00ad' -> [(Abbreviation,[Ptr "SHY\0"#])] - '\x01a2' -> [(Correction,[Ptr "LATIN CAPITAL LETTER GHA\0"#])] - '\x01a3' -> [(Correction,[Ptr "LATIN SMALL LETTER GHA\0"#])] - '\x034f' -> [(Abbreviation,[Ptr "CGJ\0"#])] - '\x0616' -> [(Correction,[Ptr "ARABIC SMALL HIGH LIGATURE ALEF WITH YEH BARREE\0"#])] - '\x061c' -> [(Abbreviation,[Ptr "ALM\0"#])] - '\x0709' -> [(Correction,[Ptr "SYRIAC SUBLINEAR COLON SKEWED LEFT\0"#])] - '\x0cde' -> [(Correction,[Ptr "KANNADA LETTER LLLA\0"#])] - '\x0e9d' -> [(Correction,[Ptr "LAO LETTER FO FON\0"#])] - '\x0e9f' -> [(Correction,[Ptr "LAO LETTER FO FAY\0"#])] - '\x0ea3' -> [(Correction,[Ptr "LAO LETTER RO\0"#])] - '\x0ea5' -> [(Correction,[Ptr "LAO LETTER LO\0"#])] - '\x0fd0' -> [(Correction,[Ptr "TIBETAN MARK BKA- SHOG GI MGO RGYAN\0"#])] - '\x11ec' -> [(Correction,[Ptr "HANGUL JONGSEONG YESIEUNG-KIYEOK\0"#])] - '\x11ed' -> [(Correction,[Ptr "HANGUL JONGSEONG YESIEUNG-SSANGKIYEOK\0"#])] - '\x11ee' -> [(Correction,[Ptr "HANGUL JONGSEONG SSANGYESIEUNG\0"#])] - '\x11ef' -> [(Correction,[Ptr "HANGUL JONGSEONG YESIEUNG-KHIEUKH\0"#])] - '\x180b' -> [(Abbreviation,[Ptr "FVS1\0"#])] - '\x180c' -> [(Abbreviation,[Ptr "FVS2\0"#])] - '\x180d' -> [(Abbreviation,[Ptr "FVS3\0"#])] - '\x180e' -> [(Abbreviation,[Ptr "MVS\0"#])] - '\x180f' -> [(Abbreviation,[Ptr "FVS4\0"#])] - '\x1bbd' -> [(Correction,[Ptr "SUNDANESE LETTER ARCHAIC I\0"#])] - '\x200b' -> [(Abbreviation,[Ptr "ZWSP\0"#])] - '\x200c' -> [(Abbreviation,[Ptr "ZWNJ\0"#])] - '\x200d' -> [(Abbreviation,[Ptr "ZWJ\0"#])] - '\x200e' -> [(Abbreviation,[Ptr "LRM\0"#])] - '\x200f' -> [(Abbreviation,[Ptr "RLM\0"#])] - '\x202a' -> [(Abbreviation,[Ptr "LRE\0"#])] - '\x202b' -> [(Abbreviation,[Ptr "RLE\0"#])] - '\x202c' -> [(Abbreviation,[Ptr "PDF\0"#])] - '\x202d' -> [(Abbreviation,[Ptr "LRO\0"#])] - '\x202e' -> [(Abbreviation,[Ptr "RLO\0"#])] - '\x202f' -> [(Abbreviation,[Ptr "NNBSP\0"#])] - '\x205f' -> [(Abbreviation,[Ptr "MMSP\0"#])] - '\x2060' -> [(Abbreviation,[Ptr "WJ\0"#])] - '\x2066' -> [(Abbreviation,[Ptr "LRI\0"#])] - '\x2067' -> [(Abbreviation,[Ptr "RLI\0"#])] - '\x2068' -> [(Abbreviation,[Ptr "FSI\0"#])] - '\x2069' -> [(Abbreviation,[Ptr "PDI\0"#])] - '\x2118' -> [(Correction,[Ptr "WEIERSTRASS ELLIPTIC FUNCTION\0"#])] - '\x2448' -> [(Correction,[Ptr "MICR ON US SYMBOL\0"#])] - '\x2449' -> [(Correction,[Ptr "MICR DASH SYMBOL\0"#])] - '\x2b7a' -> [(Correction,[Ptr "LEFTWARDS TRIANGLE-HEADED ARROW WITH DOUBLE VERTICAL STROKE\0"#])] - '\x2b7c' -> [(Correction,[Ptr "RIGHTWARDS TRIANGLE-HEADED ARROW WITH DOUBLE VERTICAL STROKE\0"#])] - '\xa015' -> [(Correction,[Ptr "YI SYLLABLE ITERATION MARK\0"#])] - '\xaa6e' -> [(Correction,[Ptr "MYANMAR LETTER KHAMTI LLA\0"#])] - '\xfe00' -> [(Abbreviation,[Ptr "VS1\0"#])] - '\xfe01' -> [(Abbreviation,[Ptr "VS2\0"#])] - '\xfe02' -> [(Abbreviation,[Ptr "VS3\0"#])] - '\xfe03' -> [(Abbreviation,[Ptr "VS4\0"#])] - '\xfe04' -> [(Abbreviation,[Ptr "VS5\0"#])] - '\xfe05' -> [(Abbreviation,[Ptr "VS6\0"#])] - '\xfe06' -> [(Abbreviation,[Ptr "VS7\0"#])] - '\xfe07' -> [(Abbreviation,[Ptr "VS8\0"#])] - '\xfe08' -> [(Abbreviation,[Ptr "VS9\0"#])] - '\xfe09' -> [(Abbreviation,[Ptr "VS10\0"#])] - '\xfe0a' -> [(Abbreviation,[Ptr "VS11\0"#])] - '\xfe0b' -> [(Abbreviation,[Ptr "VS12\0"#])] - '\xfe0c' -> [(Abbreviation,[Ptr "VS13\0"#])] - '\xfe0d' -> [(Abbreviation,[Ptr "VS14\0"#])] - '\xfe0e' -> [(Abbreviation,[Ptr "VS15\0"#])] - '\xfe0f' -> [(Abbreviation,[Ptr "VS16\0"#])] - '\xfe18' -> [(Correction,[Ptr "PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET\0"#])] - '\xfeff' -> [(Alternate,[Ptr "BYTE ORDER MARK\0"#]),(Abbreviation,[Ptr "BOM\0"#,Ptr "ZWNBSP\0"#])] - '\x122d4' -> [(Correction,[Ptr "CUNEIFORM SIGN NU11 TENU\0"#])] - '\x122d5' -> [(Correction,[Ptr "CUNEIFORM SIGN NU11 OVER NU11 BUR OVER BUR\0"#])] - '\x16e56' -> [(Correction,[Ptr "MEDEFAIDRIN CAPITAL LETTER H\0"#])] - '\x16e57' -> [(Correction,[Ptr "MEDEFAIDRIN CAPITAL LETTER NG\0"#])] - '\x16e76' -> [(Correction,[Ptr "MEDEFAIDRIN SMALL LETTER H\0"#])] - '\x16e77' -> [(Correction,[Ptr "MEDEFAIDRIN SMALL LETTER NG\0"#])] - '\x1b001' -> [(Correction,[Ptr "HENTAIGANA LETTER E-1\0"#])] - '\x1d0c5' -> [(Correction,[Ptr "BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS\0"#])] - '\xe0100' -> [(Abbreviation,[Ptr "VS17\0"#])] - '\xe0101' -> [(Abbreviation,[Ptr "VS18\0"#])] - '\xe0102' -> [(Abbreviation,[Ptr "VS19\0"#])] - '\xe0103' -> [(Abbreviation,[Ptr "VS20\0"#])] - '\xe0104' -> [(Abbreviation,[Ptr "VS21\0"#])] - '\xe0105' -> [(Abbreviation,[Ptr "VS22\0"#])] - '\xe0106' -> [(Abbreviation,[Ptr "VS23\0"#])] - '\xe0107' -> [(Abbreviation,[Ptr "VS24\0"#])] - '\xe0108' -> [(Abbreviation,[Ptr "VS25\0"#])] - '\xe0109' -> [(Abbreviation,[Ptr "VS26\0"#])] - '\xe010a' -> [(Abbreviation,[Ptr "VS27\0"#])] - '\xe010b' -> [(Abbreviation,[Ptr "VS28\0"#])] - '\xe010c' -> [(Abbreviation,[Ptr "VS29\0"#])] - '\xe010d' -> [(Abbreviation,[Ptr "VS30\0"#])] - '\xe010e' -> [(Abbreviation,[Ptr "VS31\0"#])] - '\xe010f' -> [(Abbreviation,[Ptr "VS32\0"#])] - '\xe0110' -> [(Abbreviation,[Ptr "VS33\0"#])] - '\xe0111' -> [(Abbreviation,[Ptr "VS34\0"#])] - '\xe0112' -> [(Abbreviation,[Ptr "VS35\0"#])] - '\xe0113' -> [(Abbreviation,[Ptr "VS36\0"#])] - '\xe0114' -> [(Abbreviation,[Ptr "VS37\0"#])] - '\xe0115' -> [(Abbreviation,[Ptr "VS38\0"#])] - '\xe0116' -> [(Abbreviation,[Ptr "VS39\0"#])] - '\xe0117' -> [(Abbreviation,[Ptr "VS40\0"#])] - '\xe0118' -> [(Abbreviation,[Ptr "VS41\0"#])] - '\xe0119' -> [(Abbreviation,[Ptr "VS42\0"#])] - '\xe011a' -> [(Abbreviation,[Ptr "VS43\0"#])] - '\xe011b' -> [(Abbreviation,[Ptr "VS44\0"#])] - '\xe011c' -> [(Abbreviation,[Ptr "VS45\0"#])] - '\xe011d' -> [(Abbreviation,[Ptr "VS46\0"#])] - '\xe011e' -> [(Abbreviation,[Ptr "VS47\0"#])] - '\xe011f' -> [(Abbreviation,[Ptr "VS48\0"#])] - '\xe0120' -> [(Abbreviation,[Ptr "VS49\0"#])] - '\xe0121' -> [(Abbreviation,[Ptr "VS50\0"#])] - '\xe0122' -> [(Abbreviation,[Ptr "VS51\0"#])] - '\xe0123' -> [(Abbreviation,[Ptr "VS52\0"#])] - '\xe0124' -> [(Abbreviation,[Ptr "VS53\0"#])] - '\xe0125' -> [(Abbreviation,[Ptr "VS54\0"#])] - '\xe0126' -> [(Abbreviation,[Ptr "VS55\0"#])] - '\xe0127' -> [(Abbreviation,[Ptr "VS56\0"#])] - '\xe0128' -> [(Abbreviation,[Ptr "VS57\0"#])] - '\xe0129' -> [(Abbreviation,[Ptr "VS58\0"#])] - '\xe012a' -> [(Abbreviation,[Ptr "VS59\0"#])] - '\xe012b' -> [(Abbreviation,[Ptr "VS60\0"#])] - '\xe012c' -> [(Abbreviation,[Ptr "VS61\0"#])] - '\xe012d' -> [(Abbreviation,[Ptr "VS62\0"#])] - '\xe012e' -> [(Abbreviation,[Ptr "VS63\0"#])] - '\xe012f' -> [(Abbreviation,[Ptr "VS64\0"#])] - '\xe0130' -> [(Abbreviation,[Ptr "VS65\0"#])] - '\xe0131' -> [(Abbreviation,[Ptr "VS66\0"#])] - '\xe0132' -> [(Abbreviation,[Ptr "VS67\0"#])] - '\xe0133' -> [(Abbreviation,[Ptr "VS68\0"#])] - '\xe0134' -> [(Abbreviation,[Ptr "VS69\0"#])] - '\xe0135' -> [(Abbreviation,[Ptr "VS70\0"#])] - '\xe0136' -> [(Abbreviation,[Ptr "VS71\0"#])] - '\xe0137' -> [(Abbreviation,[Ptr "VS72\0"#])] - '\xe0138' -> [(Abbreviation,[Ptr "VS73\0"#])] - '\xe0139' -> [(Abbreviation,[Ptr "VS74\0"#])] - '\xe013a' -> [(Abbreviation,[Ptr "VS75\0"#])] - '\xe013b' -> [(Abbreviation,[Ptr "VS76\0"#])] - '\xe013c' -> [(Abbreviation,[Ptr "VS77\0"#])] - '\xe013d' -> [(Abbreviation,[Ptr "VS78\0"#])] - '\xe013e' -> [(Abbreviation,[Ptr "VS79\0"#])] - '\xe013f' -> [(Abbreviation,[Ptr "VS80\0"#])] - '\xe0140' -> [(Abbreviation,[Ptr "VS81\0"#])] - '\xe0141' -> [(Abbreviation,[Ptr "VS82\0"#])] - '\xe0142' -> [(Abbreviation,[Ptr "VS83\0"#])] - '\xe0143' -> [(Abbreviation,[Ptr "VS84\0"#])] - '\xe0144' -> [(Abbreviation,[Ptr "VS85\0"#])] - '\xe0145' -> [(Abbreviation,[Ptr "VS86\0"#])] - '\xe0146' -> [(Abbreviation,[Ptr "VS87\0"#])] - '\xe0147' -> [(Abbreviation,[Ptr "VS88\0"#])] - '\xe0148' -> [(Abbreviation,[Ptr "VS89\0"#])] - '\xe0149' -> [(Abbreviation,[Ptr "VS90\0"#])] - '\xe014a' -> [(Abbreviation,[Ptr "VS91\0"#])] - '\xe014b' -> [(Abbreviation,[Ptr "VS92\0"#])] - '\xe014c' -> [(Abbreviation,[Ptr "VS93\0"#])] - '\xe014d' -> [(Abbreviation,[Ptr "VS94\0"#])] - '\xe014e' -> [(Abbreviation,[Ptr "VS95\0"#])] - '\xe014f' -> [(Abbreviation,[Ptr "VS96\0"#])] - '\xe0150' -> [(Abbreviation,[Ptr "VS97\0"#])] - '\xe0151' -> [(Abbreviation,[Ptr "VS98\0"#])] - '\xe0152' -> [(Abbreviation,[Ptr "VS99\0"#])] - '\xe0153' -> [(Abbreviation,[Ptr "VS100\0"#])] - '\xe0154' -> [(Abbreviation,[Ptr "VS101\0"#])] - '\xe0155' -> [(Abbreviation,[Ptr "VS102\0"#])] - '\xe0156' -> [(Abbreviation,[Ptr "VS103\0"#])] - '\xe0157' -> [(Abbreviation,[Ptr "VS104\0"#])] - '\xe0158' -> [(Abbreviation,[Ptr "VS105\0"#])] - '\xe0159' -> [(Abbreviation,[Ptr "VS106\0"#])] - '\xe015a' -> [(Abbreviation,[Ptr "VS107\0"#])] - '\xe015b' -> [(Abbreviation,[Ptr "VS108\0"#])] - '\xe015c' -> [(Abbreviation,[Ptr "VS109\0"#])] - '\xe015d' -> [(Abbreviation,[Ptr "VS110\0"#])] - '\xe015e' -> [(Abbreviation,[Ptr "VS111\0"#])] - '\xe015f' -> [(Abbreviation,[Ptr "VS112\0"#])] - '\xe0160' -> [(Abbreviation,[Ptr "VS113\0"#])] - '\xe0161' -> [(Abbreviation,[Ptr "VS114\0"#])] - '\xe0162' -> [(Abbreviation,[Ptr "VS115\0"#])] - '\xe0163' -> [(Abbreviation,[Ptr "VS116\0"#])] - '\xe0164' -> [(Abbreviation,[Ptr "VS117\0"#])] - '\xe0165' -> [(Abbreviation,[Ptr "VS118\0"#])] - '\xe0166' -> [(Abbreviation,[Ptr "VS119\0"#])] - '\xe0167' -> [(Abbreviation,[Ptr "VS120\0"#])] - '\xe0168' -> [(Abbreviation,[Ptr "VS121\0"#])] - '\xe0169' -> [(Abbreviation,[Ptr "VS122\0"#])] - '\xe016a' -> [(Abbreviation,[Ptr "VS123\0"#])] - '\xe016b' -> [(Abbreviation,[Ptr "VS124\0"#])] - '\xe016c' -> [(Abbreviation,[Ptr "VS125\0"#])] - '\xe016d' -> [(Abbreviation,[Ptr "VS126\0"#])] - '\xe016e' -> [(Abbreviation,[Ptr "VS127\0"#])] - '\xe016f' -> [(Abbreviation,[Ptr "VS128\0"#])] - '\xe0170' -> [(Abbreviation,[Ptr "VS129\0"#])] - '\xe0171' -> [(Abbreviation,[Ptr "VS130\0"#])] - '\xe0172' -> [(Abbreviation,[Ptr "VS131\0"#])] - '\xe0173' -> [(Abbreviation,[Ptr "VS132\0"#])] - '\xe0174' -> [(Abbreviation,[Ptr "VS133\0"#])] - '\xe0175' -> [(Abbreviation,[Ptr "VS134\0"#])] - '\xe0176' -> [(Abbreviation,[Ptr "VS135\0"#])] - '\xe0177' -> [(Abbreviation,[Ptr "VS136\0"#])] - '\xe0178' -> [(Abbreviation,[Ptr "VS137\0"#])] - '\xe0179' -> [(Abbreviation,[Ptr "VS138\0"#])] - '\xe017a' -> [(Abbreviation,[Ptr "VS139\0"#])] - '\xe017b' -> [(Abbreviation,[Ptr "VS140\0"#])] - '\xe017c' -> [(Abbreviation,[Ptr "VS141\0"#])] - '\xe017d' -> [(Abbreviation,[Ptr "VS142\0"#])] - '\xe017e' -> [(Abbreviation,[Ptr "VS143\0"#])] - '\xe017f' -> [(Abbreviation,[Ptr "VS144\0"#])] - '\xe0180' -> [(Abbreviation,[Ptr "VS145\0"#])] - '\xe0181' -> [(Abbreviation,[Ptr "VS146\0"#])] - '\xe0182' -> [(Abbreviation,[Ptr "VS147\0"#])] - '\xe0183' -> [(Abbreviation,[Ptr "VS148\0"#])] - '\xe0184' -> [(Abbreviation,[Ptr "VS149\0"#])] - '\xe0185' -> [(Abbreviation,[Ptr "VS150\0"#])] - '\xe0186' -> [(Abbreviation,[Ptr "VS151\0"#])] - '\xe0187' -> [(Abbreviation,[Ptr "VS152\0"#])] - '\xe0188' -> [(Abbreviation,[Ptr "VS153\0"#])] - '\xe0189' -> [(Abbreviation,[Ptr "VS154\0"#])] - '\xe018a' -> [(Abbreviation,[Ptr "VS155\0"#])] - '\xe018b' -> [(Abbreviation,[Ptr "VS156\0"#])] - '\xe018c' -> [(Abbreviation,[Ptr "VS157\0"#])] - '\xe018d' -> [(Abbreviation,[Ptr "VS158\0"#])] - '\xe018e' -> [(Abbreviation,[Ptr "VS159\0"#])] - '\xe018f' -> [(Abbreviation,[Ptr "VS160\0"#])] - '\xe0190' -> [(Abbreviation,[Ptr "VS161\0"#])] - '\xe0191' -> [(Abbreviation,[Ptr "VS162\0"#])] - '\xe0192' -> [(Abbreviation,[Ptr "VS163\0"#])] - '\xe0193' -> [(Abbreviation,[Ptr "VS164\0"#])] - '\xe0194' -> [(Abbreviation,[Ptr "VS165\0"#])] - '\xe0195' -> [(Abbreviation,[Ptr "VS166\0"#])] - '\xe0196' -> [(Abbreviation,[Ptr "VS167\0"#])] - '\xe0197' -> [(Abbreviation,[Ptr "VS168\0"#])] - '\xe0198' -> [(Abbreviation,[Ptr "VS169\0"#])] - '\xe0199' -> [(Abbreviation,[Ptr "VS170\0"#])] - '\xe019a' -> [(Abbreviation,[Ptr "VS171\0"#])] - '\xe019b' -> [(Abbreviation,[Ptr "VS172\0"#])] - '\xe019c' -> [(Abbreviation,[Ptr "VS173\0"#])] - '\xe019d' -> [(Abbreviation,[Ptr "VS174\0"#])] - '\xe019e' -> [(Abbreviation,[Ptr "VS175\0"#])] - '\xe019f' -> [(Abbreviation,[Ptr "VS176\0"#])] - '\xe01a0' -> [(Abbreviation,[Ptr "VS177\0"#])] - '\xe01a1' -> [(Abbreviation,[Ptr "VS178\0"#])] - '\xe01a2' -> [(Abbreviation,[Ptr "VS179\0"#])] - '\xe01a3' -> [(Abbreviation,[Ptr "VS180\0"#])] - '\xe01a4' -> [(Abbreviation,[Ptr "VS181\0"#])] - '\xe01a5' -> [(Abbreviation,[Ptr "VS182\0"#])] - '\xe01a6' -> [(Abbreviation,[Ptr "VS183\0"#])] - '\xe01a7' -> [(Abbreviation,[Ptr "VS184\0"#])] - '\xe01a8' -> [(Abbreviation,[Ptr "VS185\0"#])] - '\xe01a9' -> [(Abbreviation,[Ptr "VS186\0"#])] - '\xe01aa' -> [(Abbreviation,[Ptr "VS187\0"#])] - '\xe01ab' -> [(Abbreviation,[Ptr "VS188\0"#])] - '\xe01ac' -> [(Abbreviation,[Ptr "VS189\0"#])] - '\xe01ad' -> [(Abbreviation,[Ptr "VS190\0"#])] - '\xe01ae' -> [(Abbreviation,[Ptr "VS191\0"#])] - '\xe01af' -> [(Abbreviation,[Ptr "VS192\0"#])] - '\xe01b0' -> [(Abbreviation,[Ptr "VS193\0"#])] - '\xe01b1' -> [(Abbreviation,[Ptr "VS194\0"#])] - '\xe01b2' -> [(Abbreviation,[Ptr "VS195\0"#])] - '\xe01b3' -> [(Abbreviation,[Ptr "VS196\0"#])] - '\xe01b4' -> [(Abbreviation,[Ptr "VS197\0"#])] - '\xe01b5' -> [(Abbreviation,[Ptr "VS198\0"#])] - '\xe01b6' -> [(Abbreviation,[Ptr "VS199\0"#])] - '\xe01b7' -> [(Abbreviation,[Ptr "VS200\0"#])] - '\xe01b8' -> [(Abbreviation,[Ptr "VS201\0"#])] - '\xe01b9' -> [(Abbreviation,[Ptr "VS202\0"#])] - '\xe01ba' -> [(Abbreviation,[Ptr "VS203\0"#])] - '\xe01bb' -> [(Abbreviation,[Ptr "VS204\0"#])] - '\xe01bc' -> [(Abbreviation,[Ptr "VS205\0"#])] - '\xe01bd' -> [(Abbreviation,[Ptr "VS206\0"#])] - '\xe01be' -> [(Abbreviation,[Ptr "VS207\0"#])] - '\xe01bf' -> [(Abbreviation,[Ptr "VS208\0"#])] - '\xe01c0' -> [(Abbreviation,[Ptr "VS209\0"#])] - '\xe01c1' -> [(Abbreviation,[Ptr "VS210\0"#])] - '\xe01c2' -> [(Abbreviation,[Ptr "VS211\0"#])] - '\xe01c3' -> [(Abbreviation,[Ptr "VS212\0"#])] - '\xe01c4' -> [(Abbreviation,[Ptr "VS213\0"#])] - '\xe01c5' -> [(Abbreviation,[Ptr "VS214\0"#])] - '\xe01c6' -> [(Abbreviation,[Ptr "VS215\0"#])] - '\xe01c7' -> [(Abbreviation,[Ptr "VS216\0"#])] - '\xe01c8' -> [(Abbreviation,[Ptr "VS217\0"#])] - '\xe01c9' -> [(Abbreviation,[Ptr "VS218\0"#])] - '\xe01ca' -> [(Abbreviation,[Ptr "VS219\0"#])] - '\xe01cb' -> [(Abbreviation,[Ptr "VS220\0"#])] - '\xe01cc' -> [(Abbreviation,[Ptr "VS221\0"#])] - '\xe01cd' -> [(Abbreviation,[Ptr "VS222\0"#])] - '\xe01ce' -> [(Abbreviation,[Ptr "VS223\0"#])] - '\xe01cf' -> [(Abbreviation,[Ptr "VS224\0"#])] - '\xe01d0' -> [(Abbreviation,[Ptr "VS225\0"#])] - '\xe01d1' -> [(Abbreviation,[Ptr "VS226\0"#])] - '\xe01d2' -> [(Abbreviation,[Ptr "VS227\0"#])] - '\xe01d3' -> [(Abbreviation,[Ptr "VS228\0"#])] - '\xe01d4' -> [(Abbreviation,[Ptr "VS229\0"#])] - '\xe01d5' -> [(Abbreviation,[Ptr "VS230\0"#])] - '\xe01d6' -> [(Abbreviation,[Ptr "VS231\0"#])] - '\xe01d7' -> [(Abbreviation,[Ptr "VS232\0"#])] - '\xe01d8' -> [(Abbreviation,[Ptr "VS233\0"#])] - '\xe01d9' -> [(Abbreviation,[Ptr "VS234\0"#])] - '\xe01da' -> [(Abbreviation,[Ptr "VS235\0"#])] - '\xe01db' -> [(Abbreviation,[Ptr "VS236\0"#])] - '\xe01dc' -> [(Abbreviation,[Ptr "VS237\0"#])] - '\xe01dd' -> [(Abbreviation,[Ptr "VS238\0"#])] - '\xe01de' -> [(Abbreviation,[Ptr "VS239\0"#])] - '\xe01df' -> [(Abbreviation,[Ptr "VS240\0"#])] - '\xe01e0' -> [(Abbreviation,[Ptr "VS241\0"#])] - '\xe01e1' -> [(Abbreviation,[Ptr "VS242\0"#])] - '\xe01e2' -> [(Abbreviation,[Ptr "VS243\0"#])] - '\xe01e3' -> [(Abbreviation,[Ptr "VS244\0"#])] - '\xe01e4' -> [(Abbreviation,[Ptr "VS245\0"#])] - '\xe01e5' -> [(Abbreviation,[Ptr "VS246\0"#])] - '\xe01e6' -> [(Abbreviation,[Ptr "VS247\0"#])] - '\xe01e7' -> [(Abbreviation,[Ptr "VS248\0"#])] - '\xe01e8' -> [(Abbreviation,[Ptr "VS249\0"#])] - '\xe01e9' -> [(Abbreviation,[Ptr "VS250\0"#])] - '\xe01ea' -> [(Abbreviation,[Ptr "VS251\0"#])] - '\xe01eb' -> [(Abbreviation,[Ptr "VS252\0"#])] - '\xe01ec' -> [(Abbreviation,[Ptr "VS253\0"#])] - '\xe01ed' -> [(Abbreviation,[Ptr "VS254\0"#])] - '\xe01ee' -> [(Abbreviation,[Ptr "VS255\0"#])] - '\xe01ef' -> [(Abbreviation,[Ptr "VS256\0"#])] - - _ -> mempty +nameAliases :: Char# -> Addr# +nameAliases = \case + '\x0000'# -> "\0\5\0\0\11\4NULL\0\3NUL\0\0"# + '\x0001'# -> "\0\5\0\0\23\16START OF HEADING\0\3SOH\0\0"# + '\x0002'# -> "\0\5\0\0\20\13START OF TEXT\0\3STX\0\0"# + '\x0003'# -> "\0\5\0\0\18\11END OF TEXT\0\3ETX\0\0"# + '\x0004'# -> "\0\5\0\0\26\19END OF TRANSMISSION\0\3EOT\0\0"# + '\x0005'# -> "\0\5\0\0\14\7ENQUIRY\0\3ENQ\0\0"# + '\x0006'# -> "\0\5\0\0\18\11ACKNOWLEDGE\0\3ACK\0\0"# + '\x0007'# -> "\0\5\0\0\12\5ALERT\0\3BEL\0\0"# + '\x0008'# -> "\0\5\0\0\16\9BACKSPACE\0\2BS\0\0"# + '\x0009'# -> "\0\5\0\0\49\20CHARACTER TABULATION\21HORIZONTAL TABULATION\0\2HT\3TAB\0\0"# + '\x000A'# -> "\0\5\0\0\37\9LINE FEED\8NEW LINE\11END OF LINE\0\2LF\2NL\3EOL\0\0"# + '\x000B'# -> "\0\5\0\0\42\15LINE TABULATION\19VERTICAL TABULATION\0\2VT\0\0"# + '\x000C'# -> "\0\5\0\0\16\9FORM FEED\0\2FF\0\0"# + '\x000D'# -> "\0\5\0\0\22\15CARRIAGE RETURN\0\2CR\0\0"# + '\x000E'# -> "\0\5\0\0\34\9SHIFT OUT\17LOCKING-SHIFT ONE\0\2SO\0\0"# + '\x000F'# -> "\0\5\0\0\34\8SHIFT IN\18LOCKING-SHIFT ZERO\0\2SI\0\0"# + '\x0010'# -> "\0\5\0\0\23\16DATA LINK ESCAPE\0\3DLE\0\0"# + '\x0011'# -> "\0\5\0\0\25\18DEVICE CONTROL ONE\0\3DC1\0\0"# + '\x0012'# -> "\0\5\0\0\25\18DEVICE CONTROL TWO\0\3DC2\0\0"# + '\x0013'# -> "\0\5\0\0\27\20DEVICE CONTROL THREE\0\3DC3\0\0"# + '\x0014'# -> "\0\5\0\0\26\19DEVICE CONTROL FOUR\0\3DC4\0\0"# + '\x0015'# -> "\0\5\0\0\27\20NEGATIVE ACKNOWLEDGE\0\3NAK\0\0"# + '\x0016'# -> "\0\5\0\0\23\16SYNCHRONOUS IDLE\0\3SYN\0\0"# + '\x0017'# -> "\0\5\0\0\32\25END OF TRANSMISSION BLOCK\0\3ETB\0\0"# + '\x0018'# -> "\0\5\0\0\13\6CANCEL\0\3CAN\0\0"# + '\x0019'# -> "\0\5\0\0\20\13END OF MEDIUM\0\3EOM\2EM\0\0"# + '\x001A'# -> "\0\5\0\0\17\10SUBSTITUTE\0\3SUB\0\0"# + '\x001B'# -> "\0\5\0\0\13\6ESCAPE\0\3ESC\0\0"# + '\x001C'# -> "\0\5\0\0\48\26INFORMATION SEPARATOR FOUR\14FILE SEPARATOR\0\2FS\0\0"# + '\x001D'# -> "\0\5\0\0\50\27INFORMATION SEPARATOR THREE\15GROUP SEPARATOR\0\2GS\0\0"# + '\x001E'# -> "\0\5\0\0\49\25INFORMATION SEPARATOR TWO\16RECORD SEPARATOR\0\2RS\0\0"# + '\x001F'# -> "\0\5\0\0\47\25INFORMATION SEPARATOR ONE\14UNIT SEPARATOR\0\2US\0\0"# + '\x0020'# -> "\0\0\0\0\5\2SP\0\0"# + '\x007F'# -> "\0\5\0\0\13\6DELETE\0\3DEL\0\0"# + '\x0080'# -> "\0\0\0\5\24\17PADDING CHARACTER\0\3PAD\0\0"# + '\x0081'# -> "\0\0\0\5\24\17HIGH OCTET PRESET\0\3HOP\0\0"# + '\x0082'# -> "\0\5\0\0\27\20BREAK PERMITTED HERE\0\3BPH\0\0"# + '\x0083'# -> "\0\5\0\0\20\13NO BREAK HERE\0\3NBH\0\0"# + '\x0084'# -> "\0\5\0\0\12\5INDEX\0\3IND\0\0"# + '\x0085'# -> "\0\5\0\0\16\9NEXT LINE\0\3NEL\0\0"# + '\x0086'# -> "\0\5\0\0\29\22START OF SELECTED AREA\0\3SSA\0\0"# + '\x0087'# -> "\0\5\0\0\27\20END OF SELECTED AREA\0\3ESA\0\0"# + '\x0088'# -> "\0\5\0\0\57\24CHARACTER TABULATION SET\25HORIZONTAL TABULATION SET\0\3HTS\0\0"# + '\x0089'# -> "\0\5\0\0\87\39CHARACTER TABULATION WITH JUSTIFICATION\40HORIZONTAL TABULATION WITH JUSTIFICATION\0\3HTJ\0\0"# + '\x008A'# -> "\0\5\0\0\50\19LINE TABULATION SET\23VERTICAL TABULATION SET\0\3VTS\0\0"# + '\x008B'# -> "\0\5\0\0\45\20PARTIAL LINE FORWARD\17PARTIAL LINE DOWN\0\3PLD\0\0"# + '\x008C'# -> "\0\5\0\0\44\21PARTIAL LINE BACKWARD\15PARTIAL LINE UP\0\3PLU\0\0"# + '\x008D'# -> "\0\5\0\0\38\17REVERSE LINE FEED\13REVERSE INDEX\0\2RI\0\0"# + '\x008E'# -> "\0\5\0\0\38\16SINGLE SHIFT TWO\14SINGLE-SHIFT-2\0\3SS2\0\0"# + '\x008F'# -> "\0\5\0\0\40\18SINGLE SHIFT THREE\14SINGLE-SHIFT-3\0\3SS3\0\0"# + '\x0090'# -> "\0\5\0\0\28\21DEVICE CONTROL STRING\0\3DCS\0\0"# + '\x0091'# -> "\0\5\0\0\36\15PRIVATE USE ONE\13PRIVATE USE-1\0\3PU1\0\0"# + '\x0092'# -> "\0\5\0\0\36\15PRIVATE USE TWO\13PRIVATE USE-2\0\3PU2\0\0"# + '\x0093'# -> "\0\5\0\0\25\18SET TRANSMIT STATE\0\3STS\0\0"# + '\x0094'# -> "\0\5\0\0\23\16CANCEL CHARACTER\0\3CCH\0\0"# + '\x0095'# -> "\0\5\0\0\22\15MESSAGE WAITING\0\2MW\0\0"# + '\x0096'# -> "\0\5\0\0\52\21START OF GUARDED AREA\23START OF PROTECTED AREA\0\3SPA\0\0"# + '\x0097'# -> "\0\5\0\0\48\19END OF GUARDED AREA\21END OF PROTECTED AREA\0\3EPA\0\0"# + '\x0098'# -> "\0\5\0\0\22\15START OF STRING\0\3SOS\0\0"# + '\x0099'# -> "\0\0\0\5\42\35SINGLE GRAPHIC CHARACTER INTRODUCER\0\3SGC\0\0"# + '\x009A'# -> "\0\5\0\0\34\27SINGLE CHARACTER INTRODUCER\0\3SCI\0\0"# + '\x009B'# -> "\0\5\0\0\34\27CONTROL SEQUENCE INTRODUCER\0\3CSI\0\0"# + '\x009C'# -> "\0\5\0\0\24\17STRING TERMINATOR\0\2ST\0\0"# + '\x009D'# -> "\0\5\0\0\31\24OPERATING SYSTEM COMMAND\0\3OSC\0\0"# + '\x009E'# -> "\0\5\0\0\22\15PRIVACY MESSAGE\0\2PM\0\0"# + '\x009F'# -> "\0\5\0\0\34\27APPLICATION PROGRAM COMMAND\0\3APC\0\0"# + '\x00A0'# -> "\0\0\0\0\5\4NBSP\0\0"# + '\x00AD'# -> "\0\0\0\0\5\3SHY\0\0"# + '\x01A2'# -> "\5\0\0\0\0\24LATIN CAPITAL LETTER GHA\0\0"# + '\x01A3'# -> "\5\0\0\0\0\22LATIN SMALL LETTER GHA\0\0"# + '\x034F'# -> "\0\0\0\0\5\3CGJ\0\0"# + '\x0616'# -> "\5\0\0\0\0\47ARABIC SMALL HIGH LIGATURE ALEF WITH YEH BARREE\0\0"# + '\x061C'# -> "\0\0\0\0\5\3ALM\0\0"# + '\x0709'# -> "\5\0\0\0\0\34SYRIAC SUBLINEAR COLON SKEWED LEFT\0\0"# + '\x0CDE'# -> "\5\0\0\0\0\19KANNADA LETTER LLLA\0\0"# + '\x0E9D'# -> "\5\0\0\0\0\17LAO LETTER FO FON\0\0"# + '\x0E9F'# -> "\5\0\0\0\0\17LAO LETTER FO FAY\0\0"# + '\x0EA3'# -> "\5\0\0\0\0\13LAO LETTER RO\0\0"# + '\x0EA5'# -> "\5\0\0\0\0\13LAO LETTER LO\0\0"# + '\x0FD0'# -> "\5\0\0\0\0\35TIBETAN MARK BKA- SHOG GI MGO RGYAN\0\0"# + '\x11EC'# -> "\5\0\0\0\0\32HANGUL JONGSEONG YESIEUNG-KIYEOK\0\0"# + '\x11ED'# -> "\5\0\0\0\0\37HANGUL JONGSEONG YESIEUNG-SSANGKIYEOK\0\0"# + '\x11EE'# -> "\5\0\0\0\0\30HANGUL JONGSEONG SSANGYESIEUNG\0\0"# + '\x11EF'# -> "\5\0\0\0\0\33HANGUL JONGSEONG YESIEUNG-KHIEUKH\0\0"# + '\x180B'# -> "\0\0\0\0\5\4FVS1\0\0"# + '\x180C'# -> "\0\0\0\0\5\4FVS2\0\0"# + '\x180D'# -> "\0\0\0\0\5\4FVS3\0\0"# + '\x180E'# -> "\0\0\0\0\5\3MVS\0\0"# + '\x180F'# -> "\0\0\0\0\5\4FVS4\0\0"# + '\x1BBD'# -> "\5\0\0\0\0\26SUNDANESE LETTER ARCHAIC I\0\0"# + '\x200B'# -> "\0\0\0\0\5\4ZWSP\0\0"# + '\x200C'# -> "\0\0\0\0\5\4ZWNJ\0\0"# + '\x200D'# -> "\0\0\0\0\5\3ZWJ\0\0"# + '\x200E'# -> "\0\0\0\0\5\3LRM\0\0"# + '\x200F'# -> "\0\0\0\0\5\3RLM\0\0"# + '\x202A'# -> "\0\0\0\0\5\3LRE\0\0"# + '\x202B'# -> "\0\0\0\0\5\3RLE\0\0"# + '\x202C'# -> "\0\0\0\0\5\3PDF\0\0"# + '\x202D'# -> "\0\0\0\0\5\3LRO\0\0"# + '\x202E'# -> "\0\0\0\0\5\3RLO\0\0"# + '\x202F'# -> "\0\0\0\0\5\5NNBSP\0\0"# + '\x205F'# -> "\0\0\0\0\5\4MMSP\0\0"# + '\x2060'# -> "\0\0\0\0\5\2WJ\0\0"# + '\x2066'# -> "\0\0\0\0\5\3LRI\0\0"# + '\x2067'# -> "\0\0\0\0\5\3RLI\0\0"# + '\x2068'# -> "\0\0\0\0\5\3FSI\0\0"# + '\x2069'# -> "\0\0\0\0\5\3PDI\0\0"# + '\x2118'# -> "\5\0\0\0\0\29WEIERSTRASS ELLIPTIC FUNCTION\0\0"# + '\x2448'# -> "\5\0\0\0\0\17MICR ON US SYMBOL\0\0"# + '\x2449'# -> "\5\0\0\0\0\16MICR DASH SYMBOL\0\0"# + '\x2B7A'# -> "\5\0\0\0\0\59LEFTWARDS TRIANGLE-HEADED ARROW WITH DOUBLE VERTICAL STROKE\0\0"# + '\x2B7C'# -> "\5\0\0\0\0\60RIGHTWARDS TRIANGLE-HEADED ARROW WITH DOUBLE VERTICAL STROKE\0\0"# + '\xA015'# -> "\5\0\0\0\0\26YI SYLLABLE ITERATION MARK\0\0"# + '\xAA6E'# -> "\5\0\0\0\0\25MYANMAR LETTER KHAMTI LLA\0\0"# + '\xFE00'# -> "\0\0\0\0\5\3VS1\0\0"# + '\xFE01'# -> "\0\0\0\0\5\3VS2\0\0"# + '\xFE02'# -> "\0\0\0\0\5\3VS3\0\0"# + '\xFE03'# -> "\0\0\0\0\5\3VS4\0\0"# + '\xFE04'# -> "\0\0\0\0\5\3VS5\0\0"# + '\xFE05'# -> "\0\0\0\0\5\3VS6\0\0"# + '\xFE06'# -> "\0\0\0\0\5\3VS7\0\0"# + '\xFE07'# -> "\0\0\0\0\5\3VS8\0\0"# + '\xFE08'# -> "\0\0\0\0\5\3VS9\0\0"# + '\xFE09'# -> "\0\0\0\0\5\4VS10\0\0"# + '\xFE0A'# -> "\0\0\0\0\5\4VS11\0\0"# + '\xFE0B'# -> "\0\0\0\0\5\4VS12\0\0"# + '\xFE0C'# -> "\0\0\0\0\5\4VS13\0\0"# + '\xFE0D'# -> "\0\0\0\0\5\4VS14\0\0"# + '\xFE0E'# -> "\0\0\0\0\5\4VS15\0\0"# + '\xFE0F'# -> "\0\0\0\0\5\4VS16\0\0"# + '\xFE18'# -> "\5\0\0\0\0\61PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET\0\0"# + '\xFEFF'# -> "\0\0\5\0\22\15BYTE ORDER MARK\0\3BOM\6ZWNBSP\0\0"# + '\x122D4'# -> "\5\0\0\0\0\24CUNEIFORM SIGN NU11 TENU\0\0"# + '\x122D5'# -> "\5\0\0\0\0\42CUNEIFORM SIGN NU11 OVER NU11 BUR OVER BUR\0\0"# + '\x16E56'# -> "\5\0\0\0\0\28MEDEFAIDRIN CAPITAL LETTER H\0\0"# + '\x16E57'# -> "\5\0\0\0\0\29MEDEFAIDRIN CAPITAL LETTER NG\0\0"# + '\x16E76'# -> "\5\0\0\0\0\26MEDEFAIDRIN SMALL LETTER H\0\0"# + '\x16E77'# -> "\5\0\0\0\0\27MEDEFAIDRIN SMALL LETTER NG\0\0"# + '\x1B001'# -> "\5\0\0\0\0\21HENTAIGANA LETTER E-1\0\0"# + '\x1D0C5'# -> "\5\0\0\0\0\52BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS\0\0"# + '\xE0100'# -> "\0\0\0\0\5\4VS17\0\0"# + '\xE0101'# -> "\0\0\0\0\5\4VS18\0\0"# + '\xE0102'# -> "\0\0\0\0\5\4VS19\0\0"# + '\xE0103'# -> "\0\0\0\0\5\4VS20\0\0"# + '\xE0104'# -> "\0\0\0\0\5\4VS21\0\0"# + '\xE0105'# -> "\0\0\0\0\5\4VS22\0\0"# + '\xE0106'# -> "\0\0\0\0\5\4VS23\0\0"# + '\xE0107'# -> "\0\0\0\0\5\4VS24\0\0"# + '\xE0108'# -> "\0\0\0\0\5\4VS25\0\0"# + '\xE0109'# -> "\0\0\0\0\5\4VS26\0\0"# + '\xE010A'# -> "\0\0\0\0\5\4VS27\0\0"# + '\xE010B'# -> "\0\0\0\0\5\4VS28\0\0"# + '\xE010C'# -> "\0\0\0\0\5\4VS29\0\0"# + '\xE010D'# -> "\0\0\0\0\5\4VS30\0\0"# + '\xE010E'# -> "\0\0\0\0\5\4VS31\0\0"# + '\xE010F'# -> "\0\0\0\0\5\4VS32\0\0"# + '\xE0110'# -> "\0\0\0\0\5\4VS33\0\0"# + '\xE0111'# -> "\0\0\0\0\5\4VS34\0\0"# + '\xE0112'# -> "\0\0\0\0\5\4VS35\0\0"# + '\xE0113'# -> "\0\0\0\0\5\4VS36\0\0"# + '\xE0114'# -> "\0\0\0\0\5\4VS37\0\0"# + '\xE0115'# -> "\0\0\0\0\5\4VS38\0\0"# + '\xE0116'# -> "\0\0\0\0\5\4VS39\0\0"# + '\xE0117'# -> "\0\0\0\0\5\4VS40\0\0"# + '\xE0118'# -> "\0\0\0\0\5\4VS41\0\0"# + '\xE0119'# -> "\0\0\0\0\5\4VS42\0\0"# + '\xE011A'# -> "\0\0\0\0\5\4VS43\0\0"# + '\xE011B'# -> "\0\0\0\0\5\4VS44\0\0"# + '\xE011C'# -> "\0\0\0\0\5\4VS45\0\0"# + '\xE011D'# -> "\0\0\0\0\5\4VS46\0\0"# + '\xE011E'# -> "\0\0\0\0\5\4VS47\0\0"# + '\xE011F'# -> "\0\0\0\0\5\4VS48\0\0"# + '\xE0120'# -> "\0\0\0\0\5\4VS49\0\0"# + '\xE0121'# -> "\0\0\0\0\5\4VS50\0\0"# + '\xE0122'# -> "\0\0\0\0\5\4VS51\0\0"# + '\xE0123'# -> "\0\0\0\0\5\4VS52\0\0"# + '\xE0124'# -> "\0\0\0\0\5\4VS53\0\0"# + '\xE0125'# -> "\0\0\0\0\5\4VS54\0\0"# + '\xE0126'# -> "\0\0\0\0\5\4VS55\0\0"# + '\xE0127'# -> "\0\0\0\0\5\4VS56\0\0"# + '\xE0128'# -> "\0\0\0\0\5\4VS57\0\0"# + '\xE0129'# -> "\0\0\0\0\5\4VS58\0\0"# + '\xE012A'# -> "\0\0\0\0\5\4VS59\0\0"# + '\xE012B'# -> "\0\0\0\0\5\4VS60\0\0"# + '\xE012C'# -> "\0\0\0\0\5\4VS61\0\0"# + '\xE012D'# -> "\0\0\0\0\5\4VS62\0\0"# + '\xE012E'# -> "\0\0\0\0\5\4VS63\0\0"# + '\xE012F'# -> "\0\0\0\0\5\4VS64\0\0"# + '\xE0130'# -> "\0\0\0\0\5\4VS65\0\0"# + '\xE0131'# -> "\0\0\0\0\5\4VS66\0\0"# + '\xE0132'# -> "\0\0\0\0\5\4VS67\0\0"# + '\xE0133'# -> "\0\0\0\0\5\4VS68\0\0"# + '\xE0134'# -> "\0\0\0\0\5\4VS69\0\0"# + '\xE0135'# -> "\0\0\0\0\5\4VS70\0\0"# + '\xE0136'# -> "\0\0\0\0\5\4VS71\0\0"# + '\xE0137'# -> "\0\0\0\0\5\4VS72\0\0"# + '\xE0138'# -> "\0\0\0\0\5\4VS73\0\0"# + '\xE0139'# -> "\0\0\0\0\5\4VS74\0\0"# + '\xE013A'# -> "\0\0\0\0\5\4VS75\0\0"# + '\xE013B'# -> "\0\0\0\0\5\4VS76\0\0"# + '\xE013C'# -> "\0\0\0\0\5\4VS77\0\0"# + '\xE013D'# -> "\0\0\0\0\5\4VS78\0\0"# + '\xE013E'# -> "\0\0\0\0\5\4VS79\0\0"# + '\xE013F'# -> "\0\0\0\0\5\4VS80\0\0"# + '\xE0140'# -> "\0\0\0\0\5\4VS81\0\0"# + '\xE0141'# -> "\0\0\0\0\5\4VS82\0\0"# + '\xE0142'# -> "\0\0\0\0\5\4VS83\0\0"# + '\xE0143'# -> "\0\0\0\0\5\4VS84\0\0"# + '\xE0144'# -> "\0\0\0\0\5\4VS85\0\0"# + '\xE0145'# -> "\0\0\0\0\5\4VS86\0\0"# + '\xE0146'# -> "\0\0\0\0\5\4VS87\0\0"# + '\xE0147'# -> "\0\0\0\0\5\4VS88\0\0"# + '\xE0148'# -> "\0\0\0\0\5\4VS89\0\0"# + '\xE0149'# -> "\0\0\0\0\5\4VS90\0\0"# + '\xE014A'# -> "\0\0\0\0\5\4VS91\0\0"# + '\xE014B'# -> "\0\0\0\0\5\4VS92\0\0"# + '\xE014C'# -> "\0\0\0\0\5\4VS93\0\0"# + '\xE014D'# -> "\0\0\0\0\5\4VS94\0\0"# + '\xE014E'# -> "\0\0\0\0\5\4VS95\0\0"# + '\xE014F'# -> "\0\0\0\0\5\4VS96\0\0"# + '\xE0150'# -> "\0\0\0\0\5\4VS97\0\0"# + '\xE0151'# -> "\0\0\0\0\5\4VS98\0\0"# + '\xE0152'# -> "\0\0\0\0\5\4VS99\0\0"# + '\xE0153'# -> "\0\0\0\0\5\5VS100\0\0"# + '\xE0154'# -> "\0\0\0\0\5\5VS101\0\0"# + '\xE0155'# -> "\0\0\0\0\5\5VS102\0\0"# + '\xE0156'# -> "\0\0\0\0\5\5VS103\0\0"# + '\xE0157'# -> "\0\0\0\0\5\5VS104\0\0"# + '\xE0158'# -> "\0\0\0\0\5\5VS105\0\0"# + '\xE0159'# -> "\0\0\0\0\5\5VS106\0\0"# + '\xE015A'# -> "\0\0\0\0\5\5VS107\0\0"# + '\xE015B'# -> "\0\0\0\0\5\5VS108\0\0"# + '\xE015C'# -> "\0\0\0\0\5\5VS109\0\0"# + '\xE015D'# -> "\0\0\0\0\5\5VS110\0\0"# + '\xE015E'# -> "\0\0\0\0\5\5VS111\0\0"# + '\xE015F'# -> "\0\0\0\0\5\5VS112\0\0"# + '\xE0160'# -> "\0\0\0\0\5\5VS113\0\0"# + '\xE0161'# -> "\0\0\0\0\5\5VS114\0\0"# + '\xE0162'# -> "\0\0\0\0\5\5VS115\0\0"# + '\xE0163'# -> "\0\0\0\0\5\5VS116\0\0"# + '\xE0164'# -> "\0\0\0\0\5\5VS117\0\0"# + '\xE0165'# -> "\0\0\0\0\5\5VS118\0\0"# + '\xE0166'# -> "\0\0\0\0\5\5VS119\0\0"# + '\xE0167'# -> "\0\0\0\0\5\5VS120\0\0"# + '\xE0168'# -> "\0\0\0\0\5\5VS121\0\0"# + '\xE0169'# -> "\0\0\0\0\5\5VS122\0\0"# + '\xE016A'# -> "\0\0\0\0\5\5VS123\0\0"# + '\xE016B'# -> "\0\0\0\0\5\5VS124\0\0"# + '\xE016C'# -> "\0\0\0\0\5\5VS125\0\0"# + '\xE016D'# -> "\0\0\0\0\5\5VS126\0\0"# + '\xE016E'# -> "\0\0\0\0\5\5VS127\0\0"# + '\xE016F'# -> "\0\0\0\0\5\5VS128\0\0"# + '\xE0170'# -> "\0\0\0\0\5\5VS129\0\0"# + '\xE0171'# -> "\0\0\0\0\5\5VS130\0\0"# + '\xE0172'# -> "\0\0\0\0\5\5VS131\0\0"# + '\xE0173'# -> "\0\0\0\0\5\5VS132\0\0"# + '\xE0174'# -> "\0\0\0\0\5\5VS133\0\0"# + '\xE0175'# -> "\0\0\0\0\5\5VS134\0\0"# + '\xE0176'# -> "\0\0\0\0\5\5VS135\0\0"# + '\xE0177'# -> "\0\0\0\0\5\5VS136\0\0"# + '\xE0178'# -> "\0\0\0\0\5\5VS137\0\0"# + '\xE0179'# -> "\0\0\0\0\5\5VS138\0\0"# + '\xE017A'# -> "\0\0\0\0\5\5VS139\0\0"# + '\xE017B'# -> "\0\0\0\0\5\5VS140\0\0"# + '\xE017C'# -> "\0\0\0\0\5\5VS141\0\0"# + '\xE017D'# -> "\0\0\0\0\5\5VS142\0\0"# + '\xE017E'# -> "\0\0\0\0\5\5VS143\0\0"# + '\xE017F'# -> "\0\0\0\0\5\5VS144\0\0"# + '\xE0180'# -> "\0\0\0\0\5\5VS145\0\0"# + '\xE0181'# -> "\0\0\0\0\5\5VS146\0\0"# + '\xE0182'# -> "\0\0\0\0\5\5VS147\0\0"# + '\xE0183'# -> "\0\0\0\0\5\5VS148\0\0"# + '\xE0184'# -> "\0\0\0\0\5\5VS149\0\0"# + '\xE0185'# -> "\0\0\0\0\5\5VS150\0\0"# + '\xE0186'# -> "\0\0\0\0\5\5VS151\0\0"# + '\xE0187'# -> "\0\0\0\0\5\5VS152\0\0"# + '\xE0188'# -> "\0\0\0\0\5\5VS153\0\0"# + '\xE0189'# -> "\0\0\0\0\5\5VS154\0\0"# + '\xE018A'# -> "\0\0\0\0\5\5VS155\0\0"# + '\xE018B'# -> "\0\0\0\0\5\5VS156\0\0"# + '\xE018C'# -> "\0\0\0\0\5\5VS157\0\0"# + '\xE018D'# -> "\0\0\0\0\5\5VS158\0\0"# + '\xE018E'# -> "\0\0\0\0\5\5VS159\0\0"# + '\xE018F'# -> "\0\0\0\0\5\5VS160\0\0"# + '\xE0190'# -> "\0\0\0\0\5\5VS161\0\0"# + '\xE0191'# -> "\0\0\0\0\5\5VS162\0\0"# + '\xE0192'# -> "\0\0\0\0\5\5VS163\0\0"# + '\xE0193'# -> "\0\0\0\0\5\5VS164\0\0"# + '\xE0194'# -> "\0\0\0\0\5\5VS165\0\0"# + '\xE0195'# -> "\0\0\0\0\5\5VS166\0\0"# + '\xE0196'# -> "\0\0\0\0\5\5VS167\0\0"# + '\xE0197'# -> "\0\0\0\0\5\5VS168\0\0"# + '\xE0198'# -> "\0\0\0\0\5\5VS169\0\0"# + '\xE0199'# -> "\0\0\0\0\5\5VS170\0\0"# + '\xE019A'# -> "\0\0\0\0\5\5VS171\0\0"# + '\xE019B'# -> "\0\0\0\0\5\5VS172\0\0"# + '\xE019C'# -> "\0\0\0\0\5\5VS173\0\0"# + '\xE019D'# -> "\0\0\0\0\5\5VS174\0\0"# + '\xE019E'# -> "\0\0\0\0\5\5VS175\0\0"# + '\xE019F'# -> "\0\0\0\0\5\5VS176\0\0"# + '\xE01A0'# -> "\0\0\0\0\5\5VS177\0\0"# + '\xE01A1'# -> "\0\0\0\0\5\5VS178\0\0"# + '\xE01A2'# -> "\0\0\0\0\5\5VS179\0\0"# + '\xE01A3'# -> "\0\0\0\0\5\5VS180\0\0"# + '\xE01A4'# -> "\0\0\0\0\5\5VS181\0\0"# + '\xE01A5'# -> "\0\0\0\0\5\5VS182\0\0"# + '\xE01A6'# -> "\0\0\0\0\5\5VS183\0\0"# + '\xE01A7'# -> "\0\0\0\0\5\5VS184\0\0"# + '\xE01A8'# -> "\0\0\0\0\5\5VS185\0\0"# + '\xE01A9'# -> "\0\0\0\0\5\5VS186\0\0"# + '\xE01AA'# -> "\0\0\0\0\5\5VS187\0\0"# + '\xE01AB'# -> "\0\0\0\0\5\5VS188\0\0"# + '\xE01AC'# -> "\0\0\0\0\5\5VS189\0\0"# + '\xE01AD'# -> "\0\0\0\0\5\5VS190\0\0"# + '\xE01AE'# -> "\0\0\0\0\5\5VS191\0\0"# + '\xE01AF'# -> "\0\0\0\0\5\5VS192\0\0"# + '\xE01B0'# -> "\0\0\0\0\5\5VS193\0\0"# + '\xE01B1'# -> "\0\0\0\0\5\5VS194\0\0"# + '\xE01B2'# -> "\0\0\0\0\5\5VS195\0\0"# + '\xE01B3'# -> "\0\0\0\0\5\5VS196\0\0"# + '\xE01B4'# -> "\0\0\0\0\5\5VS197\0\0"# + '\xE01B5'# -> "\0\0\0\0\5\5VS198\0\0"# + '\xE01B6'# -> "\0\0\0\0\5\5VS199\0\0"# + '\xE01B7'# -> "\0\0\0\0\5\5VS200\0\0"# + '\xE01B8'# -> "\0\0\0\0\5\5VS201\0\0"# + '\xE01B9'# -> "\0\0\0\0\5\5VS202\0\0"# + '\xE01BA'# -> "\0\0\0\0\5\5VS203\0\0"# + '\xE01BB'# -> "\0\0\0\0\5\5VS204\0\0"# + '\xE01BC'# -> "\0\0\0\0\5\5VS205\0\0"# + '\xE01BD'# -> "\0\0\0\0\5\5VS206\0\0"# + '\xE01BE'# -> "\0\0\0\0\5\5VS207\0\0"# + '\xE01BF'# -> "\0\0\0\0\5\5VS208\0\0"# + '\xE01C0'# -> "\0\0\0\0\5\5VS209\0\0"# + '\xE01C1'# -> "\0\0\0\0\5\5VS210\0\0"# + '\xE01C2'# -> "\0\0\0\0\5\5VS211\0\0"# + '\xE01C3'# -> "\0\0\0\0\5\5VS212\0\0"# + '\xE01C4'# -> "\0\0\0\0\5\5VS213\0\0"# + '\xE01C5'# -> "\0\0\0\0\5\5VS214\0\0"# + '\xE01C6'# -> "\0\0\0\0\5\5VS215\0\0"# + '\xE01C7'# -> "\0\0\0\0\5\5VS216\0\0"# + '\xE01C8'# -> "\0\0\0\0\5\5VS217\0\0"# + '\xE01C9'# -> "\0\0\0\0\5\5VS218\0\0"# + '\xE01CA'# -> "\0\0\0\0\5\5VS219\0\0"# + '\xE01CB'# -> "\0\0\0\0\5\5VS220\0\0"# + '\xE01CC'# -> "\0\0\0\0\5\5VS221\0\0"# + '\xE01CD'# -> "\0\0\0\0\5\5VS222\0\0"# + '\xE01CE'# -> "\0\0\0\0\5\5VS223\0\0"# + '\xE01CF'# -> "\0\0\0\0\5\5VS224\0\0"# + '\xE01D0'# -> "\0\0\0\0\5\5VS225\0\0"# + '\xE01D1'# -> "\0\0\0\0\5\5VS226\0\0"# + '\xE01D2'# -> "\0\0\0\0\5\5VS227\0\0"# + '\xE01D3'# -> "\0\0\0\0\5\5VS228\0\0"# + '\xE01D4'# -> "\0\0\0\0\5\5VS229\0\0"# + '\xE01D5'# -> "\0\0\0\0\5\5VS230\0\0"# + '\xE01D6'# -> "\0\0\0\0\5\5VS231\0\0"# + '\xE01D7'# -> "\0\0\0\0\5\5VS232\0\0"# + '\xE01D8'# -> "\0\0\0\0\5\5VS233\0\0"# + '\xE01D9'# -> "\0\0\0\0\5\5VS234\0\0"# + '\xE01DA'# -> "\0\0\0\0\5\5VS235\0\0"# + '\xE01DB'# -> "\0\0\0\0\5\5VS236\0\0"# + '\xE01DC'# -> "\0\0\0\0\5\5VS237\0\0"# + '\xE01DD'# -> "\0\0\0\0\5\5VS238\0\0"# + '\xE01DE'# -> "\0\0\0\0\5\5VS239\0\0"# + '\xE01DF'# -> "\0\0\0\0\5\5VS240\0\0"# + '\xE01E0'# -> "\0\0\0\0\5\5VS241\0\0"# + '\xE01E1'# -> "\0\0\0\0\5\5VS242\0\0"# + '\xE01E2'# -> "\0\0\0\0\5\5VS243\0\0"# + '\xE01E3'# -> "\0\0\0\0\5\5VS244\0\0"# + '\xE01E4'# -> "\0\0\0\0\5\5VS245\0\0"# + '\xE01E5'# -> "\0\0\0\0\5\5VS246\0\0"# + '\xE01E6'# -> "\0\0\0\0\5\5VS247\0\0"# + '\xE01E7'# -> "\0\0\0\0\5\5VS248\0\0"# + '\xE01E8'# -> "\0\0\0\0\5\5VS249\0\0"# + '\xE01E9'# -> "\0\0\0\0\5\5VS250\0\0"# + '\xE01EA'# -> "\0\0\0\0\5\5VS251\0\0"# + '\xE01EB'# -> "\0\0\0\0\5\5VS252\0\0"# + '\xE01EC'# -> "\0\0\0\0\5\5VS253\0\0"# + '\xE01ED'# -> "\0\0\0\0\5\5VS254\0\0"# + '\xE01EE'# -> "\0\0\0\0\5\5VS255\0\0"# + '\xE01EF'# -> "\0\0\0\0\5\5VS256\0\0"# + _ -> "\xff"# diff --git a/test/ICU/NamesSpec.hs b/test/ICU/NamesSpec.hs new file mode 100644 --- /dev/null +++ b/test/ICU/NamesSpec.hs @@ -0,0 +1,110 @@ +{-# LANGUAGE CPP, BlockArguments #-} + +module ICU.NamesSpec + ( spec + ) where + +import Control.Applicative (Alternative(..)) +import Data.Foldable (traverse_) +import Data.Version (showVersion, versionBranch) +import Numeric (showHex) +import Test.Hspec + ( describe + , expectationFailure + , it + , pendingWith + , Spec + , HasCallStack, SpecWith ) + +import qualified ICU.Char as ICU +import qualified ICU.Names as ICUString +import qualified Unicode.Char as U +import qualified Unicode.Char.General.Names as String +#ifdef HAS_BYTESTRING +import qualified Data.ByteString.Char8 as B8 +import qualified Unicode.Char.General.Names.ByteString as ByteString +#endif +#ifdef HAS_TEXT +import qualified Unicode.Char.General.Names.Text as Text +import qualified ICU.Names.Text as ICUText +#endif + +spec :: Spec +spec = do + describe "name" do + checkAndGatherErrors "String" String.name ICUString.name +#ifdef HAS_BYTESTRING + checkAndGatherErrors "ByteString" + ByteString.name + (fmap B8.pack . ICUString.name) +#endif +#ifdef HAS_TEXT + checkAndGatherErrors "Text" Text.name ICUText.name +#endif + describe "correctedName" do + checkAndGatherErrors "String" + String.correctedName + ICUString.correctedName +#ifdef HAS_BYTESTRING + checkAndGatherErrors "ByteString" + ByteString.correctedName + (fmap B8.pack . ICUString.correctedName) +#endif +#ifdef HAS_TEXT + checkAndGatherErrors "Text" + Text.correctedName + ICUText.correctedName +#endif + where + ourUnicodeVersion = versionBranch U.unicodeVersion + showCodePoint c = ("U+" ++) . fmap U.toUpper . showHex (U.ord c) + + -- There is no feature to display warnings other than `trace`, so + -- hack our own: + -- 1. Compare given functions in pure code and gather warning & errors + -- 2. Create dummy spec that throw an expectation failure, if relevant. + -- 3. Create pending spec for each Char that raises a Unicode version + -- mismatch between ICU and unicode-data. + checkAndGatherErrors + :: forall a. (HasCallStack, Eq a, Show a) + => String + -> (Char -> Maybe a) + -> (Char -> Maybe a) + -> SpecWith () + checkAndGatherErrors label f fRef = do + it label (maybe (pure ()) expectationFailure err) + if null ws + then pure () + else describe (label ++ " (Unicode version conflict)") + (traverse_ mkWarning ws) + where + Acc ws err = foldr check (Acc [] Nothing) [minBound..maxBound] + check c acc + -- Test passed + | n == nRef = acc + -- Unicode version mismatch: char is not mapped in one of the libs: + -- add warning. + | ageMismatch c = acc{warnings=c : warnings acc} + -- Error + | otherwise = + let !msg = mconcat + [ showCodePoint c ": expected " + , maybe "\"\"" show nRef + , ", got ", maybe "\"\"" show n, "" ] + in acc{firstError = firstError acc <|> Just msg} + where + !n = f c + !nRef = fRef c + mkWarning c = it (showCodePoint c "") . pendingWith $ mconcat + [ "Incompatible ICU Unicode version: expected " + , showVersion U.unicodeVersion + , ", got: " + , showVersion ICU.unicodeVersion + , " (ICU character age is: " + , showVersion (ICU.charAge c) + , ")" ] + ageMismatch c = + let age = take 3 (versionBranch (ICU.charAge c)) + in age > ourUnicodeVersion || age == [0, 0, 0] + +data Acc = Acc { warnings :: ![Char], firstError :: !(Maybe String) } diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -1,1 +1,31 @@ -{-# OPTIONS_GHC -F -pgmF hspec-discover #-} +{-# LANGUAGE CPP #-} + +module Main where + +import Test.Hspec ( Spec, hspec, describe ) +import qualified Unicode.Char.General.NamesSpec as String +#ifdef HAS_TEXT +import qualified Unicode.Char.General.Names.TextSpec as Text +#endif +#ifdef HAS_BYTESTRING +import qualified Unicode.Char.General.Names.ByteStringSpec as ByteString +#endif +#ifdef HAS_ICU +import qualified ICU.NamesSpec as ICU +#endif + +main :: IO () +main = hspec spec + +spec :: Spec +spec = do + describe "Unicode.Char.General.Names" String.spec +#ifdef HAS_BYTESTRING + describe "Unicode.Char.General.Names.ByteString" ByteString.spec +#endif +#ifdef HAS_TEXT + describe "Unicode.Char.General.Names.Text" Text.spec +#endif +#ifdef HAS_ICU + describe "ICU.Names" ICU.spec +#endif diff --git a/test/Unicode/Char/General/Names/ByteStringSpec.hs b/test/Unicode/Char/General/Names/ByteStringSpec.hs new file mode 100644 --- /dev/null +++ b/test/Unicode/Char/General/Names/ByteStringSpec.hs @@ -0,0 +1,51 @@ +{-# LANGUAGE CPP, BlockArguments #-} + +module Unicode.Char.General.Names.ByteStringSpec + ( spec + ) where + +import Data.Foldable (traverse_) +import qualified Data.ByteString.Char8 as BS8 +import qualified Unicode.Char.General.Names as String +import qualified Unicode.Char.General.Names.ByteString as BS +import Test.Hspec ( Spec, it, shouldBe, Expectation ) + +spec :: Spec +spec = do + it "name" do + traverse_ ((fmap BS8.unpack . BS.name) `shouldBeEqualTo` String.name) + [minBound..maxBound] + it "nameOrAlias" do + traverse_ + ((fmap BS8.unpack . BS.nameOrAlias) + `shouldBeEqualTo` String.nameOrAlias) + [minBound..maxBound] + it "correctedName" do + traverse_ + ((fmap BS8.unpack . BS.correctedName) + `shouldBeEqualTo` String.correctedName) + [minBound..maxBound] + it "nameAliases" do + traverse_ + ((fmap BS8.unpack . BS.nameAliases) + `shouldBeEqualTo` String.nameAliases) + [minBound..maxBound] + it "nameAliasesByType" do + let check ty = traverse_ + ((fmap BS8.unpack . BS.nameAliasesByType ty) + `shouldBeEqualTo` String.nameAliasesByType ty) + [minBound..maxBound] + traverse_ check [minBound..maxBound] + it "nameAliasesWithTypes" do + traverse_ + ((fmap (fmap (fmap BS8.unpack)) . BS.nameAliasesWithTypes) + `shouldBeEqualTo` String.nameAliasesWithTypes) + [minBound..maxBound] + where + shouldBeEqualTo + :: forall a. (Eq a, Show a) + => (Char -> a) + -> (Char -> a) + -> Char + -> Expectation + shouldBeEqualTo f fRef c = f c `shouldBe` fRef c diff --git a/test/Unicode/Char/General/Names/TextSpec.hs b/test/Unicode/Char/General/Names/TextSpec.hs new file mode 100644 --- /dev/null +++ b/test/Unicode/Char/General/Names/TextSpec.hs @@ -0,0 +1,51 @@ +{-# LANGUAGE CPP, BlockArguments #-} + +module Unicode.Char.General.Names.TextSpec + ( spec + ) where + +import Data.Foldable (traverse_) +import qualified Data.Text as T +import qualified Unicode.Char.General.Names as String +import qualified Unicode.Char.General.Names.Text as Text +import Test.Hspec ( Spec, it, shouldBe, Expectation ) + +spec :: Spec +spec = do + it "name" do + traverse_ ((fmap T.unpack . Text.name) `shouldBeEqualTo` String.name) + [minBound..maxBound] + it "nameOrAlias" do + traverse_ + ((fmap T.unpack . Text.nameOrAlias) + `shouldBeEqualTo` String.nameOrAlias) + [minBound..maxBound] + it "correctedName" do + traverse_ + ((fmap T.unpack . Text.correctedName) + `shouldBeEqualTo` String.correctedName) + [minBound..maxBound] + it "nameAliases" do + traverse_ + ((fmap T.unpack . Text.nameAliases) + `shouldBeEqualTo` String.nameAliases) + [minBound..maxBound] + it "nameAliasesByType" do + let check ty = traverse_ + ((fmap T.unpack . Text.nameAliasesByType ty) + `shouldBeEqualTo` String.nameAliasesByType ty) + [minBound..maxBound] + traverse_ check [minBound..maxBound] + it "nameAliasesWithTypes" do + traverse_ + ((fmap (fmap (fmap T.unpack)) . Text.nameAliasesWithTypes) + `shouldBeEqualTo` String.nameAliasesWithTypes) + [minBound..maxBound] + where + shouldBeEqualTo + :: forall a. (Eq a, Show a) + => (Char -> a) + -> (Char -> a) + -> Char + -> Expectation + shouldBeEqualTo f fRef c = f c `shouldBe` fRef c diff --git a/test/Unicode/Char/General/NamesSpec.hs b/test/Unicode/Char/General/NamesSpec.hs --- a/test/Unicode/Char/General/NamesSpec.hs +++ b/test/Unicode/Char/General/NamesSpec.hs @@ -4,13 +4,27 @@ ( spec ) where +import GHC.Exts (Char(..), isTrue#, (<#), ord#, andI#) import Unicode.Char.General + ( generalCategory, + GeneralCategory(NotAssigned, Surrogate, PrivateUse) ) import Unicode.Char.General.Names + ( NameAliasType (..), correctedName, name, nameOrAlias, nameAliasesWithTypes, nameAliases, nameAliasesByType ) +import qualified Unicode.Internal.Char.UnicodeData.DerivedName as DerivedName import Data.Foldable (traverse_) -import Test.Hspec +import Test.Hspec ( Spec, it, shouldBe, shouldSatisfy, describe ) spec :: Spec spec = do + it "template hex code length is 4 or 5" + -- Ensure no padding is required for hexadecimal codepoints and + -- that we the allocation is correct for ByteString & Text APIs. + let { + check (C# c#) = case DerivedName.name c# of + (# _, len# #) -> + isTrue# (len# <# DerivedName.CjkCompatibilityIdeograph) || + isTrue# ((0xFFF# <# ord# c#) `andI#` (ord# c# <# 0x100000#)) + } in traverse_ (`shouldSatisfy` check) [minBound..maxBound] it "name: Test some characters" do name minBound `shouldBe` Nothing name 'A' `shouldBe` Just "LATIN CAPITAL LETTER A" @@ -71,6 +85,44 @@ -- Last name defined, as of Unicode 15.0.0 nameOrAlias '\xe01ef' `shouldBe` Just "VARIATION SELECTOR-256" nameOrAlias maxBound `shouldBe` Nothing + it "nameAliasesWithTypes: test some characters" do + nameAliasesWithTypes '\0' `shouldBe` + [(Control, ["NULL"]), (Abbreviation, ["NUL"])] + nameAliasesWithTypes '\x0A' `shouldBe` + [(Control, ["LINE FEED", "NEW LINE", "END OF LINE"]) + ,(Abbreviation, ["LF", "NL", "EOL"])] + nameAliasesWithTypes '\x80' `shouldBe` + [(Figment, ["PADDING CHARACTER"]), (Abbreviation, ["PAD"])] + nameAliasesWithTypes '\x01A2' `shouldBe` + [(Correction, ["LATIN CAPITAL LETTER GHA"])] + nameAliasesWithTypes '\xFEFF' `shouldBe` + [(Alternate, ["BYTE ORDER MARK"]), (Abbreviation, ["BOM", "ZWNBSP"])] + nameAliasesWithTypes '\xE01EF' `shouldBe` + [(Abbreviation, ["VS256"])] + it "nameAliasesByType" do + let f c = foldr + (\t -> case nameAliasesByType t c of {[] -> id;xs -> ((t,xs):)}) + mempty + [minBound..maxBound] + check c = f c == nameAliasesWithTypes c + traverse_ (`shouldSatisfy` check) [minBound..maxBound] + describe "nameAliases" do + it "test some characters" do + nameAliases '\0' `shouldBe` + ["NULL", "NUL"] + nameAliases '\x0A' `shouldBe` + ["LINE FEED", "NEW LINE", "END OF LINE", "LF", "NL", "EOL"] + nameAliases '\x80' `shouldBe` + ["PADDING CHARACTER", "PAD"] + nameAliases '\x01A2' `shouldBe` + ["LATIN CAPITAL LETTER GHA"] + nameAliases '\xFEFF' `shouldBe` + ["BYTE ORDER MARK", "BOM", "ZWNBSP"] + nameAliases '\xE01EF' `shouldBe` + ["VS256"] + it "compare to nameAliasesWithTypes" do + let check c = nameAliases c == mconcat (snd <$> nameAliasesWithTypes c) + traverse_ (`shouldSatisfy` check) [minBound..maxBound] it "Every defined character has at least a name or an alias" do let checkName c = case nameOrAlias c of Just _ -> True @@ -79,5 +131,4 @@ PrivateUse -> True NotAssigned -> True _ -> False - traverse_ (`shouldSatisfy` checkName) - [minBound..maxBound] + traverse_ (`shouldSatisfy` checkName) [minBound..maxBound] diff --git a/test/export_all_chars.hs b/test/export_all_chars.hs --- a/test/export_all_chars.hs +++ b/test/export_all_chars.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE BlockArguments #-} - ------------------------------------------------------------------------------- -- | -- Description: Export all characters and their properties to a CSV file. @@ -9,11 +7,11 @@ module Main where -import Data.Char -import Data.Foldable -import Data.Version +import Data.Char ( ord ) +import Data.Foldable ( traverse_ ) +import Data.Version ( showVersion ) import Unicode.Char (unicodeVersion) -import Numeric +import Numeric ( showHex ) import qualified Unicode.Char.General.Names as UNames @@ -39,4 +37,4 @@ addEntry c = do putStr (mkCodePointHex c) putChar ',' - putStrLn (show (UNames.name c)) + print (UNames.name c) diff --git a/unicode-data-names.cabal b/unicode-data-names.cabal --- a/unicode-data-names.cabal +++ b/unicode-data-names.cabal @@ -1,6 +1,6 @@ cabal-version: 2.2 name: unicode-data-names -version: 0.2.0 +version: 0.3.0 synopsis: Unicode characters names and aliases description: @unicode-data-names@ provides Haskell APIs to access the Unicode @@ -26,11 +26,14 @@ , GHC==8.6.5 , GHC==8.8.4 , GHC==8.10.7 - , GHC==9.0.1 - , GHC==9.2.1 - , GHC==9.4.2 + , GHC==9.0.2 + , GHC==9.2.8 + , GHC==9.4.8 + , GHC==9.6.5 + , GHC==9.8.2 + , GHC==9.10.1 -extra-source-files: +extra-doc-files: Changelog.md README.md @@ -38,6 +41,22 @@ type: git location: https://github.com/composewell/unicode-data +flag has-text + description: Expose an API using the text package + manual: True + default: False + +flag has-bytestring + description: Expose an API using the bytestring package + manual: True + default: False + +flag dev-has-icu + description: + Use ICU for test and benchmark. Intended for development on the repository. + manual: True + default: False + flag export-all-chars description: Build the export-all-chars executable manual: True @@ -55,8 +74,6 @@ LambdaCase -- Experimental, may lead to issues - DeriveAnyClass - TemplateHaskell UnboxedTuples common compile-options @@ -69,7 +86,6 @@ library import: default-extensions, compile-options - default-language: Haskell2010 exposed-modules: -- The module structure is derived from -- https://www.unicode.org/reports/tr44/#Property_Index_Table @@ -80,10 +96,28 @@ -- the properties are generated. Unicode.Internal.Char.UnicodeData.DerivedName Unicode.Internal.Char.UnicodeData.NameAliases + Unicode.Internal.Char.Names.Version + other-modules: + -- Internal files + Unicode.Internal.Bits.Names hs-source-dirs: lib build-depends: - base >= 4.7 && < 4.18 + base >= 4.7 && < 4.21, + -- Support for raw string literals unpacking is included in base ≥ 4.15 + if impl(ghc < 9.0.0) + build-depends: + ghc-prim >= 0.3.1 && < 1.0 + if flag(has-text) + exposed-modules: + Unicode.Char.General.Names.Text + build-depends: + text >= 2.0 && < 2.2 + if flag(has-bytestring) + exposed-modules: + Unicode.Char.General.Names.ByteString + build-depends: + bytestring >= 0.11.2 && < 0.13 test-suite test import: default-extensions, compile-options @@ -94,17 +128,31 @@ other-modules: Unicode.Char.General.NamesSpec build-depends: - base >= 4.7 && < 4.18 - , hspec >= 2.0 && < 2.11 - , unicode-data >= 0.4 && < 0.5 + base >= 4.7 && < 4.21 + , hspec >= 2.0 && < 2.12 + , unicode-data >= 0.5 && < 0.6 , unicode-data-names - build-tool-depends: - hspec-discover:hspec-discover >= 2.0 && < 2.11 - default-language: Haskell2010 + if flag(has-text) + cpp-options: -DHAS_TEXT + other-modules: + Unicode.Char.General.Names.TextSpec + build-depends: + text >= 2.0 && < 2.2 + if flag(has-bytestring) + cpp-options: -DHAS_BYTESTRING + other-modules: + Unicode.Char.General.Names.ByteStringSpec + build-depends: + bytestring >= 0.11.2 && < 0.13 + if flag(dev-has-icu) + cpp-options: -DHAS_ICU + other-modules: + ICU.NamesSpec + build-depends: + icu executable export-all-chars import: default-extensions, compile-options - default-language: Haskell2010 ghc-options: -O2 hs-source-dirs: test main-is: export_all_chars.hs @@ -123,9 +171,26 @@ hs-source-dirs: bench main-is: Main.hs build-depends: - base >= 4.7 && < 4.18, - deepseq >= 1.1 && < 1.5, - tasty-bench >= 0.2.5 && < 0.4, - tasty >= 1.4.1, + base >= 4.7 && < 4.21, + deepseq >= 1.1 && < 1.6, + tasty-bench >= 0.2.5 && < 0.4, + tasty >= 1.4.1 && < 1.6, + unicode-data >= 0.5 && < 0.6, unicode-data-names - ghc-options: -O2 -fdicts-strict -rtsopts + if flag(has-text) + cpp-options: -DHAS_TEXT + build-depends: + text >= 2.0 && < 2.2 + if flag(has-bytestring) + cpp-options: -DHAS_BYTESTRING + build-depends: + bytestring >= 0.11.2 && < 0.13 + if flag(dev-has-icu) + cpp-options: -DHAS_ICU + build-depends: + icu + -- [NOTE] Recommendation of tasty-bench to reduce garbage collection noisiness + ghc-options: -O2 -fdicts-strict -rtsopts -with-rtsopts=-A32m + -- [NOTE] Recommendation of tasty-bench for comparison against baseline + if impl(ghc >= 8.6) + ghc-options: -fproc-alignment=64