diff --git a/.git/COMMIT_EDITMSG b/.git/COMMIT_EDITMSG deleted file mode 100644 --- a/.git/COMMIT_EDITMSG +++ /dev/null @@ -1,14 +0,0 @@ - -# Please enter the commit message for your changes. Lines starting -# with '#' will be ignored, and an empty message aborts the commit. -# On branch master -# Changes not staged for commit: -# (use "git add ..." to update what will be committed) -# (use "git checkout -- ..." to discard changes in working directory) -# -# modified: README -# -# Untracked files: -# (use "git add ..." to include in what will be committed) -# -# dist/ diff --git a/.git/HEAD b/.git/HEAD deleted file mode 100644 --- a/.git/HEAD +++ /dev/null @@ -1,1 +0,0 @@ -ref: refs/heads/master diff --git a/.git/config b/.git/config deleted file mode 100644 --- a/.git/config +++ /dev/null @@ -1,11 +0,0 @@ -[core] - repositoryformatversion = 0 - filemode = true - bare = false - logallrefupdates = true -[remote "origin"] - url = git@github.com:jonkri/idna2008.git - fetch = +refs/heads/*:refs/remotes/origin/* -[branch "master"] - remote = origin - merge = refs/heads/master diff --git a/.git/description b/.git/description deleted file mode 100644 --- a/.git/description +++ /dev/null @@ -1,1 +0,0 @@ -Unnamed repository; edit this file 'description' to name the repository. diff --git a/.git/hooks/applypatch-msg.sample b/.git/hooks/applypatch-msg.sample deleted file mode 100644 --- a/.git/hooks/applypatch-msg.sample +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# An example hook script to check the commit log message taken by -# applypatch from an e-mail message. -# -# The hook should exit with non-zero status after issuing an -# appropriate message if it wants to stop the commit. The hook is -# allowed to edit the commit message file. -# -# To enable this hook, rename this file to "applypatch-msg". - -. git-sh-setup -test -x "$GIT_DIR/hooks/commit-msg" && - exec "$GIT_DIR/hooks/commit-msg" ${1+"$@"} -: diff --git a/.git/hooks/commit-msg.sample b/.git/hooks/commit-msg.sample deleted file mode 100644 --- a/.git/hooks/commit-msg.sample +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# -# An example hook script to check the commit log message. -# Called by "git commit" with one argument, the name of the file -# that has the commit message. The hook should exit with non-zero -# status after issuing an appropriate message if it wants to stop the -# commit. The hook is allowed to edit the commit message file. -# -# To enable this hook, rename this file to "commit-msg". - -# Uncomment the below to add a Signed-off-by line to the message. -# Doing this in a hook is a bad idea in general, but the prepare-commit-msg -# hook is more suited to it. -# -# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') -# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" - -# This example catches duplicate Signed-off-by lines. - -test "" = "$(grep '^Signed-off-by: ' "$1" | - sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { - echo >&2 Duplicate Signed-off-by lines. - exit 1 -} diff --git a/.git/hooks/post-commit.sample b/.git/hooks/post-commit.sample deleted file mode 100644 --- a/.git/hooks/post-commit.sample +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# An example hook script that is called after a successful -# commit is made. -# -# To enable this hook, rename this file to "post-commit". - -: Nothing diff --git a/.git/hooks/post-receive.sample b/.git/hooks/post-receive.sample deleted file mode 100644 --- a/.git/hooks/post-receive.sample +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# An example hook script for the "post-receive" event. -# -# The "post-receive" script is run after receive-pack has accepted a pack -# and the repository has been updated. It is passed arguments in through -# stdin in the form -# -# For example: -# aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master -# -# see contrib/hooks/ for a sample, or uncomment the next line and -# rename the file to "post-receive". - -#. /usr/share/git-core/contrib/hooks/post-receive-email diff --git a/.git/hooks/post-update.sample b/.git/hooks/post-update.sample deleted file mode 100644 --- a/.git/hooks/post-update.sample +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# An example hook script to prepare a packed repository for use over -# dumb transports. -# -# To enable this hook, rename this file to "post-update". - -exec git update-server-info diff --git a/.git/hooks/pre-applypatch.sample b/.git/hooks/pre-applypatch.sample deleted file mode 100644 --- a/.git/hooks/pre-applypatch.sample +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# -# An example hook script to verify what is about to be committed -# by applypatch from an e-mail message. -# -# The hook should exit with non-zero status after issuing an -# appropriate message if it wants to stop the commit. -# -# To enable this hook, rename this file to "pre-applypatch". - -. git-sh-setup -test -x "$GIT_DIR/hooks/pre-commit" && - exec "$GIT_DIR/hooks/pre-commit" ${1+"$@"} -: diff --git a/.git/hooks/pre-commit.sample b/.git/hooks/pre-commit.sample deleted file mode 100644 --- a/.git/hooks/pre-commit.sample +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# An example hook script to verify what is about to be committed. -# Called by "git commit" with no arguments. The hook should -# exit with non-zero status after issuing an appropriate message if -# it wants to stop the commit. -# -# To enable this hook, rename this file to "pre-commit". - -if git rev-parse --verify HEAD >/dev/null 2>&1 -then - against=HEAD -else - # Initial commit: diff against an empty tree object - against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 -fi - -# If you want to allow non-ascii filenames set this variable to true. -allownonascii=$(git config hooks.allownonascii) - -# Cross platform projects tend to avoid non-ascii filenames; prevent -# them from being added to the repository. We exploit the fact that the -# printable range starts at the space character and ends with tilde. -if [ "$allownonascii" != "true" ] && - # Note that the use of brackets around a tr range is ok here, (it's - # even required, for portability to Solaris 10's /usr/bin/tr), since - # the square bracket bytes happen to fall in the designated range. - test "$(git diff --cached --name-only --diff-filter=A -z $against | - LC_ALL=C tr -d '[ -~]\0')" -then - echo "Error: Attempt to add a non-ascii file name." - echo - echo "This can cause problems if you want to work" - echo "with people on other platforms." - echo - echo "To be portable it is advisable to rename the file ..." - echo - echo "If you know what you are doing you can disable this" - echo "check using:" - echo - echo " git config hooks.allownonascii true" - echo - exit 1 -fi - -exec git diff-index --check --cached $against -- diff --git a/.git/hooks/pre-rebase.sample b/.git/hooks/pre-rebase.sample deleted file mode 100644 --- a/.git/hooks/pre-rebase.sample +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2006, 2008 Junio C Hamano -# -# The "pre-rebase" hook is run just before "git rebase" starts doing -# its job, and can prevent the command from running by exiting with -# non-zero status. -# -# The hook is called with the following parameters: -# -# $1 -- the upstream the series was forked from. -# $2 -- the branch being rebased (or empty when rebasing the current branch). -# -# This sample shows how to prevent topic branches that are already -# merged to 'next' branch from getting rebased, because allowing it -# would result in rebasing already published history. - -publish=next -basebranch="$1" -if test "$#" = 2 -then - topic="refs/heads/$2" -else - topic=`git symbolic-ref HEAD` || - exit 0 ;# we do not interrupt rebasing detached HEAD -fi - -case "$topic" in -refs/heads/??/*) - ;; -*) - exit 0 ;# we do not interrupt others. - ;; -esac - -# Now we are dealing with a topic branch being rebased -# on top of master. Is it OK to rebase it? - -# Does the topic really exist? -git show-ref -q "$topic" || { - echo >&2 "No such branch $topic" - exit 1 -} - -# Is topic fully merged to master? -not_in_master=`git rev-list --pretty=oneline ^master "$topic"` -if test -z "$not_in_master" -then - echo >&2 "$topic is fully merged to master; better remove it." - exit 1 ;# we could allow it, but there is no point. -fi - -# Is topic ever merged to next? If so you should not be rebasing it. -only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` -only_next_2=`git rev-list ^master ${publish} | sort` -if test "$only_next_1" = "$only_next_2" -then - not_in_topic=`git rev-list "^$topic" master` - if test -z "$not_in_topic" - then - echo >&2 "$topic is already up-to-date with master" - exit 1 ;# we could allow it, but there is no point. - else - exit 0 - fi -else - not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` - /usr/bin/perl -e ' - my $topic = $ARGV[0]; - my $msg = "* $topic has commits already merged to public branch:\n"; - my (%not_in_next) = map { - /^([0-9a-f]+) /; - ($1 => 1); - } split(/\n/, $ARGV[1]); - for my $elem (map { - /^([0-9a-f]+) (.*)$/; - [$1 => $2]; - } split(/\n/, $ARGV[2])) { - if (!exists $not_in_next{$elem->[0]}) { - if ($msg) { - print STDERR $msg; - undef $msg; - } - print STDERR " $elem->[1]\n"; - } - } - ' "$topic" "$not_in_next" "$not_in_master" - exit 1 -fi - -exit 0 - -################################################################ - -This sample hook safeguards topic branches that have been -published from being rewound. - -The workflow assumed here is: - - * Once a topic branch forks from "master", "master" is never - merged into it again (either directly or indirectly). - - * Once a topic branch is fully cooked and merged into "master", - it is deleted. If you need to build on top of it to correct - earlier mistakes, a new topic branch is created by forking at - the tip of the "master". This is not strictly necessary, but - it makes it easier to keep your history simple. - - * Whenever you need to test or publish your changes to topic - branches, merge them into "next" branch. - -The script, being an example, hardcodes the publish branch name -to be "next", but it is trivial to make it configurable via -$GIT_DIR/config mechanism. - -With this workflow, you would want to know: - -(1) ... if a topic branch has ever been merged to "next". Young - topic branches can have stupid mistakes you would rather - clean up before publishing, and things that have not been - merged into other branches can be easily rebased without - affecting other people. But once it is published, you would - not want to rewind it. - -(2) ... if a topic branch has been fully merged to "master". - Then you can delete it. More importantly, you should not - build on top of it -- other people may already want to - change things related to the topic as patches against your - "master", so if you need further changes, it is better to - fork the topic (perhaps with the same name) afresh from the - tip of "master". - -Let's look at this example: - - o---o---o---o---o---o---o---o---o---o "next" - / / / / - / a---a---b A / / - / / / / - / / c---c---c---c B / - / / / \ / - / / / b---b C \ / - / / / / \ / - ---o---o---o---o---o---o---o---o---o---o---o "master" - - -A, B and C are topic branches. - - * A has one fix since it was merged up to "next". - - * B has finished. It has been fully merged up to "master" and "next", - and is ready to be deleted. - - * C has not merged to "next" at all. - -We would want to allow C to be rebased, refuse A, and encourage -B to be deleted. - -To compute (1): - - git rev-list ^master ^topic next - git rev-list ^master next - - if these match, topic has not merged in next at all. - -To compute (2): - - git rev-list master..topic - - if this is empty, it is fully merged to "master". diff --git a/.git/hooks/prepare-commit-msg.sample b/.git/hooks/prepare-commit-msg.sample deleted file mode 100644 --- a/.git/hooks/prepare-commit-msg.sample +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# An example hook script to prepare the commit log message. -# Called by "git commit" with the name of the file that has the -# commit message, followed by the description of the commit -# message's source. The hook's purpose is to edit the commit -# message file. If the hook fails with a non-zero status, -# the commit is aborted. -# -# To enable this hook, rename this file to "prepare-commit-msg". - -# This hook includes three examples. The first comments out the -# "Conflicts:" part of a merge commit. -# -# The second includes the output of "git diff --name-status -r" -# into the message, just before the "git status" output. It is -# commented because it doesn't cope with --amend or with squashed -# commits. -# -# The third example adds a Signed-off-by line to the message, that can -# still be edited. This is rarely a good idea. - -case "$2,$3" in - merge,) - /usr/bin/perl -i.bak -ne 's/^/# /, s/^# #/#/ if /^Conflicts/ .. /#/; print' "$1" ;; - -# ,|template,) -# /usr/bin/perl -i.bak -pe ' -# print "\n" . `git diff --cached --name-status -r` -# if /^#/ && $first++ == 0' "$1" ;; - - *) ;; -esac - -# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') -# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" diff --git a/.git/hooks/update.sample b/.git/hooks/update.sample deleted file mode 100644 --- a/.git/hooks/update.sample +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/sh -# -# An example hook script to blocks unannotated tags from entering. -# Called by "git receive-pack" with arguments: refname sha1-old sha1-new -# -# To enable this hook, rename this file to "update". -# -# Config -# ------ -# hooks.allowunannotated -# This boolean sets whether unannotated tags will be allowed into the -# repository. By default they won't be. -# hooks.allowdeletetag -# This boolean sets whether deleting tags will be allowed in the -# repository. By default they won't be. -# hooks.allowmodifytag -# This boolean sets whether a tag may be modified after creation. By default -# it won't be. -# hooks.allowdeletebranch -# This boolean sets whether deleting branches will be allowed in the -# repository. By default they won't be. -# hooks.denycreatebranch -# This boolean sets whether remotely creating branches will be denied -# in the repository. By default this is allowed. -# - -# --- Command line -refname="$1" -oldrev="$2" -newrev="$3" - -# --- Safety check -if [ -z "$GIT_DIR" ]; then - echo "Don't run this script from the command line." >&2 - echo " (if you want, you could supply GIT_DIR then run" >&2 - echo " $0 )" >&2 - exit 1 -fi - -if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -# --- Config -allowunannotated=$(git config --bool hooks.allowunannotated) -allowdeletebranch=$(git config --bool hooks.allowdeletebranch) -denycreatebranch=$(git config --bool hooks.denycreatebranch) -allowdeletetag=$(git config --bool hooks.allowdeletetag) -allowmodifytag=$(git config --bool hooks.allowmodifytag) - -# check for no description -projectdesc=$(sed -e '1q' "$GIT_DIR/description") -case "$projectdesc" in -"Unnamed repository"* | "") - echo "*** Project description file hasn't been set" >&2 - exit 1 - ;; -esac - -# --- Check types -# if $newrev is 0000...0000, it's a commit to delete a ref. -zero="0000000000000000000000000000000000000000" -if [ "$newrev" = "$zero" ]; then - newrev_type=delete -else - newrev_type=$(git cat-file -t $newrev) -fi - -case "$refname","$newrev_type" in - refs/tags/*,commit) - # un-annotated tag - short_refname=${refname##refs/tags/} - if [ "$allowunannotated" != "true" ]; then - echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2 - echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 - exit 1 - fi - ;; - refs/tags/*,delete) - # delete tag - if [ "$allowdeletetag" != "true" ]; then - echo "*** Deleting a tag is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/tags/*,tag) - # annotated tag - if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 - then - echo "*** Tag '$refname' already exists." >&2 - echo "*** Modifying a tag is not allowed in this repository." >&2 - exit 1 - fi - ;; - refs/heads/*,commit) - # branch - if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then - echo "*** Creating a branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/heads/*,delete) - # delete branch - if [ "$allowdeletebranch" != "true" ]; then - echo "*** Deleting a branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/remotes/*,commit) - # tracking branch - ;; - refs/remotes/*,delete) - # delete tracking branch - if [ "$allowdeletebranch" != "true" ]; then - echo "*** Deleting a tracking branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - *) - # Anything else (is there anything else?) - echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 - exit 1 - ;; -esac - -# --- Finished -exit 0 diff --git a/.git/index b/.git/index deleted file mode 100644 Binary files a/.git/index and /dev/null differ diff --git a/.git/info/exclude b/.git/info/exclude deleted file mode 100644 --- a/.git/info/exclude +++ /dev/null @@ -1,6 +0,0 @@ -# git ls-files --others --exclude-from=.git/info/exclude -# Lines that start with '#' are comments. -# For a project mostly in C, the following would be a good set of -# exclude patterns (uncomment them if you want to use them): -# *.[oa] -# *~ diff --git a/.git/logs/HEAD b/.git/logs/HEAD deleted file mode 100644 --- a/.git/logs/HEAD +++ /dev/null @@ -1,6 +0,0 @@ -0000000000000000000000000000000000000000 1fe35cf0356895918ad9ee7fe7447b3abfd86f84 Jon Kristensen 1309716388 +0200 commit (initial): initial commit with README file -1fe35cf0356895918ad9ee7fe7447b3abfd86f84 13f70729e52090d1156c3e1f62ae66be8542797b Jon Kristensen 1309716447 +0200 commit: Added LICENSE file -13f70729e52090d1156c3e1f62ae66be8542797b a244896a47106062e620c8234f79559c9774189f Jon Kristensen 1309716517 +0200 commit: added initial Cabal file -a244896a47106062e620c8234f79559c9774189f 4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 Jon Kristensen 1309716588 +0200 commit: Added the Setup.hs file -4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 d65c0dfc81c16740cddba08b247dd2e16c310f89 Jon Kristensen 1309716881 +0200 commit: corrected a small error in the Cabal file and created an empty IDNA2008 module -d65c0dfc81c16740cddba08b247dd2e16c310f89 e0d70a0343af358ab7f47f9c85967ea0c5928db2 Jon Kristensen 1309862355 +0200 commit: used punycode to convert a hostname to ASCII; updated the Cabal and README files diff --git a/.git/logs/refs/heads/master b/.git/logs/refs/heads/master deleted file mode 100644 --- a/.git/logs/refs/heads/master +++ /dev/null @@ -1,6 +0,0 @@ -0000000000000000000000000000000000000000 1fe35cf0356895918ad9ee7fe7447b3abfd86f84 Jon Kristensen 1309716388 +0200 commit (initial): initial commit with README file -1fe35cf0356895918ad9ee7fe7447b3abfd86f84 13f70729e52090d1156c3e1f62ae66be8542797b Jon Kristensen 1309716447 +0200 commit: Added LICENSE file -13f70729e52090d1156c3e1f62ae66be8542797b a244896a47106062e620c8234f79559c9774189f Jon Kristensen 1309716517 +0200 commit: added initial Cabal file -a244896a47106062e620c8234f79559c9774189f 4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 Jon Kristensen 1309716588 +0200 commit: Added the Setup.hs file -4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 d65c0dfc81c16740cddba08b247dd2e16c310f89 Jon Kristensen 1309716881 +0200 commit: corrected a small error in the Cabal file and created an empty IDNA2008 module -d65c0dfc81c16740cddba08b247dd2e16c310f89 e0d70a0343af358ab7f47f9c85967ea0c5928db2 Jon Kristensen 1309862355 +0200 commit: used punycode to convert a hostname to ASCII; updated the Cabal and README files diff --git a/.git/logs/refs/remotes/origin/master b/.git/logs/refs/remotes/origin/master deleted file mode 100644 --- a/.git/logs/refs/remotes/origin/master +++ /dev/null @@ -1,6 +0,0 @@ -0000000000000000000000000000000000000000 1fe35cf0356895918ad9ee7fe7447b3abfd86f84 Jon Kristensen 1309716418 +0200 update by push -1fe35cf0356895918ad9ee7fe7447b3abfd86f84 13f70729e52090d1156c3e1f62ae66be8542797b Jon Kristensen 1309716462 +0200 update by push -13f70729e52090d1156c3e1f62ae66be8542797b a244896a47106062e620c8234f79559c9774189f Jon Kristensen 1309716533 +0200 update by push -a244896a47106062e620c8234f79559c9774189f 4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 Jon Kristensen 1309716609 +0200 update by push -4d1bb5d7839e1cb9d68d5737ade99f37034e81e0 d65c0dfc81c16740cddba08b247dd2e16c310f89 Jon Kristensen 1309716914 +0200 update by push -d65c0dfc81c16740cddba08b247dd2e16c310f89 e0d70a0343af358ab7f47f9c85967ea0c5928db2 Jon Kristensen 1309862404 +0200 update by push diff --git a/.git/objects/08/7c8ea3ecc2c24f4d5a5eb1b0c0f811717aed82 b/.git/objects/08/7c8ea3ecc2c24f4d5a5eb1b0c0f811717aed82 deleted file mode 100644 Binary files a/.git/objects/08/7c8ea3ecc2c24f4d5a5eb1b0c0f811717aed82 and /dev/null differ diff --git a/.git/objects/0b/1c9eb06af44e8f2bc06cf9363243a59bb3ac01 b/.git/objects/0b/1c9eb06af44e8f2bc06cf9363243a59bb3ac01 deleted file mode 100644 Binary files a/.git/objects/0b/1c9eb06af44e8f2bc06cf9363243a59bb3ac01 and /dev/null differ diff --git a/.git/objects/13/f70729e52090d1156c3e1f62ae66be8542797b b/.git/objects/13/f70729e52090d1156c3e1f62ae66be8542797b deleted file mode 100644 Binary files a/.git/objects/13/f70729e52090d1156c3e1f62ae66be8542797b and /dev/null differ diff --git a/.git/objects/17/f026fe6ed4a33b4c9edc2907c963632b9b3291 b/.git/objects/17/f026fe6ed4a33b4c9edc2907c963632b9b3291 deleted file mode 100644 Binary files a/.git/objects/17/f026fe6ed4a33b4c9edc2907c963632b9b3291 and /dev/null differ diff --git a/.git/objects/1f/e35cf0356895918ad9ee7fe7447b3abfd86f84 b/.git/objects/1f/e35cf0356895918ad9ee7fe7447b3abfd86f84 deleted file mode 100644 Binary files a/.git/objects/1f/e35cf0356895918ad9ee7fe7447b3abfd86f84 and /dev/null differ diff --git a/.git/objects/1f/fea57dd526f34d90842a8c8887b2e9bf759bc9 b/.git/objects/1f/fea57dd526f34d90842a8c8887b2e9bf759bc9 deleted file mode 100644 --- a/.git/objects/1f/fea57dd526f34d90842a8c8887b2e9bf759bc9 +++ /dev/null @@ -1,3 +0,0 @@ -x]ÍNÃ0„9ç)æ˜HõG¨´·@"åœxÛX¸»‘í¶*oÄkðd¸á‚8­4š™o5•ÓÙìjÓv¢,d‹ÐGå?ÈZXÓ8åΈRøCߋ h…ä¼ö1‘¤†9V! -ʚOÒ´ì•a°Ú“GCáDıšPÔeU!-ÊUÅïlZÑâx ï|cªmÅiÅ-ádB‡j¹.&£ÑÒ·‡·óùxê?Ë_æz`NŠ¾·¦ÞòH/Ù¯N‚´b³( vz 8hjˆS; àô%m,6)«¶*ò¡~£/+—”äªFH69ÃݙᦥNa>›_¾™Ná5rŸÎ›àŸÉ Š̈́5ÚÌȚ¤V ø 2Ku€B[R¢B "Ë´ñ{¤a³¾»¾˜Í.ùàJ×­‘‡‚¸è°€ß¿àb6ŸÃ­VðÑHK¨,*>°•ÿÞï×oÝ&ïï„Tä^4úªUò2{WkÙØD›“÷$RYJj¹L£¬[–ÈÀ½6G¨ö „/ì>+éüáà\c4J°yQʟ˜'ðP x’´5Æ ¢²…ÚhÒÔ֘s=¡rכЙ˓(Š*.¬‰äPÒç1éëýj³9‡ïŒ"Y±¶@ݺt ölÆâù œìkgbË?ŸÔ˜°mŠ?]Ý6–Æ ×qߨ֛‰‘oÖ5ˆX¿Ÿˆ•VßÐÐÿrÑÎåi>àÝ :óAX,ÜÝžé•»RVÖG -Áä3Üi*˜÷„%VO”å[‘bi£0 áKªž=-÷‡oºqõ¬.é¾Ré ³¾Ç èËP«ƒ–ÐÅ gÉP4à#i¾ÂßƎuƼ%T¢î”AhqR²|ŒíߦcéÆQe‚`®#ŒŠ—:ñúûïÃè’üûHM diff --git a/.git/objects/4d/1bb5d7839e1cb9d68d5737ade99f37034e81e0 b/.git/objects/4d/1bb5d7839e1cb9d68d5737ade99f37034e81e0 deleted file mode 100644 --- a/.git/objects/4d/1bb5d7839e1cb9d68d5737ade99f37034e81e0 +++ /dev/null @@ -1,1 +0,0 @@ -x¥Ž;nÃ0]óÛ0–ËïA´NéÐä’m‘‚Dß?ªr´ó“û²ÌÈáil" ‰ V2ŒxgÃْN&èT}²Æ:ok¨՚6iYù˜‚FžÄæHæ°Ø9΂Ց«Jï1õ ®½ÁÏ6ïCÚ. >æVû×£·ç»ä¾|‚6ÈA{#œ‘ÕAÐ!ÿ¸PߥH1 Üd¼×Ë´C_¢~m˜O1 diff --git a/.git/objects/61/2cb26a52838e721b27ec57b2f8e251c34145b6 b/.git/objects/61/2cb26a52838e721b27ec57b2f8e251c34145b6 deleted file mode 100644 Binary files a/.git/objects/61/2cb26a52838e721b27ec57b2f8e251c34145b6 and /dev/null differ diff --git a/.git/objects/a2/44896a47106062e620c8234f79559c9774189f b/.git/objects/a2/44896a47106062e620c8234f79559c9774189f deleted file mode 100644 Binary files a/.git/objects/a2/44896a47106062e620c8234f79559c9774189f and /dev/null differ diff --git a/.git/objects/b9/20b58f391f218787f3c4e276f8c9dbeeec4d59 b/.git/objects/b9/20b58f391f218787f3c4e276f8c9dbeeec4d59 deleted file mode 100644 Binary files a/.git/objects/b9/20b58f391f218787f3c4e276f8c9dbeeec4d59 and /dev/null differ diff --git a/.git/objects/bb/2167f3693b784171911db91be5099894e0daed b/.git/objects/bb/2167f3693b784171911db91be5099894e0daed deleted file mode 100644 Binary files a/.git/objects/bb/2167f3693b784171911db91be5099894e0daed and /dev/null differ diff --git a/.git/objects/bf/2616e3691a3ae13936a558027ae66150c654c8 b/.git/objects/bf/2616e3691a3ae13936a558027ae66150c654c8 deleted file mode 100644 Binary files a/.git/objects/bf/2616e3691a3ae13936a558027ae66150c654c8 and /dev/null differ diff --git a/.git/objects/d3/3c109a63e86ff954aad9e27de53c8d71c64fa7 b/.git/objects/d3/3c109a63e86ff954aad9e27de53c8d71c64fa7 deleted file mode 100644 Binary files a/.git/objects/d3/3c109a63e86ff954aad9e27de53c8d71c64fa7 and /dev/null differ diff --git a/.git/objects/d4/f7d77b2b112f86977cd04fc44e1ddae5b527f5 b/.git/objects/d4/f7d77b2b112f86977cd04fc44e1ddae5b527f5 deleted file mode 100644 Binary files a/.git/objects/d4/f7d77b2b112f86977cd04fc44e1ddae5b527f5 and /dev/null differ diff --git a/.git/objects/d6/5c0dfc81c16740cddba08b247dd2e16c310f89 b/.git/objects/d6/5c0dfc81c16740cddba08b247dd2e16c310f89 deleted file mode 100644 --- a/.git/objects/d6/5c0dfc81c16740cddba08b247dd2e16c310f89 +++ /dev/null @@ -1,2 +0,0 @@ -x¥ŽMJ1]÷)Þ^¤Ó™¤ADэ -Þá%ùÂDó3¤3 oopáÜEQ¾•’)¡oFHXã-XÁûÕ¯[܂f 'ð"Z)4Œ`×åÂuФs:«vHïöp²Ae8`ߣ2Bm°báë8·No­Ò{OÇ@=Pé>ÕØ?[ýúcw¾•’JìFž¬•t+V!–IçèÀ?³Ñ;ü@ ¦£p΄ÞçTª4Πgvœ)¦ âÈwð¯[ å2¾éõåãi®X*-\3–,dc diff --git a/.git/objects/d8/540eac9493ff33c9b29ff5e5c4483cee9acfde b/.git/objects/d8/540eac9493ff33c9b29ff5e5c4483cee9acfde deleted file mode 100644 Binary files a/.git/objects/d8/540eac9493ff33c9b29ff5e5c4483cee9acfde and /dev/null differ diff --git a/.git/objects/e0/d70a0343af358ab7f47f9c85967ea0c5928db2 b/.git/objects/e0/d70a0343af358ab7f47f9c85967ea0c5928db2 deleted file mode 100644 Binary files a/.git/objects/e0/d70a0343af358ab7f47f9c85967ea0c5928db2 and /dev/null differ diff --git a/.git/objects/e3/70998db2456dd1d04228be616ac53341c986d3 b/.git/objects/e3/70998db2456dd1d04228be616ac53341c986d3 deleted file mode 100644 Binary files a/.git/objects/e3/70998db2456dd1d04228be616ac53341c986d3 and /dev/null differ diff --git a/.git/objects/e8/ef27dbba9992f80d9271a60892aadc63c9ef36 b/.git/objects/e8/ef27dbba9992f80d9271a60892aadc63c9ef36 deleted file mode 100644 Binary files a/.git/objects/e8/ef27dbba9992f80d9271a60892aadc63c9ef36 and /dev/null differ diff --git a/.git/objects/ea/920d0f23900b939c421a371af6a434564f7fd0 b/.git/objects/ea/920d0f23900b939c421a371af6a434564f7fd0 deleted file mode 100644 Binary files a/.git/objects/ea/920d0f23900b939c421a371af6a434564f7fd0 and /dev/null differ diff --git a/.git/objects/f8/949e076d1ad396f7dda4c662ef11333a8ad98f b/.git/objects/f8/949e076d1ad396f7dda4c662ef11333a8ad98f deleted file mode 100644 Binary files a/.git/objects/f8/949e076d1ad396f7dda4c662ef11333a8ad98f and /dev/null differ diff --git a/.git/refs/heads/master b/.git/refs/heads/master deleted file mode 100644 --- a/.git/refs/heads/master +++ /dev/null @@ -1,1 +0,0 @@ -e0d70a0343af358ab7f47f9c85967ea0c5928db2 diff --git a/.git/refs/remotes/origin/master b/.git/refs/remotes/origin/master deleted file mode 100644 --- a/.git/refs/remotes/origin/master +++ /dev/null @@ -1,1 +0,0 @@ -e0d70a0343af358ab7f47f9c85967ea0c5928db2 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## 1.0.0.0 + +First public release of `idna2008`. See `README.md` for the +feature list. + +## 0.1.0.0 + +- Deprecated, never completed, indepdently authored prototype. diff --git a/Data/Text/IDNA2008.hs b/Data/Text/IDNA2008.hs deleted file mode 100644 --- a/Data/Text/IDNA2008.hs +++ /dev/null @@ -1,40 +0,0 @@ --- | --- Module: $Header$ --- Description: Encoding hostnames according to IDNA2008 --- Copyright: Copyright © 2011 Jon Kristensen --- License: BSD3 --- --- Maintainer: jon.kristensen@pontarius.org --- Stability: unstable --- Portability: portable --- --- Makes a Unicode hostname internationalized. The code is currently prototyped --- and untested. - - -module Data.Text.IDNA2008 (toASCII) where - -import Data.List (intersperse) -import Data.List.Split (splitOn) -import Data.Maybe (fromJust) -import Data.Text.Punycode (encode) - - --- | --- Converts a Unicode hostname into an internationalized ASCII hostname. - -toASCII :: String -> Maybe String - -toASCII t = if Nothing `elem` encodedLabels - then Nothing - else Just encodedHostname - where - - labels :: [String] - labels = splitOn "." t - - encodedLabels :: [Maybe String] - encodedLabels = map encode labels - - encodedHostname :: String - encodedHostname = concat $ intersperse "." $ map fromJust encodedLabels diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,28 +1,31 @@ -Copyright © 2010-2011, Jon Kristensen. +Copyright (c) 2026 Viktor Dukhovni All rights reserved. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. - * The names of the idna2008 project's contributors may not be used to endorse - or promote products derived from this software without specific prior - written permission. + * Neither the name of the author nor the names of his + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL JON KRISTENSEN BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README b/README deleted file mode 100644 --- a/README +++ /dev/null @@ -1,14 +0,0 @@ -The goal of this Haskell library is to support conversions of -(internationalized) domain names between the ASCII (ACE) and Unicode encodings -in accordance with IDNA2008 (RFC 5991: Internationalized Domain Names in -Applications (IDNA): Protocol). - -The current version, 0.0.1.0, supports encoding a Unicode hostname to an ASCII -hostname. Note however that the code is currently prototyped and untested. - -A later version will add validation support. - -Copyright © 2011, Jon Kristensen. -All rights reserved. - -This library is licensed under a BSD-3 license. See LICENSE for details. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +# idna2008 + +A Haskell library for parsing and validating internationalized +domain names: domain names that may contain characters from +non-Latin scripts (Greek, Hebrew, Arabic, CJK, ...) alongside the +conventional letters, digits, and hyphens. + +## What it does + +Given a domain name as a string (with whatever mix of ASCII and +non-ASCII characters the user typed), the library: + + * Checks that every label (the parts between dots) is allowed. + * Encodes valid non-ASCII Unicode IDN labels (U-labels) to their + ACE-prefixed (`xn--...`) ASCII (A-label) forms, suitable for + inclusion in zone files or use in DNS queries. + * Tells the caller what kind of label each one is (see below), + and lets the caller pick which kinds are accepted in the first + place — strict IDN, hostname-shaped, every form a DNS zone + file might carry plus U-labels, or anything in between. + * Optionally normalises display-form input (case folding, NFC, + full-width to ASCII, alternate label separators) before parsing. + * Optionally renders the parsed name back to display form + (Unicode where possible, ASCII where not). + +## Per-label classification + +A single domain name often mixes different kinds of labels. The +library reports each label as one of: + +| Class | What it is | +|-------------|---------------------------------------------------------| +| `LDH` | A valid label consisting of letters, digits and hyphens.| +| `RLDH` | Legacy reserved labels with `--` at positions 3-4. | +| `FAKEA` | An ACE-prefixed label that isn't a valid A-label. | +| `ALABEL` | An ACE-prefixed label that encodes a valid IDN label. | +| `ULABEL` | A non-ASCII label that can be part of a valid IDN. | +| `ATTRLEAF` | An underscore-prefixed label (e.g. `_25._tcp`). | +| `OCTET` | A label with characters outside the LDH alphabet. | +| `WILDLABEL` | The DNS wildcard label `*`. | +| `LAXULABEL` | A U-label that fails strict IDN validation. | + +A name like `_25._tcp.müllers.example.de` parses cleanly with +five labels in three different classes (`ATTRLEAF`, `ULABEL`, +`LDH`). Most existing IDNA libraries don't make these +distinctions; they typically support only LDH + ALABEL + ULABEL. + +The caller controls which classes are admitted via a +`LabelFormSet`. Pre-built sets cover the common policies: + + * `idnLabelForms` — strict IDN: `LDH` + `ALABEL` + `ULABEL`. + * `hostnameLabelForms` — the IDN set plus `RLDH` and `FAKEA`, + for hostname-shaped names from the wild where unusual but + syntactically valid LDH labels do appear. + * `allLabelForms` — every label class a DNS zone file might + carry (`LDH`, `RLDH`, `FAKEA`, `ALABEL`, `ATTRLEAF`, `OCTET`, + `WILDLABEL`) plus `ULABEL`. Zone files are 8-bit and contain + no U-labels in practice, but admitting U-labels alongside + the on-the-wire forms matches what this library is for — + parsing presentation-form input that may carry either. + +`LAXULABEL` is excluded from every pre-built set: admitting a +U-label that fails strict IDN validation is a deliberate choice +the caller makes by writing it in, e.g. `idnLabelForms <+> +LAXULABEL`. + +## What's distinctive + +* **Strict.** Some browsers and language standard libraries use + a more permissive variant of the IDNA standard that accepts + characters strict IDNA2008 rejects. This library does not use + that variant; if a name is admitted, it's by-the-book valid. + +* **Bidirectional-text rules in two layers.** When right-to-left + scripts (Hebrew, Arabic) appear in a domain name, special rules + prevent visual confusion with neighbouring left-to-right text. + The library splits these rules into a per-label check (does the + label make sense on its own?) and a cross-label check (do the + labels make sense together?), each independently configurable. + An ASCII-fallback option lets display code show a safe ASCII + spelling when the cross-label check would otherwise reject the + name. + +* **Up-to-date Unicode coverage.** The Unicode Consortium + publishes new versions of its character database every year or + so; this library derives its tables directly from those + publications and stays current. + +* **Conformance test vectors.** Test cases are published as + JSON, reusable by ports to other programming languages. + +## Status + +Initial public release (`1.0.0.0`). The conformance suite in +`tests/` carries 186 JSON test vectors with a documented schema +so ports to other languages can reuse the fixtures. + +## Demo + +Given the below `demo.hs`: +```haskell +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE OverloadedStrings #-} +module Main(main) where +import qualified Data.Text.IO as T +import Text.IDNA2008 + +-- Strict default: idnLabelForms + defaultIdnaFlags. +ex1 :: Domain +ex1 = $$(dnLit mkDomain "αβγ.gr") + +-- Enable mappings via @(parseDomainOpts forms flags)@: +ex2 :: Domain +ex2 = $$(let forms = idnLabelForms + flags = defaultIdnaFlags <> allIdnaMappings + parser = parseDomainOpts forms flags + in dnLit (fmap fst . parser) "ΑβΓ.GR") + +main :: IO() +main = do + -- Print A-label form + ascOut ex1 + -- Print U-label form + uniOut ex1 + -- Print A-label + U-label forms and label types: + mapM_ dump $ parseDomain allLabelForms "_25._tcp.*.\\097bc.αβγ.gr" + -- An invalid domain, with code point 95 ('_') in the second label. + -- Only LDH ASCII characters can appear in a U-label. The offset + -- within that label is non-specific because it may have gone + -- through some "mappings" that mask the real byte offset. + print $ parseDomain idnLabelForms "foo.αβ_γδ.gr" + where + ascOut, uniOut :: Domain -> IO () + ascOut = T.putStrLn . domainToAscii + uniOut = T.putStrLn . domainToUnicode + dump (dom, inf) = do + ascOut dom + uniOut dom + print inf +``` +Compiling and running it we get the below output: +``` +xn--mxacd.gr +αβγ.gr +_25._tcp.*.abc.xn--mxacd.gr +_25._tcp.*.abc.αβγ.gr +[ATTRLEAF,ATTRLEAF,WILDLABEL,OCTET,ULABEL,LDH] +Left (ErrLabelInvalid 1 (DisallowedCodepoint 95)) +``` + +## License + +BSD-3-Clause. diff --git a/Setup.hs b/Setup.hs deleted file mode 100644 --- a/Setup.hs +++ /dev/null @@ -1,3 +0,0 @@ -import Distribution.Simple - -main = defaultMain diff --git a/idna2008.cabal b/idna2008.cabal --- a/idna2008.cabal +++ b/idna2008.cabal @@ -1,63 +1,233 @@ -Name: idna2008 -Version: 0.0.1.0 -Cabal-Version: >= 1.6 -Build-Type: Simple -License: BSD3 -License-File: LICENSE -Copyright: Copyright © 2011, Jon Kristensen -Author: Jon Kristensen -Maintainer: jon.kristensen@pontarius.org -Stability: alpha --- Homepage: -Bug-Reports: mailto:jon.kristensen@pontarius.org --- Package-URL: -Synopsis: Converts Unicode hostnames into ASCII -Description: Library to convert internationalized domain names between - the ASCII (ACE) and Unicode encodings in accordance with - IDNA2008 (RFC 5991: Internationalized Domain Names in - Applications (IDNA): Protocol). Unlike IDNA2003, IDNA2008 is - not based on RFC 3454: Preparation of Internationalized - Strings (\"stringprep\"). -Category: Network -Tested-With: GHC ==7.0.2 --- Data-Files: --- Data-Dir: --- Extra-Source-Files: --- Extra-Tmp-Files: +cabal-version: 3.8 -Library - Exposed-Modules: Data.Text.IDNA2008 - Exposed: True - Build-Depends: base >= 2 && < 5, punycode ==0.0.1.0, split - -- Other-Modules: - -- HS-Source-Dirs: - -- Extensions: - -- Build-Tools: - -- Buildable: - -- GHC-Options: - -- GHC-Prof-Options: - -- Hugs-Options: - -- NHC98-Options: - -- Includes: - -- Install-Includes: - -- Include-Dirs: - -- C-Sources: - -- Extra-Libraries: - -- Extra-Lib-Dirs: - -- CC-Options: - -- LD-Options: - -- Pkgconfig-Depends: - -- Frameworks: +name: idna2008 +version: 1.0.0.0 +synopsis: Strict IDNA2008 for Haskell +description: + A Haskell library for parsing and validating internationalized domain + names, which may contain characters from non-Latin scripts (Greek, + Hebrew, Arabic, CJK, ...) alongside the conventional letters, digits, + and hyphens. + . + Given a domain name as the user typed it, the library checks + that every label is well-formed, encodes any non-ASCII labels + into their ACE-prefixed form for the wire, tells the + caller what kind of label each one is, and (optionally) + renders the parsed name back to display form. + . + A single domain name often mixes several kinds of labels. + The library reports each label as one of: a conventional + hostname-style letter-digit-hyphen label, a legacy reserved + label, an internationalized label encoded as Punycode, an + @\"xn--\"@-prefix that turns out not to decode cleanly, a + Unicode label, an underscore-prefixed service-discovery + label (e.g. @_25._tcp@, @_dmarc@), an arbitrary-bytes label, + or the DNS wildcard. Most existing IDNA libraries don't + make these distinctions; this library does. + . + Strict IDNA2008. Some browsers and language standard + libraries use a more permissive variant of the IDNA standard + that accepts characters strict IDNA2008 rejects; this + library does not use that variant. + . + Originally factored out of the @dnsbase@ library; conformance + test vectors are published as JSON for reuse by ports to + other languages. -Source-Repository head - Type: git - -- Module: - Location: git://github.com/jonkri/idna2008.git - -- Subdir: +author: Viktor Dukhovni +maintainer: ietf-dane@dukhovni.org +copyright: 2026 Viktor Dukhovni +license: BSD-3-Clause +license-file: LICENSE +category: Text +homepage: https://github.com/dnsbase/idna2008 +bug-reports: https://github.com/dnsbase/idna2008/issues +build-type: Simple +tested-with: GHC == 9.6.7 + , GHC == 9.8.4 + , GHC == 9.10.3 + , GHC == 9.12.4 + , GHC == 9.14.1 -Source-Repository this - Type: git - -- Module: - Location: git://github.com/jonkri/idna2008.git - Tag: 0.0.1.0 - -- Subdir: +-- The Unicode version the library's generated tables track. This is +-- the single source of truth: 'internal/tools/update' uses it as the +-- default version to fetch the UCD inputs from, and the +-- 'unicode-conformance' test suite uses it to fetch the matching +-- IdnaTestV2.txt and IdnaMappingTable.txt files. Bump this value +-- when re-running 'internal/tools/update' against a new Unicode +-- release. The @x-@ prefix marks this as a custom Cabal field that +-- the cabal tool ignores but other consumers can read. +x-unicode-version: 17.0.0 + +extra-doc-files: + CHANGELOG.md + README.md + +-- Gzip-compressed snapshots of the UTS #46 reference data the +-- 'unicode-conformance' test suite consults. Bundled in the source +-- distribution so 'cabal test' has no network dependency. The +-- 'internal/tools/update' script regenerates these whenever the +-- 'x-unicode-version' field above is bumped. Gzip is used rather +-- than xz so that the test suite can rely on the Haskell @zlib@ +-- package (which ships everywhere) rather than @lzma@ (which needs +-- the liblzma C library wired up through pkg-config). +extra-source-files: + internal/tools/genIdnaBidi.py + internal/tools/genIdnaCombining.py + internal/tools/genIdnaDerive.py + internal/tools/genIdnaEmoji.py + internal/tools/genIdnaJoining.py + internal/tools/genIdnaNFC.py + internal/tools/genIdnaNFCTables.py + internal/tools/genIdnaProperty.py + internal/tools/genIdnaScript.py + internal/tools/genIdnaSimpleLower.py + internal/tools/genIdnaWidth.py + internal/tools/update + tests/data/idna-test-v2-expected-diffs.txt + tests/data/IdnaMappingTable.txt.gz + tests/data/IdnaTestV2.txt.gz + tests/README.md + tests/vectors.json + +source-repository head + type: git + location: https://github.com/dnsbase/idna2008.git + +common all + ghc-options: -O2 -Wall + default-language: GHC2021 + default-extensions: + BangPatterns + BlockArguments + MultiWayIf + PatternSynonyms + StrictData + +common deps + build-depends: + base >=4.18 && <5 + , bytestring >=0.11 && <0.13 + , primitive >=0.9 && <0.10 + , template-haskell >=2.20 && <2.25 + , text >=2.0 && <2.2 + +-- Narrowly-scoped private sublibrary holding only +-- 'Text.IDNA2008.Internal.Punycode'. Its sole reason for +-- existing is to let the @punycode@ test suite (which exercises +-- the codec against RFC 3492 section 7.1 vectors as raw +-- round-trips) import the module directly without forcing the +-- entire @Text.IDNA2008.Internal.*@ surface into either the +-- public Hackage API or Haddock's view. We keep the sublibrary +-- as small as possible because @cabal haddock@ has known issues +-- generating documentation when private sublibraries carry a +-- large module set; the rest of the Internal.* modules continue +-- to live as @other-modules@ of the main library. +library punycode-internal + import: all, deps + visibility: private + hs-source-dirs: internal-punycode + exposed-modules: + Text.IDNA2008.Internal.Punycode + +library + import: all, deps + hs-source-dirs: src internal + build-depends: + idna2008:punycode-internal + exposed-modules: + Text.IDNA2008 + Text.IDNA2008.Wire + other-modules: + Text.IDNA2008.Internal.Bidi + Text.IDNA2008.Internal.Bidi.Data + Text.IDNA2008.Internal.Case + Text.IDNA2008.Internal.Case.Data + Text.IDNA2008.Internal.Combining + Text.IDNA2008.Internal.Combining.Data + Text.IDNA2008.Internal.Emoji + Text.IDNA2008.Internal.Emoji.Data + Text.IDNA2008.Internal.Error + Text.IDNA2008.Internal.Flags + Text.IDNA2008.Internal.Joining + Text.IDNA2008.Internal.Joining.Data + Text.IDNA2008.Internal.LabelForm + Text.IDNA2008.Internal.LabelFormSet + Text.IDNA2008.Internal.LabelInfo + Text.IDNA2008.Internal.NFC + Text.IDNA2008.Internal.NFC.Data + Text.IDNA2008.Internal.NFC.Tables + Text.IDNA2008.Internal.NFC.Tables.Data + Text.IDNA2008.Internal.Parse + Text.IDNA2008.Internal.Property + Text.IDNA2008.Internal.Property.Data + Text.IDNA2008.Internal.Ranges + Text.IDNA2008.Internal.Script + Text.IDNA2008.Internal.Script.Data + Text.IDNA2008.Internal.Tokens + Text.IDNA2008.Internal.UTS46 + Text.IDNA2008.Internal.Width + Text.IDNA2008.Internal.Width.Data + +-- RFC 3492 section 7.1 Punycode vectors, tested as raw codec +-- round-trips through the internal Punycode module rather than +-- through the full IDNA pipeline. The vectors live as Haskell +-- code (not in @tests/vectors.json@) because they exercise a +-- module whose API is not part of the public IDNA library +-- surface. +test-suite punycode + import: all, deps + type: exitcode-stdio-1.0 + hs-source-dirs: tests + main-is: punycode.hs + -- @idna2008@ is listed here even though the test only imports + -- from @idna2008:punycode-internal@. Without this fictional + -- dependency, building under @--enable-coverage@ races: Cabal + -- starts linking the punycode test before it has finished + -- building (and registering) the main library, which the + -- coverage-instrumented test links against by way of HPC's + -- cross-component tixing. Listing @idna2008@ explicitly forces + -- the build order, so a clean coverage build works on the first + -- attempt without needing a separate 'cabal build all' pass. + build-depends: + idna2008 + , idna2008:punycode-internal + , tasty >=1.5 && <1.6 + , tasty-hunit >=0.10 && <0.11 + +-- Unicode @IdnaTestV2.txt@ conformance suite. Reads +-- XZ-compressed snapshots of the UTS \#46 reference data bundled +-- under @tests/data/@ (see 'extra-source-files') so 'cabal test' +-- runs offline. Known systematic disagreements between strict +-- IDNA2008 and UTS #46 Nontransitional are listed in +-- @tests\/data\/idna-test-v2-expected-diffs.txt@; an additional +-- auto-skip uses @IdnaMappingTable.txt@ to identify codepoints +-- where UTS \#46's preprocessing legitimately differs from strict +-- IDNA2008. An @IDNA_TEST_V2_FILE@\/@IDNA_MAPPING_TABLE_FILE@ +-- environment variable can point at an uncompressed local copy +-- to test against a different Unicode version without re-bundling. +test-suite unicode-conformance + import: all, deps + type: exitcode-stdio-1.0 + hs-source-dirs: tests + main-is: unicode_conformance.hs + build-depends: + idna2008 + , idna2008:punycode-internal + , containers >=0.6 && <0.9 + , directory >=1.0 && <1.4 + , tasty >=1.5 && <1.6 + , tasty-hunit >=0.10 && <0.11 + , zlib >=0.6 && <0.8 + +test-suite conformance + import: all, deps + type: exitcode-stdio-1.0 + hs-source-dirs: tests + main-is: conformance.hs + build-depends: + idna2008 + , idna2008:punycode-internal + , aeson >=2.2 && <2.4 + , tasty >=1.5 && <1.6 + , tasty-hunit >=0.10 && <0.11 diff --git a/internal-punycode/Text/IDNA2008/Internal/Punycode.hs b/internal-punycode/Text/IDNA2008/Internal/Punycode.hs new file mode 100644 --- /dev/null +++ b/internal-punycode/Text/IDNA2008/Internal/Punycode.hs @@ -0,0 +1,439 @@ +-- | +-- Module : Text.IDNA2008.Internal.Punycode +-- Description : RFC 3492 Punycode encoder and decoder +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- A self-contained implementation of RFC 3492 Punycode used by +-- IDNA A-label \(\Leftrightarrow\) U-label conversion. The +-- encoder and decoder both write into caller-supplied mutable +-- buffers; no intermediate storage is allocated, and arithmetic +-- is performed in machine 'Int' with explicit overflow checks. +-- +-- Inputs: +-- +-- * The encoder consumes a 'PrimArray' of codepoint values +-- (each in @0..0x10FFFF@, surrogates excluded — the caller +-- is responsible for enforcing that) and writes the +-- corresponding base-36 ACE bytes into a 'MutableByteArray'. +-- The output does /not/ include the @\"xn--\"@ prefix; +-- callers add that. +-- +-- * The decoder consumes a slice of an immutable 'ByteArray' +-- (the ACE body, /not/ including @\"xn--\"@) and writes the +-- recovered codepoints into a 'MutablePrimArray'. +-- +-- Both routines return the number of elements written, or a +-- 'PunycodeErr' on failure. +module Text.IDNA2008.Internal.Punycode + ( PunycodeErr(..) + , punycodeEncode + , punycodeDecode + ) where + +import Control.Monad.ST (ST) +import Data.Bits (unsafeShiftR) +import Data.Primitive.ByteArray + ( ByteArray + , MutableByteArray + , indexByteArray + , writeByteArray + ) +import Data.Primitive.PrimArray + ( MutablePrimArray + , PrimArray + , indexPrimArray + , readPrimArray + , writePrimArray + ) +import Data.Word (Word8) + +-- | Errors that can be produced by the encoder or decoder. +data PunycodeErr + = PunycodeOverflow + -- ^ Internal Punycode arithmetic exceeded the representable range. + | PunycodeBadDigit + -- ^ Decoder encountered a non-base-36 byte where a digit was required, + -- or a non-basic byte in the basic prefix. + | PunycodeTruncated + -- ^ Decoder ran out of input mid-integer. + | PunycodeOutputFull + -- ^ The caller-provided output buffer is too small for the result. + deriving (Eq, Show) + +---------------------------------------------------------------------- +-- RFC 3492 constants +---------------------------------------------------------------------- + +pBase, pTmin, pTmax, pSkew, pDamp, pInitialBias, pInitialN :: Int +pBase = 36 +pTmin = 1 +pTmax = 26 +pSkew = 38 +pDamp = 700 +pInitialBias = 72 +pInitialN = 0x80 + +-- | Threshold used by 'adapt': @((base - tmin) * tmax) / 2@. +adaptThreshold :: Int +adaptThreshold = ((pBase - pTmin) * pTmax) `unsafeShiftR` 1 + +-- | Maximum 'Int' value, used in the encoder to find the smallest +-- non-basic codepoint not yet processed in a given round. +maxInt :: Int +maxInt = maxBound + +---------------------------------------------------------------------- +-- Basic-codepoint <-> digit conversions +---------------------------------------------------------------------- + +-- | Map a base-36 digit value (0..35) to its ASCII byte. +-- +-- @0..25@ -> @\'a\'..\'z\'@ (0x61..0x7a) +-- +-- @26..35@ -> @\'0\'..\'9\'@ (0x30..0x39) +{-# INLINE digitToBasic #-} +digitToBasic :: Int -> Word8 +digitToBasic d + | d < 26 = fromIntegral (d + 0x61) + | otherwise = fromIntegral (d + 22) -- 26 + 22 = 0x30 + +-- | Map an ASCII byte to its base-36 digit value, or @-1@ if the byte is +-- not a base-36 digit. Both upper- and lower-case ASCII letters are +-- accepted (case-insensitive on input, per RFC 3492 section 5). +{-# INLINE basicToDigit #-} +basicToDigit :: Word8 -> Int +basicToDigit w + | l <- w - 0x61 + , l < 26 = fromIntegral l + | d <- w - 0x30 + , d <= 9 = fromIntegral d + 26 + | l <- w - 0x41 + , l < 26 = fromIntegral l + | otherwise = -1 + +-- | Bias adaptation, RFC 3492 section 6.1. +adapt :: Int -> Int -> Bool -> Int +adapt d0 numpoints firstTime = + let !d1 = if firstTime then d0 `quot` pDamp else d0 `unsafeShiftR` 1 + !d2 = d1 + d1 `quot` numpoints + in adaptLoop d2 0 + where + adaptLoop !d !k + | d > adaptThreshold + = adaptLoop (d `quot` (pBase - pTmin)) (k + pBase) + | otherwise + = k + ((pBase - pTmin + 1) * d) `quot` (d + pSkew) +{-# INLINE adapt #-} + +---------------------------------------------------------------------- +-- Encoder +---------------------------------------------------------------------- + +-- | Encode a sequence of codepoints to Punycode ASCII bytes (without the +-- @\"xn--\"@ prefix). On success returns the number of bytes written. +punycodeEncode + :: forall s + . PrimArray Int -- ^ Input codepoints + -> Int -- ^ Number of valid codepoints (@>= 0@) + -> MutableByteArray s -- ^ Output buffer + -> Int -- ^ Output start offset + -> Int -- ^ Output capacity in bytes + -> ST s (Either PunycodeErr Int) +punycodeEncode !codepoints !inLen !outBuf !outStart !outCap + | inLen <= 0 = pure (Right 0) + | otherwise = do + -- Pass 1: emit basic (ASCII) codepoints in input order. + r1 <- emitBasics 0 outStart 0 + case r1 of + Left e -> pure (Left e) + Right (off1, b) + | b == 0 + -- No basics: omit the delimiter (RFC 3492 section + -- 6.3: \"copy [basic code points] in order, + -- followed by a delimiter if b > 0\"). + -> mainLoop pInitialN 0 pInitialBias 0 off1 True + | b == inLen -> do + -- All-basic input: emit a trailing delimiter and + -- skip the main loop. RFC 3492 section 6.3 + -- emits the delimiter whenever @b > 0@ -- including + -- the all-basic case, even though there's no + -- extension that follows. IDNA doesn't exercise + -- this branch in production (the U-label path + -- requires at least one non-ASCII codepoint + -- before calling the encoder), but the RFC + -- 3492 conformance vectors -- e.g. section 7.1 + -- example (S) @\"-> $1.00 <-\"@ -- pin this + -- behaviour down, and the decoder requires it + -- for round-tripping. + r2 <- writeOne off1 0x2d + case r2 of + Left e -> pure (Left e) + Right off2 -> pure (Right (off2 - outStart)) + | otherwise -> do + r2 <- writeOne off1 0x2d + case r2 of + Left e -> pure (Left e) + Right off2 -> mainLoop pInitialN 0 pInitialBias b off2 True + where + !endOff = outStart + outCap + + writeOne :: Int -> Word8 -> ST s (Either PunycodeErr Int) + writeOne !off !w + | off >= endOff = pure (Left PunycodeOutputFull) + | otherwise = do + writeByteArray outBuf off w + pure (Right $! off + 1) + {-# INLINE writeOne #-} + + -- Emit basic (ASCII < 0x80) codepoints in their original order; ignore + -- non-basic codepoints in this pass. Returns (new offset, basic count) + -- or an output-full error. + emitBasics :: Int -> Int -> Int -> ST s (Either PunycodeErr (Int, Int)) + emitBasics !i !off !b + | i >= inLen = pure (Right (off, b)) + | otherwise = + let !c = indexPrimArray codepoints i + in if c < pInitialN + then if off >= endOff + then pure (Left PunycodeOutputFull) + else do + writeByteArray outBuf off (fromIntegral c :: Word8) + emitBasics (i+1) (off+1) (b+1) + else emitBasics (i+1) off b + + -- Main encoder loop, RFC 3492 section 6.3. Each iteration handles one new + -- "round" of non-basic codepoints sharing the same minimum value m. + mainLoop :: Int -- ^ n: current minimum codepoint + -> Int -- ^ delta: current delta value + -> Int -- ^ bias + -> Int -- ^ h: codepoints emitted so far + -> Int -- ^ output offset + -> Bool -- ^ firstTime flag for adapt + -> ST s (Either PunycodeErr Int) + mainLoop !n !delta !bias !h !off !firstTime + | h >= inLen = pure (Right (off - outStart)) + | otherwise = + let !m = findMin n + !mn = m - n + !hp1 = h + 1 + in if mn /= 0 && hp1 /= 0 && mn > maxInt `quot` hp1 + then pure (Left PunycodeOverflow) + else + let !inc = mn * hp1 + !delta' = delta + inc + in if delta' < delta + then pure (Left PunycodeOverflow) + else processCp 0 m delta' bias h off firstTime + + -- Find the smallest codepoint in the input that is >= n. Returns + -- maxInt if none exists (shouldn't happen while h < inLen). + findMin :: Int -> Int + findMin !n0 = go 0 maxInt + where + go !i !m + | i >= inLen = m + | otherwise = + let !c = indexPrimArray codepoints i + !m' = if c >= n0 && c < m then c else m + in go (i+1) m' + + -- Inner per-codepoint loop, RFC 3492 section 6.3 step 3. For each input + -- codepoint c: bump delta if c < m, emit a varint and adapt if c == m. + processCp :: Int -> Int -> Int -> Int -> Int -> Int -> Bool + -> ST s (Either PunycodeErr Int) + processCp !i !m !delta !bias !h !off !firstTime + | i >= inLen + -- End of pass: bump delta, bump n, recurse to next round. + = let !delta1 = delta + 1 + in if delta1 < delta + then pure (Left PunycodeOverflow) + else mainLoop (m + 1) delta1 bias h off firstTime + | otherwise = do + let !c = indexPrimArray codepoints i + case compare c m of + LT -> let !d' = delta + 1 + in if d' < delta + then pure (Left PunycodeOverflow) + else processCp (i+1) m d' bias h off firstTime + EQ -> do + rEmit <- emitVarint delta bias off + case rEmit of + Left e -> pure (Left e) + Right off' -> + let !bias' = adapt delta (h + 1) firstTime + in processCp (i+1) m 0 bias' (h+1) off' False + GT -> processCp (i+1) m delta bias h off firstTime + + -- Emit the variable-length base-36 encoding of @q@ with the given bias. + emitVarint :: Int -> Int -> Int -> ST s (Either PunycodeErr Int) + emitVarint !q0 !bias !off0 = go q0 pBase off0 + where + go !q !k !off = + let !t = if k <= bias + pTmin then pTmin + else if k >= bias + pTmax then pTmax + else k - bias + in if q < t + then writeOne off (digitToBasic q) + else do + let !d = t + (q - t) `mod` (pBase - t) + !qn = (q - t) `quot` (pBase - t) + rW <- writeOne off (digitToBasic d) + case rW of + Left e -> pure (Left e) + Right off' -> go qn (k + pBase) off' + +---------------------------------------------------------------------- +-- Decoder +---------------------------------------------------------------------- + +-- | Decode a Punycode ASCII body (no @\"xn--\"@ prefix) into +-- a sequence of codepoints, written into the caller-supplied +-- mutable codepoint buffer. Returns the number of codepoints +-- produced. +-- +-- The decoder accepts both upper- and lower-case base-36 digits; +-- callers performing strict A-label round-trip checks should +-- compare the /lower-cased/ input ACE bytes against a freshly +-- encoded form. +punycodeDecode + :: forall s + . ByteArray -- ^ Input ASCII bytes (Punycode body) + -> Int -- ^ Input start offset + -> Int -- ^ Input length + -> MutablePrimArray s Int -- ^ Output codepoint buffer + -> Int -- ^ Output start offset + -> Int -- ^ Output capacity (codepoints) + -> ST s (Either PunycodeErr Int) +punycodeDecode !inBuf !inStart !inLen !outBuf !outStart !outCap + | inLen < 0 = pure (Left PunycodeTruncated) + | otherwise = do + -- Locate the last delimiter '-': everything before it is basic, + -- everything after is base-36 encoded. If no '-' is present, the + -- whole input is encoded (no basic prefix). + let !delimAt = lastDelim (inEnd - 1) + if delimAt < inStart + then -- No delimiter: whole input is base-36. + decodeLoop pInitialN pInitialBias 0 outStart 0 True inStart + else do + -- Copy basics [inStart .. delimAt-1] verbatim as codepoints. + r <- copyBasics inStart (delimAt - 1) outStart 0 + case r of + Left e -> pure (Left e) + Right b + | b == 0 -> + -- RFC 3492: do not consume the delimiter when no basics + -- were copied; this means the input is malformed because + -- the next byte is the delimiter, which is not a digit. + decodeLoop pInitialN pInitialBias 0 outStart 0 True delimAt + | otherwise -> + decodeLoop pInitialN pInitialBias 0 (outStart + b) b + True (delimAt + 1) + where + !inEnd = inStart + inLen + !outEnd = outStart + outCap + + -- Find the index of the last '-' in [inStart .. p], or inStart-1 if + -- there isn't one. + lastDelim :: Int -> Int + lastDelim !p + | p < inStart = inStart - 1 + | (indexByteArray inBuf p :: Word8) == 0x2d = p + | otherwise = lastDelim (p - 1) + + -- Copy [from..toIncl] of basics into the output buffer. Returns the + -- number of codepoints written, or PunycodeOutputFull / PunycodeBadDigit + -- on error. + copyBasics :: Int -> Int -> Int -> Int + -> ST s (Either PunycodeErr Int) + copyBasics !i !iEnd !o !count + | i > iEnd = pure (Right count) + | o >= outEnd = pure (Left PunycodeOutputFull) + | otherwise = do + let !w = indexByteArray inBuf i :: Word8 + if w >= 0x80 + then pure (Left PunycodeBadDigit) + else do + writePrimArray outBuf o (fromIntegral w :: Int) + copyBasics (i + 1) iEnd (o + 1) (count + 1) + + -- Main decode loop, RFC 3492 section 6.2. + decodeLoop + :: Int -- ^ n: current codepoint base + -> Int -- ^ bias + -> Int -- ^ i: insertion-position accumulator (RFC 'i') + -> Int -- ^ output offset (== outStart + count) + -> Int -- ^ count of codepoints written so far + -> Bool -- ^ firstTime flag for adapt + -> Int -- ^ input offset + -> ST s (Either PunycodeErr Int) + decodeLoop !n !bias !i !out !count !firstTime !ip + | ip >= inEnd = + pure (Right (out - outStart)) + | otherwise = do + r <- readVarint i bias 1 pBase ip + case r of + Left e -> pure (Left e) + Right (i', ip') -> do + let !numpts = count + 1 + !bias' = adapt (i' - i) numpts firstTime + !n' = n + i' `quot` numpts + !pos = i' `mod` numpts + if n' > 0x10FFFF + then pure (Left PunycodeOverflow) + else if out >= outEnd + then pure (Left PunycodeOutputFull) + else do + shiftRight (outStart + pos) (out - 1) + writePrimArray outBuf (outStart + pos) n' + decodeLoop n' bias' (pos + 1) (out + 1) + (count + 1) False ip' + + -- Read one base-36 variable-length integer starting at input offset 'ip', + -- accumulating into 'i'. Returns (new value of i, input offset after + -- the integer's last digit) or an error. + readVarint :: Int -> Int -> Int -> Int -> Int + -> ST s (Either PunycodeErr (Int, Int)) + readVarint !i !bias !w !k !ip + | ip >= inEnd = pure (Left PunycodeTruncated) + | otherwise = + let !digit = basicToDigit (indexByteArray inBuf ip) + in if digit < 0 + then pure (Left PunycodeBadDigit) + else + let !contribution = digit * w + in if w /= 0 && contribution `quot` w /= digit + then pure (Left PunycodeOverflow) + else + let !i' = i + contribution + in if i' < i + then pure (Left PunycodeOverflow) + else + let !t = if k <= bias + pTmin then pTmin + else if k >= bias + pTmax then pTmax + else k - bias + in if digit < t + then pure (Right (i', ip + 1)) + else + let !w' = w * (pBase - t) + in if pBase - t /= 0 + && w' `quot` (pBase - t) /= w + then pure (Left PunycodeOverflow) + else readVarint i' bias w' + (k + pBase) + (ip + 1) + + -- Shift codepoints in [low .. lastIdx] one slot to the right. Walks + -- backwards so we don't trample data we still need to read. + shiftRight :: Int -> Int -> ST s () + shiftRight !low !lastIdx = go lastIdx + where + go !p + | p < low = pure () + | otherwise = do + !v <- readPrimArray outBuf p + writePrimArray outBuf (p + 1) v + go (p - 1) diff --git a/internal/Text/IDNA2008/Internal/Bidi.hs b/internal/Text/IDNA2008/Internal/Bidi.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Bidi.hs @@ -0,0 +1,349 @@ +-- | +-- Module : Text.IDNA2008.Internal.Bidi +-- Description : Typed wrapper over the Bidi_Class table consumed by +-- the per-label RFC 5893 check in the IDNA parser. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- One lookup, 'bidiClassCp', backed by binary search over the +-- generated range table in "Text.IDNA2008.Internal.Bidi.Data". +-- The table covers every codepoint in @[0, 0x10FFFF]@ with no gaps, +-- so 'bidiClassCp' is a total function. +-- +-- The eleven classes named in RFC 5893 (L, R, AL, AN, EN, ES, CS, +-- ET, ON, BN, NSM) get their own pattern synonyms; everything else +-- (paragraph\/segment\/whitespace separators, embedding\/override +-- and isolate format characters) is reported as 'BidiOther', which +-- the parser treats as a Bidi-rule violation in any label where it +-- appears. +{-# LANGUAGE CPP #-} + +module Text.IDNA2008.Internal.Bidi + ( -- * Class lookup + bidiClassCp + -- * Per-label summary + , BidiSummary + , emptyBidiSummary + , extendBidiSummary + , buildBidiSummary + , bidiSummaryFromCps + -- * Cross-label trigger fold + , addBidiTrigger + -- * Rule check + , checkBidiSummary + ) where + +import Control.Monad.ST (ST) +import Data.Bits ((.&.), (.|.), bit, testBit, unsafeShiftL, unsafeShiftR) +#if !MIN_VERSION_base(4,20,0) +import Data.Foldable(foldl') +#endif +import Data.Primitive.ByteArray (indexByteArray) +import Data.Primitive.PrimArray + ( MutablePrimArray + , readPrimArray + ) +import Data.Word (Word8, Word16, Word32) + +import Text.IDNA2008.Internal.Bidi.Data + ( bidiRangeCount + , bidiRangeStarts + , bidiRangeTags + ) +import Text.IDNA2008.Internal.Error (BidiRuleViolation(..)) + +---------------------------------------------------------------------- +-- Class tag +---------------------------------------------------------------------- + +-- | Resolved Unicode @Bidi_Class@ for the purposes of RFC 5893. +-- Stored as a 'Word8' tag in @[0, 11]@; pattern synonyms below +-- expose the named classes. Tag 11 ('BidiOther') aggregates the +-- classes that RFC 5893 forbids unconditionally (B, S, WS, the +-- embedding\/override\/isolate format characters), so any label +-- containing one of them fails the rule check. +newtype BidiClass = BidiClass_ Word8 deriving Eq + +-- The patterns below cover the classes the parser inspects by +-- name; the underlying type also represents tags 5..9 for the +-- neutral classes (ES, CS, ET, ON, BN) that RFC 5893 permits in +-- both LTR and RTL labels but which never trigger a rule +-- decision -- they flow through the validation code via @_@ +-- catch-alls. Naming them as patterns would add nothing the +-- parser can use, so the tags stay unnamed. +pattern BidiL :: BidiClass +pattern BidiL = BidiClass_ 0 -- ^ Left_To_Right +pattern BidiR :: BidiClass +pattern BidiR = BidiClass_ 1 -- ^ Right_To_Left +pattern BidiAL :: BidiClass +pattern BidiAL = BidiClass_ 2 -- ^ Arabic_Letter +pattern BidiAN :: BidiClass +pattern BidiAN = BidiClass_ 3 -- ^ Arabic_Number +pattern BidiEN :: BidiClass +pattern BidiEN = BidiClass_ 4 -- ^ European_Number +pattern BidiNSM :: BidiClass +pattern BidiNSM = BidiClass_ 10 -- ^ Nonspacing_Mark +pattern BidiOther :: BidiClass +pattern BidiOther = BidiClass_ 11 -- ^ everything else: B, S, WS, + -- embedding\/override\/isolate + -- format characters. Forbidden + -- in any Bidi-rule-checked label. + +-- | Wrap a 'Word8' tag from the generated table. Tags @0..10@ map +-- to the named classes; anything else collapses to 'BidiOther' on +-- the principle of conservative classification (would-be-failing +-- values fail). +{-# INLINE word8ToClass #-} +word8ToClass :: Word8 -> BidiClass +word8ToClass w + | w <= 10 = BidiClass_ w + | otherwise = BidiOther + +---------------------------------------------------------------------- +-- Public lookup +---------------------------------------------------------------------- + +-- | Return the resolved 'BidiClass' of a codepoint. Out-of-range +-- inputs (negative or greater than @0x10FFFF@) report 'BidiOther', +-- which causes any label containing them to fail the Bidi check. +bidiClassCp :: Int -> BidiClass +bidiClassCp !cp + | cp < 0 || cp > 0x10FFFF = BidiOther + | otherwise = word8ToClass (lookupTag cp) +{-# INLINE bidiClassCp #-} + +---------------------------------------------------------------------- +-- Binary search over the codegen'd range table +---------------------------------------------------------------------- + +-- | Binary-search for the largest index @i@ in +-- @[0, bidiRangeCount - 1]@ such that @bidiRangeStarts[i] <= cp@. +-- Returns the tag at that index. Pre-condition: the table starts +-- at @0x000000@, so a hit is guaranteed for any @cp >= 0@. +{-# INLINE lookupTag #-} +lookupTag :: Int -> Word8 +lookupTag !cp = go 0 (bidiRangeCount - 1) + where + !target = fromIntegral cp :: Word32 + + go !lo !hi + | lo >= hi = readTag lo + | otherwise = + let !mid = (lo + hi + 1) `unsafeShiftR` 1 + !midKey = readStart mid + in if midKey <= target + then go mid hi + else go lo (mid - 1) + + readStart i = indexByteArray bidiRangeStarts i :: Word32 + readTag i = indexByteArray bidiRangeTags i :: Word8 + +---------------------------------------------------------------------- +-- Per-label summary +---------------------------------------------------------------------- + +-- | Per-label Bidi summary, packed into a single 'Word16': +-- +-- @ +-- bits 0..3 : first-codepoint class (0..11) +-- bits 4..7 : last non-NSM class (0..11) +-- bit 8 : has any L +-- bit 9 : has any R or AL +-- bit 10 : has any AN +-- bit 11 : has any EN +-- bit 12 : has any Other +-- @ +-- +-- Captures everything Rules 1-6 of RFC 5893 need to decide a +-- label's verdict without retaining the codepoint sequence. +-- Accumulated by left-fold via 'extendBidiSummary' starting from +-- 'emptyBidiSummary'. The bit layout is internal — inspect via +-- the accessor functions ('bsFirstClass', 'bsLastNonNSM', +-- 'bsHasL', 'bsHasRorAL', 'bsHasAN', 'bsHasEN', 'bsHasOther'). +newtype BidiSummary = BidiSummary_ Word16 deriving Eq + +-- Bit indices for the presence flags within a 'BidiSummary's +-- 'Word16'. The low byte stores the first / last class slots +-- (bits 0..3 and 4..7); presence flags live above that. +bsHasLIdx, bsHasRorALIdx, bsHasANIdx, bsHasENIdx, bsHasOtherIdx :: Int +bsHasLIdx = 8 +bsHasRorALIdx = 9 +bsHasANIdx = 10 +bsHasENIdx = 11 +bsHasOtherIdx = 12 + +-- | Class of the first codepoint. Rule 1 requires this to be +-- 'BidiL', 'BidiR', or 'BidiAL'. +bsFirstClass :: BidiSummary -> BidiClass +bsFirstClass (BidiSummary_ w) = BidiClass_ (fromIntegral (w .&. 0x000F)) +{-# INLINE bsFirstClass #-} + +-- | Class of the last codepoint that is not 'BidiNSM'. Rules 3 +-- and 6 examine this to decide if the label ends correctly past +-- any trailing combining marks. +bsLastNonNSM :: BidiSummary -> BidiClass +bsLastNonNSM (BidiSummary_ w) = + BidiClass_ (fromIntegral ((w `unsafeShiftR` 4) .&. 0x000F)) +{-# INLINE bsLastNonNSM #-} + +bsHasL, bsHasRorAL, bsHasAN, bsHasEN, bsHasOther :: BidiSummary -> Bool +bsHasL (BidiSummary_ w) = testBit w bsHasLIdx +bsHasRorAL (BidiSummary_ w) = testBit w bsHasRorALIdx +bsHasAN (BidiSummary_ w) = testBit w bsHasANIdx +bsHasEN (BidiSummary_ w) = testBit w bsHasENIdx +bsHasOther (BidiSummary_ w) = testBit w bsHasOtherIdx +{-# INLINE bsHasL #-} +{-# INLINE bsHasRorAL #-} +{-# INLINE bsHasAN #-} +{-# INLINE bsHasEN #-} +{-# INLINE bsHasOther #-} + +-- | OR a per-label summary's cross-label trigger contribution +-- into a running flag. Designed for left-fold over a label +-- sequence: @'addBidiTrigger' trig sm@ is 'True' if either @trig@ +-- was already 'True' or @sm@ contains an @R@, @AL@, or @AN@ +-- codepoint — the conditions under which RFC 5893 fires +-- cross-label Rules 1-6. Encapsulates the bit-level details of +-- which classes trigger global checks so callers don't need to +-- import the per-flag predicates. +addBidiTrigger :: Bool -> BidiSummary -> Bool +addBidiTrigger !trig (BidiSummary_ w) = + trig || testBit w bsHasRorALIdx || testBit w bsHasANIdx +{-# INLINE addBidiTrigger #-} + +-- | The summary of an empty (zero-codepoint) label. Encodes +-- 'BidiOther' in both first and last slots and clears every +-- presence flag, so any rule check on it fails Rule 1 cleanly. +-- Empty labels shouldn't normally reach a per-label Bidi check +-- (the parser rejects them earlier) but the constant gives +-- 'extendBidiSummary' something to fold over. +-- +-- The empty / initial-state predicate is exactly +-- @(w `.&.` 0xFF00) == 0 && (w `.&.` 0x000F) == 11@, which +-- 'extendBidiSummary' uses to distinguish the first update from +-- subsequent updates. Encoded value: @(BidiOther << 4) | +-- BidiOther = 0xBB@. +emptyBidiSummary :: BidiSummary +emptyBidiSummary = BidiSummary_ 0x00BB +{-# INLINE emptyBidiSummary #-} + +-- | Map a 'BidiClass' to the presence-flag bit it contributes +-- (zero for classes that don't carry a flag: ES, CS, ET, ON, +-- BN, NSM). +classFlagBit :: BidiClass -> Word16 +classFlagBit BidiL = bit bsHasLIdx +classFlagBit BidiR = bit bsHasRorALIdx +classFlagBit BidiAL = bit bsHasRorALIdx +classFlagBit BidiAN = bit bsHasANIdx +classFlagBit BidiEN = bit bsHasENIdx +classFlagBit BidiOther = bit bsHasOtherIdx +classFlagBit _ = 0 +{-# INLINE classFlagBit #-} + +-- | Update a 'BidiSummary' with one more codepoint's class. +-- Designed for left-fold over a sequence of codepoints starting +-- from 'emptyBidiSummary'. +extendBidiSummary :: BidiSummary -> BidiClass -> BidiSummary +extendBidiSummary !(BidiSummary_ w) !c@(BidiClass_ cw) = + let !cw16 = fromIntegral cw :: Word16 + !flag = classFlagBit c + in if (w .&. 0xFF00) == 0 && (w .&. 0x000F) == 11 + -- Initial state: first slot still BidiOther, no flags + -- set. Seed both first and last slots with this + -- codepoint's class, plus its flag bit. + then BidiSummary_ (cw16 .|. (cw16 `unsafeShiftL` 4) .|. flag) + -- Subsequent update: always OR in the flag bit; update + -- the last-non-NSM slot only if the codepoint is not + -- NSM. + else case c of + BidiNSM -> BidiSummary_ (w .|. flag) + _ -> BidiSummary_ + ((w .&. 0xFF0F) .|. (cw16 `unsafeShiftL` 4) .|. flag) +{-# INLINE extendBidiSummary #-} + +-- | Walk @cpBuf[0..cnt)@ accumulating a 'BidiSummary'. Used by +-- the parser-time per-label check (which already holds a +-- 'MutablePrimArray' of decoded codepoints). +buildBidiSummary + :: forall s. MutablePrimArray s Int -> Int -> ST s BidiSummary +buildBidiSummary !cpBuf !cnt = go 0 emptyBidiSummary + where + go :: Int -> BidiSummary -> ST s BidiSummary + go !i !s + | i >= cnt = pure s + | otherwise = do + cp <- readPrimArray cpBuf i + go (i + 1) (extendBidiSummary s (bidiClassCp cp)) +{-# INLINE buildBidiSummary #-} + +-- | Pure 'BidiSummary' computation over an arbitrary sequence of +-- codepoints; used by the presentation-time global check, which +-- consumes already-decoded U-label codepoints. +bidiSummaryFromCps :: [Int] -> BidiSummary +bidiSummaryFromCps = + foldl' (\ !s !cp -> extendBidiSummary s (bidiClassCp cp)) + emptyBidiSummary +{-# INLINE bidiSummaryFromCps #-} + +---------------------------------------------------------------------- +-- Rule check +---------------------------------------------------------------------- + +-- | Validate a 'BidiSummary' against the per-label conditions of +-- RFC 5893 Rules 1-6. Returns 'Nothing' if the label is clean, +-- or the offending 'BidiRuleViolation' otherwise. +-- +-- The first 'Bool' argument controls the trigger short-circuit: +-- +-- * 'False' (per-label mode) -- only check labels whose own +-- codepoints contain an @R@, @AL@, or @AN@ codepoint. +-- Labels without any such content (pure-LTR labels) always +-- return 'Nothing'. This is the right semantic for the +-- parser, which validates each label in isolation. +-- * 'True' (global mode) -- check every label unconditionally. +-- This is the right semantic for the cross-label +-- presentation-time check, which fires Rules 1-6 on every +-- label of a domain once any label has RTL content. +-- +-- In either mode, the rule application itself is identical: +-- Rule 1, then either the RTL trio (Rules 2 \/ 3 \/ 4) or the +-- LTR pair (Rules 5 \/ 6) depending on the first-codepoint +-- direction. An out-of-set codepoint (anything @BidiOther@) +-- fails the corresponding allowed-set rule (2 in RTL, 5 in LTR). +checkBidiSummary :: Bool -> BidiSummary -> Maybe BidiRuleViolation +checkBidiSummary !forceCheck !s + -- Per-label trigger: skip clean LTR-only labels unless we're + -- in global mode. + | not forceCheck + , not (bsHasRorAL s || bsHasAN s) = Nothing + -- Rule 1: first must be L, R, or AL. + | first /= BidiL + , first /= BidiR + , first /= BidiAL = Just BidiRule1FirstNotLRAL + | first == BidiL = + -- LTR rules 5 and 6. Either a forbidden codepoint class + -- or a wrong-direction trailing class fails. + if bsHasRorAL s || bsHasAN s || bsHasOther s + then Just BidiRule5LTRDisallowed + else case bsLastNonNSM s of + BidiL -> Nothing + BidiEN -> Nothing + _ -> Just BidiRule6LTRBadEnd + | otherwise = + -- 'first' is BidiR or BidiAL. RTL rules 2, 3, 4. + if bsHasL s || bsHasOther s + then Just BidiRule2RTLDisallowed + else if bsHasEN s && bsHasAN s + then Just BidiRule4ENANMix + else case bsLastNonNSM s of + BidiR -> Nothing + BidiAL -> Nothing + BidiEN -> Nothing + BidiAN -> Nothing + _ -> Just BidiRule3RTLBadEnd + where + !first = bsFirstClass s +{-# INLINE checkBidiSummary #-} diff --git a/internal/Text/IDNA2008/Internal/Bidi/Data.hs b/internal/Text/IDNA2008/Internal/Bidi/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Bidi/Data.hs @@ -0,0 +1,261 @@ +-- | +-- Module : Text.IDNA2008.Internal.Bidi.Data +-- Description : Codegen'd Bidi_Class table for the per-label RFC 5893 +-- check applied in "Text.IDNA2008.Internal.Parse". +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED FROM DerivedBidiClass.txt. Do not edit by +-- hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaBidi.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedBidiClass.txt +-- Aligned with: Unicode 17.0.0 +-- Range count: 1252 +-- +-- The table covers every Unicode codepoint in @[0, 0x10FFFF]@ with no +-- gaps: each entry says "from this start until the next entry's start +-- (or 0x10FFFF for the last), the resolved Bidi_Class is /T/". The +-- eleven classes relevant to RFC 5893 -- L, R, AL, AN, EN, ES, CS, +-- ET, ON, BN, NSM -- get distinct tags 0..10; every other class (B, +-- S, WS, the embedding\/override\/isolate format characters) +-- collapses to a single @Other@ tag (11), which the parser treats as +-- a Bidi-rule failure for any label in which it appears. +module Text.IDNA2008.Internal.Bidi.Data + ( bidiRangeCount + , bidiRangeStarts + , bidiRangeTags + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +-- | Number of (start, class) entries. +bidiRangeCount :: Int +bidiRangeCount = 1252 + +-- | Range starts; sorted ascending. +bidiRangeStarts :: ByteArray +bidiRangeStarts = PBA.byteArrayFromList + [ (0x000000 :: Word32), 0x000009, 0x00000E, 0x00001C, 0x000021, 0x000023, 0x000026, 0x00002B + , 0x00002C, 0x00002D, 0x00002E, 0x000030, 0x00003A, 0x00003B, 0x000041, 0x00005B + , 0x000061, 0x00007B, 0x00007F, 0x000085, 0x000086, 0x0000A0, 0x0000A1, 0x0000A2 + , 0x0000A6, 0x0000AA, 0x0000AB, 0x0000AD, 0x0000AE, 0x0000B0, 0x0000B2, 0x0000B4 + , 0x0000B5, 0x0000B6, 0x0000B9, 0x0000BA, 0x0000BB, 0x0000C0, 0x0000D7, 0x0000D8 + , 0x0000F7, 0x0000F8, 0x0002B9, 0x0002BB, 0x0002C2, 0x0002D0, 0x0002D2, 0x0002E0 + , 0x0002E5, 0x0002EE, 0x0002EF, 0x000300, 0x000370, 0x000374, 0x000376, 0x00037E + , 0x00037F, 0x000384, 0x000386, 0x000387, 0x000388, 0x0003F6, 0x0003F7, 0x000483 + , 0x00048A, 0x00058A, 0x00058B, 0x00058D, 0x00058F, 0x000590, 0x000591, 0x0005BE + , 0x0005BF, 0x0005C0, 0x0005C1, 0x0005C3, 0x0005C4, 0x0005C6, 0x0005C7, 0x0005C8 + , 0x000600, 0x000606, 0x000608, 0x000609, 0x00060B, 0x00060C, 0x00060D, 0x00060E + , 0x000610, 0x00061B, 0x00064B, 0x000660, 0x00066A, 0x00066B, 0x00066D, 0x000670 + , 0x000671, 0x0006D6, 0x0006DD, 0x0006DE, 0x0006DF, 0x0006E5, 0x0006E7, 0x0006E9 + , 0x0006EA, 0x0006EE, 0x0006F0, 0x0006FA, 0x000711, 0x000712, 0x000730, 0x00074B + , 0x0007A6, 0x0007B1, 0x0007C0, 0x0007EB, 0x0007F4, 0x0007F6, 0x0007FA, 0x0007FD + , 0x0007FE, 0x000816, 0x00081A, 0x00081B, 0x000824, 0x000825, 0x000828, 0x000829 + , 0x00082E, 0x000859, 0x00085C, 0x000860, 0x000890, 0x000892, 0x000897, 0x0008A0 + , 0x0008CA, 0x0008E2, 0x0008E3, 0x000903, 0x00093A, 0x00093B, 0x00093C, 0x00093D + , 0x000941, 0x000949, 0x00094D, 0x00094E, 0x000951, 0x000958, 0x000962, 0x000964 + , 0x000981, 0x000982, 0x0009BC, 0x0009BD, 0x0009C1, 0x0009C5, 0x0009CD, 0x0009CE + , 0x0009E2, 0x0009E4, 0x0009F2, 0x0009F4, 0x0009FB, 0x0009FC, 0x0009FE, 0x0009FF + , 0x000A01, 0x000A03, 0x000A3C, 0x000A3D, 0x000A41, 0x000A43, 0x000A47, 0x000A49 + , 0x000A4B, 0x000A4E, 0x000A51, 0x000A52, 0x000A70, 0x000A72, 0x000A75, 0x000A76 + , 0x000A81, 0x000A83, 0x000ABC, 0x000ABD, 0x000AC1, 0x000AC6, 0x000AC7, 0x000AC9 + , 0x000ACD, 0x000ACE, 0x000AE2, 0x000AE4, 0x000AF1, 0x000AF2, 0x000AFA, 0x000B00 + , 0x000B01, 0x000B02, 0x000B3C, 0x000B3D, 0x000B3F, 0x000B40, 0x000B41, 0x000B45 + , 0x000B4D, 0x000B4E, 0x000B55, 0x000B57, 0x000B62, 0x000B64, 0x000B82, 0x000B83 + , 0x000BC0, 0x000BC1, 0x000BCD, 0x000BCE, 0x000BF3, 0x000BF9, 0x000BFA, 0x000BFB + , 0x000C00, 0x000C01, 0x000C04, 0x000C05, 0x000C3C, 0x000C3D, 0x000C3E, 0x000C41 + , 0x000C46, 0x000C49, 0x000C4A, 0x000C4E, 0x000C55, 0x000C57, 0x000C62, 0x000C64 + , 0x000C78, 0x000C7F, 0x000C81, 0x000C82, 0x000CBC, 0x000CBD, 0x000CCC, 0x000CCE + , 0x000CE2, 0x000CE4, 0x000D00, 0x000D02, 0x000D3B, 0x000D3D, 0x000D41, 0x000D45 + , 0x000D4D, 0x000D4E, 0x000D62, 0x000D64, 0x000D81, 0x000D82, 0x000DCA, 0x000DCB + , 0x000DD2, 0x000DD5, 0x000DD6, 0x000DD7, 0x000E31, 0x000E32, 0x000E34, 0x000E3B + , 0x000E3F, 0x000E40, 0x000E47, 0x000E4F, 0x000EB1, 0x000EB2, 0x000EB4, 0x000EBD + , 0x000EC8, 0x000ECF, 0x000F18, 0x000F1A, 0x000F35, 0x000F36, 0x000F37, 0x000F38 + , 0x000F39, 0x000F3A, 0x000F3E, 0x000F71, 0x000F7F, 0x000F80, 0x000F85, 0x000F86 + , 0x000F88, 0x000F8D, 0x000F98, 0x000F99, 0x000FBD, 0x000FC6, 0x000FC7, 0x00102D + , 0x001031, 0x001032, 0x001038, 0x001039, 0x00103B, 0x00103D, 0x00103F, 0x001058 + , 0x00105A, 0x00105E, 0x001061, 0x001071, 0x001075, 0x001082, 0x001083, 0x001085 + , 0x001087, 0x00108D, 0x00108E, 0x00109D, 0x00109E, 0x00135D, 0x001360, 0x001390 + , 0x00139A, 0x001400, 0x001401, 0x001680, 0x001681, 0x00169B, 0x00169D, 0x001712 + , 0x001715, 0x001732, 0x001734, 0x001752, 0x001754, 0x001772, 0x001774, 0x0017B4 + , 0x0017B6, 0x0017B7, 0x0017BE, 0x0017C6, 0x0017C7, 0x0017C9, 0x0017D4, 0x0017DB + , 0x0017DC, 0x0017DD, 0x0017DE, 0x0017F0, 0x0017FA, 0x001800, 0x00180B, 0x00180E + , 0x00180F, 0x001810, 0x001885, 0x001887, 0x0018A9, 0x0018AA, 0x001920, 0x001923 + , 0x001927, 0x001929, 0x001932, 0x001933, 0x001939, 0x00193C, 0x001940, 0x001941 + , 0x001944, 0x001946, 0x0019DE, 0x001A00, 0x001A17, 0x001A19, 0x001A1B, 0x001A1C + , 0x001A56, 0x001A57, 0x001A58, 0x001A5F, 0x001A60, 0x001A61, 0x001A62, 0x001A63 + , 0x001A65, 0x001A6D, 0x001A73, 0x001A7D, 0x001A7F, 0x001A80, 0x001AB0, 0x001ADE + , 0x001AE0, 0x001AEC, 0x001B00, 0x001B04, 0x001B34, 0x001B35, 0x001B36, 0x001B3B + , 0x001B3C, 0x001B3D, 0x001B42, 0x001B43, 0x001B6B, 0x001B74, 0x001B80, 0x001B82 + , 0x001BA2, 0x001BA6, 0x001BA8, 0x001BAA, 0x001BAB, 0x001BAE, 0x001BE6, 0x001BE7 + , 0x001BE8, 0x001BEA, 0x001BED, 0x001BEE, 0x001BEF, 0x001BF2, 0x001C2C, 0x001C34 + , 0x001C36, 0x001C38, 0x001CD0, 0x001CD3, 0x001CD4, 0x001CE1, 0x001CE2, 0x001CE9 + , 0x001CED, 0x001CEE, 0x001CF4, 0x001CF5, 0x001CF8, 0x001CFA, 0x001DC0, 0x001E00 + , 0x001FBD, 0x001FBE, 0x001FBF, 0x001FC2, 0x001FCD, 0x001FD0, 0x001FDD, 0x001FE0 + , 0x001FED, 0x001FF0, 0x001FFD, 0x001FFF, 0x002000, 0x00200B, 0x00200E, 0x00200F + , 0x002010, 0x002028, 0x00202F, 0x002030, 0x002035, 0x002044, 0x002045, 0x00205F + , 0x002060, 0x002066, 0x00206A, 0x002070, 0x002071, 0x002074, 0x00207A, 0x00207C + , 0x00207F, 0x002080, 0x00208A, 0x00208C, 0x00208F, 0x0020A0, 0x0020D0, 0x0020F1 + , 0x002100, 0x002102, 0x002103, 0x002107, 0x002108, 0x00210A, 0x002114, 0x002115 + , 0x002116, 0x002119, 0x00211E, 0x002124, 0x002125, 0x002126, 0x002127, 0x002128 + , 0x002129, 0x00212A, 0x00212E, 0x00212F, 0x00213A, 0x00213C, 0x002140, 0x002145 + , 0x00214A, 0x00214E, 0x002150, 0x002160, 0x002189, 0x00218C, 0x002190, 0x002212 + , 0x002213, 0x002214, 0x002336, 0x00237B, 0x002395, 0x002396, 0x00242A, 0x002440 + , 0x00244B, 0x002460, 0x002488, 0x00249C, 0x0024EA, 0x0026AC, 0x0026AD, 0x002800 + , 0x002900, 0x002B74, 0x002B76, 0x002C00, 0x002CE5, 0x002CEB, 0x002CEF, 0x002CF2 + , 0x002CF9, 0x002D00, 0x002D7F, 0x002D80, 0x002DE0, 0x002E00, 0x002E5E, 0x002E80 + , 0x002E9A, 0x002E9B, 0x002EF4, 0x002F00, 0x002FD6, 0x002FF0, 0x003000, 0x003001 + , 0x003005, 0x003008, 0x003021, 0x00302A, 0x00302E, 0x003030, 0x003031, 0x003036 + , 0x003038, 0x00303D, 0x003040, 0x003099, 0x00309B, 0x00309D, 0x0030A0, 0x0030A1 + , 0x0030FB, 0x0030FC, 0x0031C0, 0x0031E6, 0x0031EF, 0x0031F0, 0x00321D, 0x00321F + , 0x003250, 0x003260, 0x00327C, 0x00327F, 0x0032B1, 0x0032C0, 0x0032CC, 0x0032D0 + , 0x003377, 0x00337B, 0x0033DE, 0x0033E0, 0x0033FF, 0x003400, 0x004DC0, 0x004E00 + , 0x00A490, 0x00A4C7, 0x00A60D, 0x00A610, 0x00A66F, 0x00A673, 0x00A674, 0x00A67E + , 0x00A680, 0x00A69E, 0x00A6A0, 0x00A6F0, 0x00A6F2, 0x00A700, 0x00A722, 0x00A788 + , 0x00A789, 0x00A802, 0x00A803, 0x00A806, 0x00A807, 0x00A80B, 0x00A80C, 0x00A825 + , 0x00A827, 0x00A828, 0x00A82C, 0x00A82D, 0x00A838, 0x00A83A, 0x00A874, 0x00A878 + , 0x00A8C4, 0x00A8C6, 0x00A8E0, 0x00A8F2, 0x00A8FF, 0x00A900, 0x00A926, 0x00A92E + , 0x00A947, 0x00A952, 0x00A980, 0x00A983, 0x00A9B3, 0x00A9B4, 0x00A9B6, 0x00A9BA + , 0x00A9BC, 0x00A9BE, 0x00A9E5, 0x00A9E6, 0x00AA29, 0x00AA2F, 0x00AA31, 0x00AA33 + , 0x00AA35, 0x00AA37, 0x00AA43, 0x00AA44, 0x00AA4C, 0x00AA4D, 0x00AA7C, 0x00AA7D + , 0x00AAB0, 0x00AAB1, 0x00AAB2, 0x00AAB5, 0x00AAB7, 0x00AAB9, 0x00AABE, 0x00AAC0 + , 0x00AAC1, 0x00AAC2, 0x00AAEC, 0x00AAEE, 0x00AAF6, 0x00AAF7, 0x00AB6A, 0x00AB6C + , 0x00ABE5, 0x00ABE6, 0x00ABE8, 0x00ABE9, 0x00ABED, 0x00ABEE, 0x00FB1D, 0x00FB1E + , 0x00FB1F, 0x00FB29, 0x00FB2A, 0x00FB50, 0x00FBC3, 0x00FBD3, 0x00FD3E, 0x00FD50 + , 0x00FD90, 0x00FD92, 0x00FDC8, 0x00FDD0, 0x00FDF0, 0x00FDFD, 0x00FE00, 0x00FE10 + , 0x00FE1A, 0x00FE20, 0x00FE30, 0x00FE50, 0x00FE51, 0x00FE52, 0x00FE53, 0x00FE54 + , 0x00FE55, 0x00FE56, 0x00FE5F, 0x00FE60, 0x00FE62, 0x00FE64, 0x00FE67, 0x00FE68 + , 0x00FE69, 0x00FE6B, 0x00FE6C, 0x00FE70, 0x00FEFF, 0x00FF00, 0x00FF01, 0x00FF03 + , 0x00FF06, 0x00FF0B, 0x00FF0C, 0x00FF0D, 0x00FF0E, 0x00FF10, 0x00FF1A, 0x00FF1B + , 0x00FF21, 0x00FF3B, 0x00FF41, 0x00FF5B, 0x00FF66, 0x00FFE0, 0x00FFE2, 0x00FFE5 + , 0x00FFE7, 0x00FFE8, 0x00FFEF, 0x00FFF0, 0x00FFF9, 0x00FFFE, 0x010000, 0x010101 + , 0x010102, 0x010140, 0x01018D, 0x010190, 0x01019D, 0x0101A0, 0x0101A1, 0x0101FD + , 0x0101FE, 0x0102E0, 0x0102E1, 0x0102FC, 0x010376, 0x01037B, 0x010800, 0x01091F + , 0x010920, 0x010A01, 0x010A04, 0x010A05, 0x010A07, 0x010A0C, 0x010A10, 0x010A38 + , 0x010A3B, 0x010A3F, 0x010A40, 0x010AE5, 0x010AE7, 0x010B39, 0x010B40, 0x010D00 + , 0x010D24, 0x010D28, 0x010D30, 0x010D3A, 0x010D40, 0x010D4A, 0x010D69, 0x010D6E + , 0x010D6F, 0x010E60, 0x010E7F, 0x010EAB, 0x010EAD, 0x010EC0, 0x010ED0, 0x010ED9 + , 0x010EFA, 0x010F00, 0x010F30, 0x010F46, 0x010F51, 0x010F70, 0x010F82, 0x010F86 + , 0x011000, 0x011001, 0x011002, 0x011038, 0x011047, 0x011052, 0x011066, 0x011070 + , 0x011071, 0x011073, 0x011075, 0x01107F, 0x011082, 0x0110B3, 0x0110B7, 0x0110B9 + , 0x0110BB, 0x0110C2, 0x0110C3, 0x011100, 0x011103, 0x011127, 0x01112C, 0x01112D + , 0x011135, 0x011173, 0x011174, 0x011180, 0x011182, 0x0111B6, 0x0111BF, 0x0111C9 + , 0x0111CD, 0x0111CF, 0x0111D0, 0x01122F, 0x011232, 0x011234, 0x011235, 0x011236 + , 0x011238, 0x01123E, 0x01123F, 0x011241, 0x011242, 0x0112DF, 0x0112E0, 0x0112E3 + , 0x0112EB, 0x011300, 0x011302, 0x01133B, 0x01133D, 0x011340, 0x011341, 0x011366 + , 0x01136D, 0x011370, 0x011375, 0x0113BB, 0x0113C1, 0x0113CE, 0x0113CF, 0x0113D0 + , 0x0113D1, 0x0113D2, 0x0113D3, 0x0113E1, 0x0113E3, 0x011438, 0x011440, 0x011442 + , 0x011445, 0x011446, 0x011447, 0x01145E, 0x01145F, 0x0114B3, 0x0114B9, 0x0114BA + , 0x0114BB, 0x0114BF, 0x0114C1, 0x0114C2, 0x0114C4, 0x0115B2, 0x0115B6, 0x0115BC + , 0x0115BE, 0x0115BF, 0x0115C1, 0x0115DC, 0x0115DE, 0x011633, 0x01163B, 0x01163D + , 0x01163E, 0x01163F, 0x011641, 0x011660, 0x01166D, 0x0116AB, 0x0116AC, 0x0116AD + , 0x0116AE, 0x0116B0, 0x0116B6, 0x0116B7, 0x0116B8, 0x01171D, 0x01171E, 0x01171F + , 0x011720, 0x011722, 0x011726, 0x011727, 0x01172C, 0x01182F, 0x011838, 0x011839 + , 0x01183B, 0x01193B, 0x01193D, 0x01193E, 0x01193F, 0x011943, 0x011944, 0x0119D4 + , 0x0119D8, 0x0119DA, 0x0119DC, 0x0119E0, 0x0119E1, 0x011A01, 0x011A07, 0x011A09 + , 0x011A0B, 0x011A33, 0x011A39, 0x011A3B, 0x011A3F, 0x011A47, 0x011A48, 0x011A51 + , 0x011A57, 0x011A59, 0x011A5C, 0x011A8A, 0x011A97, 0x011A98, 0x011A9A, 0x011B60 + , 0x011B61, 0x011B62, 0x011B65, 0x011B66, 0x011B67, 0x011C30, 0x011C37, 0x011C38 + , 0x011C3E, 0x011C92, 0x011CA8, 0x011CAA, 0x011CB1, 0x011CB2, 0x011CB4, 0x011CB5 + , 0x011CB7, 0x011D31, 0x011D37, 0x011D3A, 0x011D3B, 0x011D3C, 0x011D3E, 0x011D3F + , 0x011D46, 0x011D47, 0x011D48, 0x011D90, 0x011D92, 0x011D95, 0x011D96, 0x011D97 + , 0x011D98, 0x011EF3, 0x011EF5, 0x011F00, 0x011F02, 0x011F36, 0x011F3B, 0x011F40 + , 0x011F41, 0x011F42, 0x011F43, 0x011F5A, 0x011F5B, 0x011FD5, 0x011FDD, 0x011FE1 + , 0x011FF2, 0x013440, 0x013441, 0x013447, 0x013456, 0x01611E, 0x01612A, 0x01612D + , 0x016130, 0x016AF0, 0x016AF5, 0x016B30, 0x016B37, 0x016F4F, 0x016F50, 0x016F8F + , 0x016F93, 0x016FE2, 0x016FE3, 0x016FE4, 0x016FE5, 0x01BC9D, 0x01BC9F, 0x01BCA0 + , 0x01BCA4, 0x01CC00, 0x01CCD6, 0x01CCF0, 0x01CCFA, 0x01CCFD, 0x01CD00, 0x01CEB4 + , 0x01CEBA, 0x01CED1, 0x01CEE0, 0x01CEF1, 0x01CF00, 0x01CF2E, 0x01CF30, 0x01CF47 + , 0x01D167, 0x01D16A, 0x01D173, 0x01D17B, 0x01D183, 0x01D185, 0x01D18C, 0x01D1AA + , 0x01D1AE, 0x01D1E9, 0x01D1EB, 0x01D200, 0x01D242, 0x01D245, 0x01D246, 0x01D300 + , 0x01D357, 0x01D6C1, 0x01D6C2, 0x01D6DB, 0x01D6DC, 0x01D6FB, 0x01D6FC, 0x01D715 + , 0x01D716, 0x01D735, 0x01D736, 0x01D74F, 0x01D750, 0x01D76F, 0x01D770, 0x01D789 + , 0x01D78A, 0x01D7A9, 0x01D7AA, 0x01D7C3, 0x01D7C4, 0x01D7CE, 0x01D800, 0x01DA00 + , 0x01DA37, 0x01DA3B, 0x01DA6D, 0x01DA75, 0x01DA76, 0x01DA84, 0x01DA85, 0x01DA9B + , 0x01DAA0, 0x01DAA1, 0x01DAB0, 0x01E000, 0x01E007, 0x01E008, 0x01E019, 0x01E01B + , 0x01E022, 0x01E023, 0x01E025, 0x01E026, 0x01E02B, 0x01E08F, 0x01E090, 0x01E130 + , 0x01E137, 0x01E2AE, 0x01E2AF, 0x01E2EC, 0x01E2F0, 0x01E2FF, 0x01E300, 0x01E4EC + , 0x01E4F0, 0x01E5EE, 0x01E5F0, 0x01E6E3, 0x01E6E4, 0x01E6E6, 0x01E6E7, 0x01E6EE + , 0x01E6F0, 0x01E6F5, 0x01E6F6, 0x01E800, 0x01E8D0, 0x01E8D7, 0x01E944, 0x01E94B + , 0x01EC70, 0x01ECC0, 0x01ED00, 0x01ED50, 0x01EE00, 0x01EEF0, 0x01EEF2, 0x01EF00 + , 0x01F000, 0x01F02C, 0x01F030, 0x01F094, 0x01F0A0, 0x01F0AF, 0x01F0B1, 0x01F0C0 + , 0x01F0C1, 0x01F0D0, 0x01F0D1, 0x01F0F6, 0x01F100, 0x01F10B, 0x01F110, 0x01F12F + , 0x01F130, 0x01F16A, 0x01F170, 0x01F1AD, 0x01F1AE, 0x01F260, 0x01F266, 0x01F300 + , 0x01F6D9, 0x01F6DC, 0x01F6ED, 0x01F6F0, 0x01F6FD, 0x01F700, 0x01F7DA, 0x01F7E0 + , 0x01F7EC, 0x01F7F0, 0x01F7F1, 0x01F800, 0x01F80C, 0x01F810, 0x01F848, 0x01F850 + , 0x01F85A, 0x01F860, 0x01F888, 0x01F890, 0x01F8AE, 0x01F8B0, 0x01F8BC, 0x01F8C0 + , 0x01F8C2, 0x01F8D0, 0x01F8D9, 0x01F900, 0x01FA58, 0x01FA60, 0x01FA6E, 0x01FA70 + , 0x01FA7D, 0x01FA80, 0x01FA8B, 0x01FA8E, 0x01FAC7, 0x01FAC8, 0x01FAC9, 0x01FACD + , 0x01FADD, 0x01FADF, 0x01FAEB, 0x01FAEF, 0x01FAF9, 0x01FB00, 0x01FB93, 0x01FB94 + , 0x01FBF0, 0x01FBFA, 0x01FBFB, 0x01FFFE, 0x020000, 0x02FFFE, 0x030000, 0x03FFFE + , 0x040000, 0x04FFFE, 0x050000, 0x05FFFE, 0x060000, 0x06FFFE, 0x070000, 0x07FFFE + , 0x080000, 0x08FFFE, 0x090000, 0x09FFFE, 0x0A0000, 0x0AFFFE, 0x0B0000, 0x0BFFFE + , 0x0C0000, 0x0CFFFE, 0x0D0000, 0x0DFFFE, 0x0E0100, 0x0E01F0, 0x0E1000, 0x0EFFFE + , 0x0F0000, 0x0FFFFE, 0x100000, 0x10FFFE + ] + +-- | Class tags; parallel to 'bidiRangeStarts'. +-- +-- * 0 -- L -- Left_To_Right +-- * 1 -- R -- Right_To_Left +-- * 2 -- AL -- Arabic_Letter +-- * 3 -- AN -- Arabic_Number +-- * 4 -- EN -- European_Number +-- * 5 -- ES -- European_Separator +-- * 6 -- CS -- Common_Separator +-- * 7 -- ET -- European_Terminator +-- * 8 -- ON -- Other_Neutral +-- * 9 -- BN -- Boundary_Neutral +-- * 10 -- NSM -- Nonspacing_Mark +-- * 11 -- Other -- everything else (B, S, WS, embedding\/override\/isolated format characters); rejected by the per-label Bidi check. +bidiRangeTags :: ByteArray +bidiRangeTags = PBA.byteArrayFromList @Word8 + [ 9, 11, 9, 11, 8, 7, 8, 5, 6, 5, 6, 4, 6, 8, 0, 8, 0, 8, 9, 11, 9, 6, 8, 7, 8, 0, 8, 9, 8, 7, 4, 8 + , 0, 8, 4, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 10, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 10 + , 0, 8, 0, 8, 7, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 3, 8, 2, 7, 2, 6, 2, 8, 10, 2, 10, 3, 7, 3, 2, 10 + , 2, 10, 3, 8, 10, 2, 10, 8, 10, 2, 4, 2, 10, 2, 10, 2, 10, 2, 1, 10, 1, 8, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10 + , 1, 10, 1, 2, 3, 2, 10, 2, 10, 3, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 7, 0, 7, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 7, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 7, 8, 0 + , 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 7, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 8, 0, 8, 0, 11, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 7 + , 0, 10, 0, 8, 0, 8, 10, 9, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 0, 8, 0, 8, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 11, 9, 0, 1, 8, 11, 6, 7, 8, 6, 8, 11, 9, 11, 9, 4, 0, 4, 5, 8 + , 0, 4, 5, 8, 0, 7, 10, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 7, 0, 8, 0, 8, 0 + , 8, 0, 8, 0, 8, 0, 8, 5, 7, 8, 0, 8, 0, 8, 0, 8, 0, 8, 4, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 10, 0 + , 8, 0, 10, 0, 10, 8, 0, 8, 0, 8, 0, 8, 0, 8, 11, 8, 0, 8, 0, 10, 0, 8, 0, 8, 0, 8, 0, 10, 8, 0, 8, 0 + , 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 10, 8, 10, 8 + , 0, 10, 0, 10, 0, 8, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 10, 0, 7, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0 + , 10, 0, 10, 0, 10, 0, 8, 0, 10, 0, 10, 0, 10, 0, 1, 10, 1, 5, 1, 2, 8, 2, 8, 2, 8, 2, 8, 9, 2, 8, 10, 8 + , 0, 10, 8, 6, 8, 6, 0, 8, 6, 8, 7, 8, 5, 8, 0, 8, 7, 8, 0, 2, 9, 0, 8, 7, 8, 5, 6, 5, 6, 4, 6, 8 + , 0, 8, 0, 8, 0, 7, 8, 7, 0, 8, 0, 9, 8, 9, 0, 8, 0, 8, 0, 8, 0, 8, 0, 10, 0, 10, 4, 0, 10, 0, 1, 8 + , 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 8, 1, 2, 10, 2, 3, 2, 3, 1, 10, 8, 1, 3, 1, 10, 1, 2, 8, 2 + , 10, 1, 2, 10, 2, 1, 10, 1, 0, 10, 0, 10, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 8, 7, 8, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 + , 0, 8, 0, 10, 0, 10, 0, 9, 0, 8, 0, 4, 8, 0, 8, 0, 8, 0, 8, 0, 10, 0, 10, 0, 10, 0, 9, 10, 0, 10, 0, 10 + , 0, 8, 0, 8, 10, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 7, 0, 10 + , 0, 10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 1, 10, 1, 10, 1, 2, 1, 2, 1, 2, 8, 2, 1, 8, 0, 8, 0, 8, 0, 8, 0 + , 8, 0, 8, 0, 4, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 + , 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 + , 4, 8, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 10, 9, 0, 9 + , 0, 9, 0, 9 + ] diff --git a/internal/Text/IDNA2008/Internal/Case.hs b/internal/Text/IDNA2008/Internal/Case.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Case.hs @@ -0,0 +1,31 @@ +-- | +-- Module : Text.IDNA2008.Internal.Case +-- Description : Unicode simple case folding for MAPCASE. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Simple (1:1) Unicode lowercase mapping pinned to the UCD version +-- this library tracks (see @x-unicode-version@ in @idna2008.cabal@, +-- and re-run @internal/tools/update@ to advance). +-- +-- This is the case folder consulted by the @MAPCASE@ input mapping. +-- Compared to "Data.Char"'s @toLower@, the behaviour is the same in +-- shape — @Char -> Char@, single-codepoint output, no language or +-- locale-specific exceptions — but the table is fixed by this +-- library's UCD pin rather than by whatever @base@ happens to ship, +-- so cased letters introduced in Unicode releases newer than @base@ +-- still fold correctly. +-- +-- The @Char -> Char@ signature is the spec: ligatures (e.g. +-- @U+0132@) and digraphs (e.g. @U+01C4@) stay as single +-- codepoints, mapping to the corresponding lower case form; no +-- decomposition into multi-letter sequences happens here. + +module Text.IDNA2008.Internal.Case + ( toLower + ) where + +import Text.IDNA2008.Internal.Case.Data (toLower) diff --git a/internal/Text/IDNA2008/Internal/Case/Data.hs b/internal/Text/IDNA2008/Internal/Case/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Case/Data.hs @@ -0,0 +1,1528 @@ +-- DO NOT EDIT: This file is automatically generated by +-- internal/tools/genIdnaSimpleLower.py, with data from: +-- https://www.unicode.org/Public/17.0.0/ucd/UnicodeData.txt +-- +-- Pinned to Unicode 17.0.0; bump @x-unicode-version@ in +-- @idna2008.cabal@ and re-run @internal/tools/update@ to advance. + +{-# LANGUAGE NoImplicitPrelude, LambdaCase #-} +{-# OPTIONS_HADDOCK hide #-} + +----------------------------------------------------------------------------- +-- | +-- Module : Text.IDNA2008.Internal.Case.Data +-- Description : Unicode @Simple_Lowercase_Mapping@ table for the +-- UCD version pinned by the @idna2008@ library. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaSimpleLower.py \\ +-- +-- +-- Aligned with: Unicode 17.0.0 +-- Mapping entries: 1488 +----------------------------------------------------------------------------- + +module Text.IDNA2008.Internal.Case.Data +(toLower) +where + +import Prelude (Char) + +{-# NOINLINE toLower #-} +toLower :: Char -> Char +toLower = \case + '\x41' -> '\x61' + '\x42' -> '\x62' + '\x43' -> '\x63' + '\x44' -> '\x64' + '\x45' -> '\x65' + '\x46' -> '\x66' + '\x47' -> '\x67' + '\x48' -> '\x68' + '\x49' -> '\x69' + '\x4a' -> '\x6a' + '\x4b' -> '\x6b' + '\x4c' -> '\x6c' + '\x4d' -> '\x6d' + '\x4e' -> '\x6e' + '\x4f' -> '\x6f' + '\x50' -> '\x70' + '\x51' -> '\x71' + '\x52' -> '\x72' + '\x53' -> '\x73' + '\x54' -> '\x74' + '\x55' -> '\x75' + '\x56' -> '\x76' + '\x57' -> '\x77' + '\x58' -> '\x78' + '\x59' -> '\x79' + '\x5a' -> '\x7a' + '\xc0' -> '\xe0' + '\xc1' -> '\xe1' + '\xc2' -> '\xe2' + '\xc3' -> '\xe3' + '\xc4' -> '\xe4' + '\xc5' -> '\xe5' + '\xc6' -> '\xe6' + '\xc7' -> '\xe7' + '\xc8' -> '\xe8' + '\xc9' -> '\xe9' + '\xca' -> '\xea' + '\xcb' -> '\xeb' + '\xcc' -> '\xec' + '\xcd' -> '\xed' + '\xce' -> '\xee' + '\xcf' -> '\xef' + '\xd0' -> '\xf0' + '\xd1' -> '\xf1' + '\xd2' -> '\xf2' + '\xd3' -> '\xf3' + '\xd4' -> '\xf4' + '\xd5' -> '\xf5' + '\xd6' -> '\xf6' + '\xd8' -> '\xf8' + '\xd9' -> '\xf9' + '\xda' -> '\xfa' + '\xdb' -> '\xfb' + '\xdc' -> '\xfc' + '\xdd' -> '\xfd' + '\xde' -> '\xfe' + '\x100' -> '\x101' + '\x102' -> '\x103' + '\x104' -> '\x105' + '\x106' -> '\x107' + '\x108' -> '\x109' + '\x10a' -> '\x10b' + '\x10c' -> '\x10d' + '\x10e' -> '\x10f' + '\x110' -> '\x111' + '\x112' -> '\x113' + '\x114' -> '\x115' + '\x116' -> '\x117' + '\x118' -> '\x119' + '\x11a' -> '\x11b' + '\x11c' -> '\x11d' + '\x11e' -> '\x11f' + '\x120' -> '\x121' + '\x122' -> '\x123' + '\x124' -> '\x125' + '\x126' -> '\x127' + '\x128' -> '\x129' + '\x12a' -> '\x12b' + '\x12c' -> '\x12d' + '\x12e' -> '\x12f' + '\x130' -> '\x69' + '\x132' -> '\x133' + '\x134' -> '\x135' + '\x136' -> '\x137' + '\x139' -> '\x13a' + '\x13b' -> '\x13c' + '\x13d' -> '\x13e' + '\x13f' -> '\x140' + '\x141' -> '\x142' + '\x143' -> '\x144' + '\x145' -> '\x146' + '\x147' -> '\x148' + '\x14a' -> '\x14b' + '\x14c' -> '\x14d' + '\x14e' -> '\x14f' + '\x150' -> '\x151' + '\x152' -> '\x153' + '\x154' -> '\x155' + '\x156' -> '\x157' + '\x158' -> '\x159' + '\x15a' -> '\x15b' + '\x15c' -> '\x15d' + '\x15e' -> '\x15f' + '\x160' -> '\x161' + '\x162' -> '\x163' + '\x164' -> '\x165' + '\x166' -> '\x167' + '\x168' -> '\x169' + '\x16a' -> '\x16b' + '\x16c' -> '\x16d' + '\x16e' -> '\x16f' + '\x170' -> '\x171' + '\x172' -> '\x173' + '\x174' -> '\x175' + '\x176' -> '\x177' + '\x178' -> '\xff' + '\x179' -> '\x17a' + '\x17b' -> '\x17c' + '\x17d' -> '\x17e' + '\x181' -> '\x253' + '\x182' -> '\x183' + '\x184' -> '\x185' + '\x186' -> '\x254' + '\x187' -> '\x188' + '\x189' -> '\x256' + '\x18a' -> '\x257' + '\x18b' -> '\x18c' + '\x18e' -> '\x1dd' + '\x18f' -> '\x259' + '\x190' -> '\x25b' + '\x191' -> '\x192' + '\x193' -> '\x260' + '\x194' -> '\x263' + '\x196' -> '\x269' + '\x197' -> '\x268' + '\x198' -> '\x199' + '\x19c' -> '\x26f' + '\x19d' -> '\x272' + '\x19f' -> '\x275' + '\x1a0' -> '\x1a1' + '\x1a2' -> '\x1a3' + '\x1a4' -> '\x1a5' + '\x1a6' -> '\x280' + '\x1a7' -> '\x1a8' + '\x1a9' -> '\x283' + '\x1ac' -> '\x1ad' + '\x1ae' -> '\x288' + '\x1af' -> '\x1b0' + '\x1b1' -> '\x28a' + '\x1b2' -> '\x28b' + '\x1b3' -> '\x1b4' + '\x1b5' -> '\x1b6' + '\x1b7' -> '\x292' + '\x1b8' -> '\x1b9' + '\x1bc' -> '\x1bd' + '\x1c4' -> '\x1c6' + '\x1c5' -> '\x1c6' + '\x1c7' -> '\x1c9' + '\x1c8' -> '\x1c9' + '\x1ca' -> '\x1cc' + '\x1cb' -> '\x1cc' + '\x1cd' -> '\x1ce' + '\x1cf' -> '\x1d0' + '\x1d1' -> '\x1d2' + '\x1d3' -> '\x1d4' + '\x1d5' -> '\x1d6' + '\x1d7' -> '\x1d8' + '\x1d9' -> '\x1da' + '\x1db' -> '\x1dc' + '\x1de' -> '\x1df' + '\x1e0' -> '\x1e1' + '\x1e2' -> '\x1e3' + '\x1e4' -> '\x1e5' + '\x1e6' -> '\x1e7' + '\x1e8' -> '\x1e9' + '\x1ea' -> '\x1eb' + '\x1ec' -> '\x1ed' + '\x1ee' -> '\x1ef' + '\x1f1' -> '\x1f3' + '\x1f2' -> '\x1f3' + '\x1f4' -> '\x1f5' + '\x1f6' -> '\x195' + '\x1f7' -> '\x1bf' + '\x1f8' -> '\x1f9' + '\x1fa' -> '\x1fb' + '\x1fc' -> '\x1fd' + '\x1fe' -> '\x1ff' + '\x200' -> '\x201' + '\x202' -> '\x203' + '\x204' -> '\x205' + '\x206' -> '\x207' + '\x208' -> '\x209' + '\x20a' -> '\x20b' + '\x20c' -> '\x20d' + '\x20e' -> '\x20f' + '\x210' -> '\x211' + '\x212' -> '\x213' + '\x214' -> '\x215' + '\x216' -> '\x217' + '\x218' -> '\x219' + '\x21a' -> '\x21b' + '\x21c' -> '\x21d' + '\x21e' -> '\x21f' + '\x220' -> '\x19e' + '\x222' -> '\x223' + '\x224' -> '\x225' + '\x226' -> '\x227' + '\x228' -> '\x229' + '\x22a' -> '\x22b' + '\x22c' -> '\x22d' + '\x22e' -> '\x22f' + '\x230' -> '\x231' + '\x232' -> '\x233' + '\x23a' -> '\x2c65' + '\x23b' -> '\x23c' + '\x23d' -> '\x19a' + '\x23e' -> '\x2c66' + '\x241' -> '\x242' + '\x243' -> '\x180' + '\x244' -> '\x289' + '\x245' -> '\x28c' + '\x246' -> '\x247' + '\x248' -> '\x249' + '\x24a' -> '\x24b' + '\x24c' -> '\x24d' + '\x24e' -> '\x24f' + '\x370' -> '\x371' + '\x372' -> '\x373' + '\x376' -> '\x377' + '\x37f' -> '\x3f3' + '\x386' -> '\x3ac' + '\x388' -> '\x3ad' + '\x389' -> '\x3ae' + '\x38a' -> '\x3af' + '\x38c' -> '\x3cc' + '\x38e' -> '\x3cd' + '\x38f' -> '\x3ce' + '\x391' -> '\x3b1' + '\x392' -> '\x3b2' + '\x393' -> '\x3b3' + '\x394' -> '\x3b4' + '\x395' -> '\x3b5' + '\x396' -> '\x3b6' + '\x397' -> '\x3b7' + '\x398' -> '\x3b8' + '\x399' -> '\x3b9' + '\x39a' -> '\x3ba' + '\x39b' -> '\x3bb' + '\x39c' -> '\x3bc' + '\x39d' -> '\x3bd' + '\x39e' -> '\x3be' + '\x39f' -> '\x3bf' + '\x3a0' -> '\x3c0' + '\x3a1' -> '\x3c1' + '\x3a3' -> '\x3c3' + '\x3a4' -> '\x3c4' + '\x3a5' -> '\x3c5' + '\x3a6' -> '\x3c6' + '\x3a7' -> '\x3c7' + '\x3a8' -> '\x3c8' + '\x3a9' -> '\x3c9' + '\x3aa' -> '\x3ca' + '\x3ab' -> '\x3cb' + '\x3cf' -> '\x3d7' + '\x3d8' -> '\x3d9' + '\x3da' -> '\x3db' + '\x3dc' -> '\x3dd' + '\x3de' -> '\x3df' + '\x3e0' -> '\x3e1' + '\x3e2' -> '\x3e3' + '\x3e4' -> '\x3e5' + '\x3e6' -> '\x3e7' + '\x3e8' -> '\x3e9' + '\x3ea' -> '\x3eb' + '\x3ec' -> '\x3ed' + '\x3ee' -> '\x3ef' + '\x3f4' -> '\x3b8' + '\x3f7' -> '\x3f8' + '\x3f9' -> '\x3f2' + '\x3fa' -> '\x3fb' + '\x3fd' -> '\x37b' + '\x3fe' -> '\x37c' + '\x3ff' -> '\x37d' + '\x400' -> '\x450' + '\x401' -> '\x451' + '\x402' -> '\x452' + '\x403' -> '\x453' + '\x404' -> '\x454' + '\x405' -> '\x455' + '\x406' -> '\x456' + '\x407' -> '\x457' + '\x408' -> '\x458' + '\x409' -> '\x459' + '\x40a' -> '\x45a' + '\x40b' -> '\x45b' + '\x40c' -> '\x45c' + '\x40d' -> '\x45d' + '\x40e' -> '\x45e' + '\x40f' -> '\x45f' + '\x410' -> '\x430' + '\x411' -> '\x431' + '\x412' -> '\x432' + '\x413' -> '\x433' + '\x414' -> '\x434' + '\x415' -> '\x435' + '\x416' -> '\x436' + '\x417' -> '\x437' + '\x418' -> '\x438' + '\x419' -> '\x439' + '\x41a' -> '\x43a' + '\x41b' -> '\x43b' + '\x41c' -> '\x43c' + '\x41d' -> '\x43d' + '\x41e' -> '\x43e' + '\x41f' -> '\x43f' + '\x420' -> '\x440' + '\x421' -> '\x441' + '\x422' -> '\x442' + '\x423' -> '\x443' + '\x424' -> '\x444' + '\x425' -> '\x445' + '\x426' -> '\x446' + '\x427' -> '\x447' + '\x428' -> '\x448' + '\x429' -> '\x449' + '\x42a' -> '\x44a' + '\x42b' -> '\x44b' + '\x42c' -> '\x44c' + '\x42d' -> '\x44d' + '\x42e' -> '\x44e' + '\x42f' -> '\x44f' + '\x460' -> '\x461' + '\x462' -> '\x463' + '\x464' -> '\x465' + '\x466' -> '\x467' + '\x468' -> '\x469' + '\x46a' -> '\x46b' + '\x46c' -> '\x46d' + '\x46e' -> '\x46f' + '\x470' -> '\x471' + '\x472' -> '\x473' + '\x474' -> '\x475' + '\x476' -> '\x477' + '\x478' -> '\x479' + '\x47a' -> '\x47b' + '\x47c' -> '\x47d' + '\x47e' -> '\x47f' + '\x480' -> '\x481' + '\x48a' -> '\x48b' + '\x48c' -> '\x48d' + '\x48e' -> '\x48f' + '\x490' -> '\x491' + '\x492' -> '\x493' + '\x494' -> '\x495' + '\x496' -> '\x497' + '\x498' -> '\x499' + '\x49a' -> '\x49b' + '\x49c' -> '\x49d' + '\x49e' -> '\x49f' + '\x4a0' -> '\x4a1' + '\x4a2' -> '\x4a3' + '\x4a4' -> '\x4a5' + '\x4a6' -> '\x4a7' + '\x4a8' -> '\x4a9' + '\x4aa' -> '\x4ab' + '\x4ac' -> '\x4ad' + '\x4ae' -> '\x4af' + '\x4b0' -> '\x4b1' + '\x4b2' -> '\x4b3' + '\x4b4' -> '\x4b5' + '\x4b6' -> '\x4b7' + '\x4b8' -> '\x4b9' + '\x4ba' -> '\x4bb' + '\x4bc' -> '\x4bd' + '\x4be' -> '\x4bf' + '\x4c0' -> '\x4cf' + '\x4c1' -> '\x4c2' + '\x4c3' -> '\x4c4' + '\x4c5' -> '\x4c6' + '\x4c7' -> '\x4c8' + '\x4c9' -> '\x4ca' + '\x4cb' -> '\x4cc' + '\x4cd' -> '\x4ce' + '\x4d0' -> '\x4d1' + '\x4d2' -> '\x4d3' + '\x4d4' -> '\x4d5' + '\x4d6' -> '\x4d7' + '\x4d8' -> '\x4d9' + '\x4da' -> '\x4db' + '\x4dc' -> '\x4dd' + '\x4de' -> '\x4df' + '\x4e0' -> '\x4e1' + '\x4e2' -> '\x4e3' + '\x4e4' -> '\x4e5' + '\x4e6' -> '\x4e7' + '\x4e8' -> '\x4e9' + '\x4ea' -> '\x4eb' + '\x4ec' -> '\x4ed' + '\x4ee' -> '\x4ef' + '\x4f0' -> '\x4f1' + '\x4f2' -> '\x4f3' + '\x4f4' -> '\x4f5' + '\x4f6' -> '\x4f7' + '\x4f8' -> '\x4f9' + '\x4fa' -> '\x4fb' + '\x4fc' -> '\x4fd' + '\x4fe' -> '\x4ff' + '\x500' -> '\x501' + '\x502' -> '\x503' + '\x504' -> '\x505' + '\x506' -> '\x507' + '\x508' -> '\x509' + '\x50a' -> '\x50b' + '\x50c' -> '\x50d' + '\x50e' -> '\x50f' + '\x510' -> '\x511' + '\x512' -> '\x513' + '\x514' -> '\x515' + '\x516' -> '\x517' + '\x518' -> '\x519' + '\x51a' -> '\x51b' + '\x51c' -> '\x51d' + '\x51e' -> '\x51f' + '\x520' -> '\x521' + '\x522' -> '\x523' + '\x524' -> '\x525' + '\x526' -> '\x527' + '\x528' -> '\x529' + '\x52a' -> '\x52b' + '\x52c' -> '\x52d' + '\x52e' -> '\x52f' + '\x531' -> '\x561' + '\x532' -> '\x562' + '\x533' -> '\x563' + '\x534' -> '\x564' + '\x535' -> '\x565' + '\x536' -> '\x566' + '\x537' -> '\x567' + '\x538' -> '\x568' + '\x539' -> '\x569' + '\x53a' -> '\x56a' + '\x53b' -> '\x56b' + '\x53c' -> '\x56c' + '\x53d' -> '\x56d' + '\x53e' -> '\x56e' + '\x53f' -> '\x56f' + '\x540' -> '\x570' + '\x541' -> '\x571' + '\x542' -> '\x572' + '\x543' -> '\x573' + '\x544' -> '\x574' + '\x545' -> '\x575' + '\x546' -> '\x576' + '\x547' -> '\x577' + '\x548' -> '\x578' + '\x549' -> '\x579' + '\x54a' -> '\x57a' + '\x54b' -> '\x57b' + '\x54c' -> '\x57c' + '\x54d' -> '\x57d' + '\x54e' -> '\x57e' + '\x54f' -> '\x57f' + '\x550' -> '\x580' + '\x551' -> '\x581' + '\x552' -> '\x582' + '\x553' -> '\x583' + '\x554' -> '\x584' + '\x555' -> '\x585' + '\x556' -> '\x586' + '\x10a0' -> '\x2d00' + '\x10a1' -> '\x2d01' + '\x10a2' -> '\x2d02' + '\x10a3' -> '\x2d03' + '\x10a4' -> '\x2d04' + '\x10a5' -> '\x2d05' + '\x10a6' -> '\x2d06' + '\x10a7' -> '\x2d07' + '\x10a8' -> '\x2d08' + '\x10a9' -> '\x2d09' + '\x10aa' -> '\x2d0a' + '\x10ab' -> '\x2d0b' + '\x10ac' -> '\x2d0c' + '\x10ad' -> '\x2d0d' + '\x10ae' -> '\x2d0e' + '\x10af' -> '\x2d0f' + '\x10b0' -> '\x2d10' + '\x10b1' -> '\x2d11' + '\x10b2' -> '\x2d12' + '\x10b3' -> '\x2d13' + '\x10b4' -> '\x2d14' + '\x10b5' -> '\x2d15' + '\x10b6' -> '\x2d16' + '\x10b7' -> '\x2d17' + '\x10b8' -> '\x2d18' + '\x10b9' -> '\x2d19' + '\x10ba' -> '\x2d1a' + '\x10bb' -> '\x2d1b' + '\x10bc' -> '\x2d1c' + '\x10bd' -> '\x2d1d' + '\x10be' -> '\x2d1e' + '\x10bf' -> '\x2d1f' + '\x10c0' -> '\x2d20' + '\x10c1' -> '\x2d21' + '\x10c2' -> '\x2d22' + '\x10c3' -> '\x2d23' + '\x10c4' -> '\x2d24' + '\x10c5' -> '\x2d25' + '\x10c7' -> '\x2d27' + '\x10cd' -> '\x2d2d' + '\x13a0' -> '\xab70' + '\x13a1' -> '\xab71' + '\x13a2' -> '\xab72' + '\x13a3' -> '\xab73' + '\x13a4' -> '\xab74' + '\x13a5' -> '\xab75' + '\x13a6' -> '\xab76' + '\x13a7' -> '\xab77' + '\x13a8' -> '\xab78' + '\x13a9' -> '\xab79' + '\x13aa' -> '\xab7a' + '\x13ab' -> '\xab7b' + '\x13ac' -> '\xab7c' + '\x13ad' -> '\xab7d' + '\x13ae' -> '\xab7e' + '\x13af' -> '\xab7f' + '\x13b0' -> '\xab80' + '\x13b1' -> '\xab81' + '\x13b2' -> '\xab82' + '\x13b3' -> '\xab83' + '\x13b4' -> '\xab84' + '\x13b5' -> '\xab85' + '\x13b6' -> '\xab86' + '\x13b7' -> '\xab87' + '\x13b8' -> '\xab88' + '\x13b9' -> '\xab89' + '\x13ba' -> '\xab8a' + '\x13bb' -> '\xab8b' + '\x13bc' -> '\xab8c' + '\x13bd' -> '\xab8d' + '\x13be' -> '\xab8e' + '\x13bf' -> '\xab8f' + '\x13c0' -> '\xab90' + '\x13c1' -> '\xab91' + '\x13c2' -> '\xab92' + '\x13c3' -> '\xab93' + '\x13c4' -> '\xab94' + '\x13c5' -> '\xab95' + '\x13c6' -> '\xab96' + '\x13c7' -> '\xab97' + '\x13c8' -> '\xab98' + '\x13c9' -> '\xab99' + '\x13ca' -> '\xab9a' + '\x13cb' -> '\xab9b' + '\x13cc' -> '\xab9c' + '\x13cd' -> '\xab9d' + '\x13ce' -> '\xab9e' + '\x13cf' -> '\xab9f' + '\x13d0' -> '\xaba0' + '\x13d1' -> '\xaba1' + '\x13d2' -> '\xaba2' + '\x13d3' -> '\xaba3' + '\x13d4' -> '\xaba4' + '\x13d5' -> '\xaba5' + '\x13d6' -> '\xaba6' + '\x13d7' -> '\xaba7' + '\x13d8' -> '\xaba8' + '\x13d9' -> '\xaba9' + '\x13da' -> '\xabaa' + '\x13db' -> '\xabab' + '\x13dc' -> '\xabac' + '\x13dd' -> '\xabad' + '\x13de' -> '\xabae' + '\x13df' -> '\xabaf' + '\x13e0' -> '\xabb0' + '\x13e1' -> '\xabb1' + '\x13e2' -> '\xabb2' + '\x13e3' -> '\xabb3' + '\x13e4' -> '\xabb4' + '\x13e5' -> '\xabb5' + '\x13e6' -> '\xabb6' + '\x13e7' -> '\xabb7' + '\x13e8' -> '\xabb8' + '\x13e9' -> '\xabb9' + '\x13ea' -> '\xabba' + '\x13eb' -> '\xabbb' + '\x13ec' -> '\xabbc' + '\x13ed' -> '\xabbd' + '\x13ee' -> '\xabbe' + '\x13ef' -> '\xabbf' + '\x13f0' -> '\x13f8' + '\x13f1' -> '\x13f9' + '\x13f2' -> '\x13fa' + '\x13f3' -> '\x13fb' + '\x13f4' -> '\x13fc' + '\x13f5' -> '\x13fd' + '\x1c89' -> '\x1c8a' + '\x1c90' -> '\x10d0' + '\x1c91' -> '\x10d1' + '\x1c92' -> '\x10d2' + '\x1c93' -> '\x10d3' + '\x1c94' -> '\x10d4' + '\x1c95' -> '\x10d5' + '\x1c96' -> '\x10d6' + '\x1c97' -> '\x10d7' + '\x1c98' -> '\x10d8' + '\x1c99' -> '\x10d9' + '\x1c9a' -> '\x10da' + '\x1c9b' -> '\x10db' + '\x1c9c' -> '\x10dc' + '\x1c9d' -> '\x10dd' + '\x1c9e' -> '\x10de' + '\x1c9f' -> '\x10df' + '\x1ca0' -> '\x10e0' + '\x1ca1' -> '\x10e1' + '\x1ca2' -> '\x10e2' + '\x1ca3' -> '\x10e3' + '\x1ca4' -> '\x10e4' + '\x1ca5' -> '\x10e5' + '\x1ca6' -> '\x10e6' + '\x1ca7' -> '\x10e7' + '\x1ca8' -> '\x10e8' + '\x1ca9' -> '\x10e9' + '\x1caa' -> '\x10ea' + '\x1cab' -> '\x10eb' + '\x1cac' -> '\x10ec' + '\x1cad' -> '\x10ed' + '\x1cae' -> '\x10ee' + '\x1caf' -> '\x10ef' + '\x1cb0' -> '\x10f0' + '\x1cb1' -> '\x10f1' + '\x1cb2' -> '\x10f2' + '\x1cb3' -> '\x10f3' + '\x1cb4' -> '\x10f4' + '\x1cb5' -> '\x10f5' + '\x1cb6' -> '\x10f6' + '\x1cb7' -> '\x10f7' + '\x1cb8' -> '\x10f8' + '\x1cb9' -> '\x10f9' + '\x1cba' -> '\x10fa' + '\x1cbd' -> '\x10fd' + '\x1cbe' -> '\x10fe' + '\x1cbf' -> '\x10ff' + '\x1e00' -> '\x1e01' + '\x1e02' -> '\x1e03' + '\x1e04' -> '\x1e05' + '\x1e06' -> '\x1e07' + '\x1e08' -> '\x1e09' + '\x1e0a' -> '\x1e0b' + '\x1e0c' -> '\x1e0d' + '\x1e0e' -> '\x1e0f' + '\x1e10' -> '\x1e11' + '\x1e12' -> '\x1e13' + '\x1e14' -> '\x1e15' + '\x1e16' -> '\x1e17' + '\x1e18' -> '\x1e19' + '\x1e1a' -> '\x1e1b' + '\x1e1c' -> '\x1e1d' + '\x1e1e' -> '\x1e1f' + '\x1e20' -> '\x1e21' + '\x1e22' -> '\x1e23' + '\x1e24' -> '\x1e25' + '\x1e26' -> '\x1e27' + '\x1e28' -> '\x1e29' + '\x1e2a' -> '\x1e2b' + '\x1e2c' -> '\x1e2d' + '\x1e2e' -> '\x1e2f' + '\x1e30' -> '\x1e31' + '\x1e32' -> '\x1e33' + '\x1e34' -> '\x1e35' + '\x1e36' -> '\x1e37' + '\x1e38' -> '\x1e39' + '\x1e3a' -> '\x1e3b' + '\x1e3c' -> '\x1e3d' + '\x1e3e' -> '\x1e3f' + '\x1e40' -> '\x1e41' + '\x1e42' -> '\x1e43' + '\x1e44' -> '\x1e45' + '\x1e46' -> '\x1e47' + '\x1e48' -> '\x1e49' + '\x1e4a' -> '\x1e4b' + '\x1e4c' -> '\x1e4d' + '\x1e4e' -> '\x1e4f' + '\x1e50' -> '\x1e51' + '\x1e52' -> '\x1e53' + '\x1e54' -> '\x1e55' + '\x1e56' -> '\x1e57' + '\x1e58' -> '\x1e59' + '\x1e5a' -> '\x1e5b' + '\x1e5c' -> '\x1e5d' + '\x1e5e' -> '\x1e5f' + '\x1e60' -> '\x1e61' + '\x1e62' -> '\x1e63' + '\x1e64' -> '\x1e65' + '\x1e66' -> '\x1e67' + '\x1e68' -> '\x1e69' + '\x1e6a' -> '\x1e6b' + '\x1e6c' -> '\x1e6d' + '\x1e6e' -> '\x1e6f' + '\x1e70' -> '\x1e71' + '\x1e72' -> '\x1e73' + '\x1e74' -> '\x1e75' + '\x1e76' -> '\x1e77' + '\x1e78' -> '\x1e79' + '\x1e7a' -> '\x1e7b' + '\x1e7c' -> '\x1e7d' + '\x1e7e' -> '\x1e7f' + '\x1e80' -> '\x1e81' + '\x1e82' -> '\x1e83' + '\x1e84' -> '\x1e85' + '\x1e86' -> '\x1e87' + '\x1e88' -> '\x1e89' + '\x1e8a' -> '\x1e8b' + '\x1e8c' -> '\x1e8d' + '\x1e8e' -> '\x1e8f' + '\x1e90' -> '\x1e91' + '\x1e92' -> '\x1e93' + '\x1e94' -> '\x1e95' + '\x1e9e' -> '\xdf' + '\x1ea0' -> '\x1ea1' + '\x1ea2' -> '\x1ea3' + '\x1ea4' -> '\x1ea5' + '\x1ea6' -> '\x1ea7' + '\x1ea8' -> '\x1ea9' + '\x1eaa' -> '\x1eab' + '\x1eac' -> '\x1ead' + '\x1eae' -> '\x1eaf' + '\x1eb0' -> '\x1eb1' + '\x1eb2' -> '\x1eb3' + '\x1eb4' -> '\x1eb5' + '\x1eb6' -> '\x1eb7' + '\x1eb8' -> '\x1eb9' + '\x1eba' -> '\x1ebb' + '\x1ebc' -> '\x1ebd' + '\x1ebe' -> '\x1ebf' + '\x1ec0' -> '\x1ec1' + '\x1ec2' -> '\x1ec3' + '\x1ec4' -> '\x1ec5' + '\x1ec6' -> '\x1ec7' + '\x1ec8' -> '\x1ec9' + '\x1eca' -> '\x1ecb' + '\x1ecc' -> '\x1ecd' + '\x1ece' -> '\x1ecf' + '\x1ed0' -> '\x1ed1' + '\x1ed2' -> '\x1ed3' + '\x1ed4' -> '\x1ed5' + '\x1ed6' -> '\x1ed7' + '\x1ed8' -> '\x1ed9' + '\x1eda' -> '\x1edb' + '\x1edc' -> '\x1edd' + '\x1ede' -> '\x1edf' + '\x1ee0' -> '\x1ee1' + '\x1ee2' -> '\x1ee3' + '\x1ee4' -> '\x1ee5' + '\x1ee6' -> '\x1ee7' + '\x1ee8' -> '\x1ee9' + '\x1eea' -> '\x1eeb' + '\x1eec' -> '\x1eed' + '\x1eee' -> '\x1eef' + '\x1ef0' -> '\x1ef1' + '\x1ef2' -> '\x1ef3' + '\x1ef4' -> '\x1ef5' + '\x1ef6' -> '\x1ef7' + '\x1ef8' -> '\x1ef9' + '\x1efa' -> '\x1efb' + '\x1efc' -> '\x1efd' + '\x1efe' -> '\x1eff' + '\x1f08' -> '\x1f00' + '\x1f09' -> '\x1f01' + '\x1f0a' -> '\x1f02' + '\x1f0b' -> '\x1f03' + '\x1f0c' -> '\x1f04' + '\x1f0d' -> '\x1f05' + '\x1f0e' -> '\x1f06' + '\x1f0f' -> '\x1f07' + '\x1f18' -> '\x1f10' + '\x1f19' -> '\x1f11' + '\x1f1a' -> '\x1f12' + '\x1f1b' -> '\x1f13' + '\x1f1c' -> '\x1f14' + '\x1f1d' -> '\x1f15' + '\x1f28' -> '\x1f20' + '\x1f29' -> '\x1f21' + '\x1f2a' -> '\x1f22' + '\x1f2b' -> '\x1f23' + '\x1f2c' -> '\x1f24' + '\x1f2d' -> '\x1f25' + '\x1f2e' -> '\x1f26' + '\x1f2f' -> '\x1f27' + '\x1f38' -> '\x1f30' + '\x1f39' -> '\x1f31' + '\x1f3a' -> '\x1f32' + '\x1f3b' -> '\x1f33' + '\x1f3c' -> '\x1f34' + '\x1f3d' -> '\x1f35' + '\x1f3e' -> '\x1f36' + '\x1f3f' -> '\x1f37' + '\x1f48' -> '\x1f40' + '\x1f49' -> '\x1f41' + '\x1f4a' -> '\x1f42' + '\x1f4b' -> '\x1f43' + '\x1f4c' -> '\x1f44' + '\x1f4d' -> '\x1f45' + '\x1f59' -> '\x1f51' + '\x1f5b' -> '\x1f53' + '\x1f5d' -> '\x1f55' + '\x1f5f' -> '\x1f57' + '\x1f68' -> '\x1f60' + '\x1f69' -> '\x1f61' + '\x1f6a' -> '\x1f62' + '\x1f6b' -> '\x1f63' + '\x1f6c' -> '\x1f64' + '\x1f6d' -> '\x1f65' + '\x1f6e' -> '\x1f66' + '\x1f6f' -> '\x1f67' + '\x1f88' -> '\x1f80' + '\x1f89' -> '\x1f81' + '\x1f8a' -> '\x1f82' + '\x1f8b' -> '\x1f83' + '\x1f8c' -> '\x1f84' + '\x1f8d' -> '\x1f85' + '\x1f8e' -> '\x1f86' + '\x1f8f' -> '\x1f87' + '\x1f98' -> '\x1f90' + '\x1f99' -> '\x1f91' + '\x1f9a' -> '\x1f92' + '\x1f9b' -> '\x1f93' + '\x1f9c' -> '\x1f94' + '\x1f9d' -> '\x1f95' + '\x1f9e' -> '\x1f96' + '\x1f9f' -> '\x1f97' + '\x1fa8' -> '\x1fa0' + '\x1fa9' -> '\x1fa1' + '\x1faa' -> '\x1fa2' + '\x1fab' -> '\x1fa3' + '\x1fac' -> '\x1fa4' + '\x1fad' -> '\x1fa5' + '\x1fae' -> '\x1fa6' + '\x1faf' -> '\x1fa7' + '\x1fb8' -> '\x1fb0' + '\x1fb9' -> '\x1fb1' + '\x1fba' -> '\x1f70' + '\x1fbb' -> '\x1f71' + '\x1fbc' -> '\x1fb3' + '\x1fc8' -> '\x1f72' + '\x1fc9' -> '\x1f73' + '\x1fca' -> '\x1f74' + '\x1fcb' -> '\x1f75' + '\x1fcc' -> '\x1fc3' + '\x1fd8' -> '\x1fd0' + '\x1fd9' -> '\x1fd1' + '\x1fda' -> '\x1f76' + '\x1fdb' -> '\x1f77' + '\x1fe8' -> '\x1fe0' + '\x1fe9' -> '\x1fe1' + '\x1fea' -> '\x1f7a' + '\x1feb' -> '\x1f7b' + '\x1fec' -> '\x1fe5' + '\x1ff8' -> '\x1f78' + '\x1ff9' -> '\x1f79' + '\x1ffa' -> '\x1f7c' + '\x1ffb' -> '\x1f7d' + '\x1ffc' -> '\x1ff3' + '\x2126' -> '\x3c9' + '\x212a' -> '\x6b' + '\x212b' -> '\xe5' + '\x2132' -> '\x214e' + '\x2160' -> '\x2170' + '\x2161' -> '\x2171' + '\x2162' -> '\x2172' + '\x2163' -> '\x2173' + '\x2164' -> '\x2174' + '\x2165' -> '\x2175' + '\x2166' -> '\x2176' + '\x2167' -> '\x2177' + '\x2168' -> '\x2178' + '\x2169' -> '\x2179' + '\x216a' -> '\x217a' + '\x216b' -> '\x217b' + '\x216c' -> '\x217c' + '\x216d' -> '\x217d' + '\x216e' -> '\x217e' + '\x216f' -> '\x217f' + '\x2183' -> '\x2184' + '\x24b6' -> '\x24d0' + '\x24b7' -> '\x24d1' + '\x24b8' -> '\x24d2' + '\x24b9' -> '\x24d3' + '\x24ba' -> '\x24d4' + '\x24bb' -> '\x24d5' + '\x24bc' -> '\x24d6' + '\x24bd' -> '\x24d7' + '\x24be' -> '\x24d8' + '\x24bf' -> '\x24d9' + '\x24c0' -> '\x24da' + '\x24c1' -> '\x24db' + '\x24c2' -> '\x24dc' + '\x24c3' -> '\x24dd' + '\x24c4' -> '\x24de' + '\x24c5' -> '\x24df' + '\x24c6' -> '\x24e0' + '\x24c7' -> '\x24e1' + '\x24c8' -> '\x24e2' + '\x24c9' -> '\x24e3' + '\x24ca' -> '\x24e4' + '\x24cb' -> '\x24e5' + '\x24cc' -> '\x24e6' + '\x24cd' -> '\x24e7' + '\x24ce' -> '\x24e8' + '\x24cf' -> '\x24e9' + '\x2c00' -> '\x2c30' + '\x2c01' -> '\x2c31' + '\x2c02' -> '\x2c32' + '\x2c03' -> '\x2c33' + '\x2c04' -> '\x2c34' + '\x2c05' -> '\x2c35' + '\x2c06' -> '\x2c36' + '\x2c07' -> '\x2c37' + '\x2c08' -> '\x2c38' + '\x2c09' -> '\x2c39' + '\x2c0a' -> '\x2c3a' + '\x2c0b' -> '\x2c3b' + '\x2c0c' -> '\x2c3c' + '\x2c0d' -> '\x2c3d' + '\x2c0e' -> '\x2c3e' + '\x2c0f' -> '\x2c3f' + '\x2c10' -> '\x2c40' + '\x2c11' -> '\x2c41' + '\x2c12' -> '\x2c42' + '\x2c13' -> '\x2c43' + '\x2c14' -> '\x2c44' + '\x2c15' -> '\x2c45' + '\x2c16' -> '\x2c46' + '\x2c17' -> '\x2c47' + '\x2c18' -> '\x2c48' + '\x2c19' -> '\x2c49' + '\x2c1a' -> '\x2c4a' + '\x2c1b' -> '\x2c4b' + '\x2c1c' -> '\x2c4c' + '\x2c1d' -> '\x2c4d' + '\x2c1e' -> '\x2c4e' + '\x2c1f' -> '\x2c4f' + '\x2c20' -> '\x2c50' + '\x2c21' -> '\x2c51' + '\x2c22' -> '\x2c52' + '\x2c23' -> '\x2c53' + '\x2c24' -> '\x2c54' + '\x2c25' -> '\x2c55' + '\x2c26' -> '\x2c56' + '\x2c27' -> '\x2c57' + '\x2c28' -> '\x2c58' + '\x2c29' -> '\x2c59' + '\x2c2a' -> '\x2c5a' + '\x2c2b' -> '\x2c5b' + '\x2c2c' -> '\x2c5c' + '\x2c2d' -> '\x2c5d' + '\x2c2e' -> '\x2c5e' + '\x2c2f' -> '\x2c5f' + '\x2c60' -> '\x2c61' + '\x2c62' -> '\x26b' + '\x2c63' -> '\x1d7d' + '\x2c64' -> '\x27d' + '\x2c67' -> '\x2c68' + '\x2c69' -> '\x2c6a' + '\x2c6b' -> '\x2c6c' + '\x2c6d' -> '\x251' + '\x2c6e' -> '\x271' + '\x2c6f' -> '\x250' + '\x2c70' -> '\x252' + '\x2c72' -> '\x2c73' + '\x2c75' -> '\x2c76' + '\x2c7e' -> '\x23f' + '\x2c7f' -> '\x240' + '\x2c80' -> '\x2c81' + '\x2c82' -> '\x2c83' + '\x2c84' -> '\x2c85' + '\x2c86' -> '\x2c87' + '\x2c88' -> '\x2c89' + '\x2c8a' -> '\x2c8b' + '\x2c8c' -> '\x2c8d' + '\x2c8e' -> '\x2c8f' + '\x2c90' -> '\x2c91' + '\x2c92' -> '\x2c93' + '\x2c94' -> '\x2c95' + '\x2c96' -> '\x2c97' + '\x2c98' -> '\x2c99' + '\x2c9a' -> '\x2c9b' + '\x2c9c' -> '\x2c9d' + '\x2c9e' -> '\x2c9f' + '\x2ca0' -> '\x2ca1' + '\x2ca2' -> '\x2ca3' + '\x2ca4' -> '\x2ca5' + '\x2ca6' -> '\x2ca7' + '\x2ca8' -> '\x2ca9' + '\x2caa' -> '\x2cab' + '\x2cac' -> '\x2cad' + '\x2cae' -> '\x2caf' + '\x2cb0' -> '\x2cb1' + '\x2cb2' -> '\x2cb3' + '\x2cb4' -> '\x2cb5' + '\x2cb6' -> '\x2cb7' + '\x2cb8' -> '\x2cb9' + '\x2cba' -> '\x2cbb' + '\x2cbc' -> '\x2cbd' + '\x2cbe' -> '\x2cbf' + '\x2cc0' -> '\x2cc1' + '\x2cc2' -> '\x2cc3' + '\x2cc4' -> '\x2cc5' + '\x2cc6' -> '\x2cc7' + '\x2cc8' -> '\x2cc9' + '\x2cca' -> '\x2ccb' + '\x2ccc' -> '\x2ccd' + '\x2cce' -> '\x2ccf' + '\x2cd0' -> '\x2cd1' + '\x2cd2' -> '\x2cd3' + '\x2cd4' -> '\x2cd5' + '\x2cd6' -> '\x2cd7' + '\x2cd8' -> '\x2cd9' + '\x2cda' -> '\x2cdb' + '\x2cdc' -> '\x2cdd' + '\x2cde' -> '\x2cdf' + '\x2ce0' -> '\x2ce1' + '\x2ce2' -> '\x2ce3' + '\x2ceb' -> '\x2cec' + '\x2ced' -> '\x2cee' + '\x2cf2' -> '\x2cf3' + '\xa640' -> '\xa641' + '\xa642' -> '\xa643' + '\xa644' -> '\xa645' + '\xa646' -> '\xa647' + '\xa648' -> '\xa649' + '\xa64a' -> '\xa64b' + '\xa64c' -> '\xa64d' + '\xa64e' -> '\xa64f' + '\xa650' -> '\xa651' + '\xa652' -> '\xa653' + '\xa654' -> '\xa655' + '\xa656' -> '\xa657' + '\xa658' -> '\xa659' + '\xa65a' -> '\xa65b' + '\xa65c' -> '\xa65d' + '\xa65e' -> '\xa65f' + '\xa660' -> '\xa661' + '\xa662' -> '\xa663' + '\xa664' -> '\xa665' + '\xa666' -> '\xa667' + '\xa668' -> '\xa669' + '\xa66a' -> '\xa66b' + '\xa66c' -> '\xa66d' + '\xa680' -> '\xa681' + '\xa682' -> '\xa683' + '\xa684' -> '\xa685' + '\xa686' -> '\xa687' + '\xa688' -> '\xa689' + '\xa68a' -> '\xa68b' + '\xa68c' -> '\xa68d' + '\xa68e' -> '\xa68f' + '\xa690' -> '\xa691' + '\xa692' -> '\xa693' + '\xa694' -> '\xa695' + '\xa696' -> '\xa697' + '\xa698' -> '\xa699' + '\xa69a' -> '\xa69b' + '\xa722' -> '\xa723' + '\xa724' -> '\xa725' + '\xa726' -> '\xa727' + '\xa728' -> '\xa729' + '\xa72a' -> '\xa72b' + '\xa72c' -> '\xa72d' + '\xa72e' -> '\xa72f' + '\xa732' -> '\xa733' + '\xa734' -> '\xa735' + '\xa736' -> '\xa737' + '\xa738' -> '\xa739' + '\xa73a' -> '\xa73b' + '\xa73c' -> '\xa73d' + '\xa73e' -> '\xa73f' + '\xa740' -> '\xa741' + '\xa742' -> '\xa743' + '\xa744' -> '\xa745' + '\xa746' -> '\xa747' + '\xa748' -> '\xa749' + '\xa74a' -> '\xa74b' + '\xa74c' -> '\xa74d' + '\xa74e' -> '\xa74f' + '\xa750' -> '\xa751' + '\xa752' -> '\xa753' + '\xa754' -> '\xa755' + '\xa756' -> '\xa757' + '\xa758' -> '\xa759' + '\xa75a' -> '\xa75b' + '\xa75c' -> '\xa75d' + '\xa75e' -> '\xa75f' + '\xa760' -> '\xa761' + '\xa762' -> '\xa763' + '\xa764' -> '\xa765' + '\xa766' -> '\xa767' + '\xa768' -> '\xa769' + '\xa76a' -> '\xa76b' + '\xa76c' -> '\xa76d' + '\xa76e' -> '\xa76f' + '\xa779' -> '\xa77a' + '\xa77b' -> '\xa77c' + '\xa77d' -> '\x1d79' + '\xa77e' -> '\xa77f' + '\xa780' -> '\xa781' + '\xa782' -> '\xa783' + '\xa784' -> '\xa785' + '\xa786' -> '\xa787' + '\xa78b' -> '\xa78c' + '\xa78d' -> '\x265' + '\xa790' -> '\xa791' + '\xa792' -> '\xa793' + '\xa796' -> '\xa797' + '\xa798' -> '\xa799' + '\xa79a' -> '\xa79b' + '\xa79c' -> '\xa79d' + '\xa79e' -> '\xa79f' + '\xa7a0' -> '\xa7a1' + '\xa7a2' -> '\xa7a3' + '\xa7a4' -> '\xa7a5' + '\xa7a6' -> '\xa7a7' + '\xa7a8' -> '\xa7a9' + '\xa7aa' -> '\x266' + '\xa7ab' -> '\x25c' + '\xa7ac' -> '\x261' + '\xa7ad' -> '\x26c' + '\xa7ae' -> '\x26a' + '\xa7b0' -> '\x29e' + '\xa7b1' -> '\x287' + '\xa7b2' -> '\x29d' + '\xa7b3' -> '\xab53' + '\xa7b4' -> '\xa7b5' + '\xa7b6' -> '\xa7b7' + '\xa7b8' -> '\xa7b9' + '\xa7ba' -> '\xa7bb' + '\xa7bc' -> '\xa7bd' + '\xa7be' -> '\xa7bf' + '\xa7c0' -> '\xa7c1' + '\xa7c2' -> '\xa7c3' + '\xa7c4' -> '\xa794' + '\xa7c5' -> '\x282' + '\xa7c6' -> '\x1d8e' + '\xa7c7' -> '\xa7c8' + '\xa7c9' -> '\xa7ca' + '\xa7cb' -> '\x264' + '\xa7cc' -> '\xa7cd' + '\xa7ce' -> '\xa7cf' + '\xa7d0' -> '\xa7d1' + '\xa7d2' -> '\xa7d3' + '\xa7d4' -> '\xa7d5' + '\xa7d6' -> '\xa7d7' + '\xa7d8' -> '\xa7d9' + '\xa7da' -> '\xa7db' + '\xa7dc' -> '\x19b' + '\xa7f5' -> '\xa7f6' + '\xff21' -> '\xff41' + '\xff22' -> '\xff42' + '\xff23' -> '\xff43' + '\xff24' -> '\xff44' + '\xff25' -> '\xff45' + '\xff26' -> '\xff46' + '\xff27' -> '\xff47' + '\xff28' -> '\xff48' + '\xff29' -> '\xff49' + '\xff2a' -> '\xff4a' + '\xff2b' -> '\xff4b' + '\xff2c' -> '\xff4c' + '\xff2d' -> '\xff4d' + '\xff2e' -> '\xff4e' + '\xff2f' -> '\xff4f' + '\xff30' -> '\xff50' + '\xff31' -> '\xff51' + '\xff32' -> '\xff52' + '\xff33' -> '\xff53' + '\xff34' -> '\xff54' + '\xff35' -> '\xff55' + '\xff36' -> '\xff56' + '\xff37' -> '\xff57' + '\xff38' -> '\xff58' + '\xff39' -> '\xff59' + '\xff3a' -> '\xff5a' + '\x10400' -> '\x10428' + '\x10401' -> '\x10429' + '\x10402' -> '\x1042a' + '\x10403' -> '\x1042b' + '\x10404' -> '\x1042c' + '\x10405' -> '\x1042d' + '\x10406' -> '\x1042e' + '\x10407' -> '\x1042f' + '\x10408' -> '\x10430' + '\x10409' -> '\x10431' + '\x1040a' -> '\x10432' + '\x1040b' -> '\x10433' + '\x1040c' -> '\x10434' + '\x1040d' -> '\x10435' + '\x1040e' -> '\x10436' + '\x1040f' -> '\x10437' + '\x10410' -> '\x10438' + '\x10411' -> '\x10439' + '\x10412' -> '\x1043a' + '\x10413' -> '\x1043b' + '\x10414' -> '\x1043c' + '\x10415' -> '\x1043d' + '\x10416' -> '\x1043e' + '\x10417' -> '\x1043f' + '\x10418' -> '\x10440' + '\x10419' -> '\x10441' + '\x1041a' -> '\x10442' + '\x1041b' -> '\x10443' + '\x1041c' -> '\x10444' + '\x1041d' -> '\x10445' + '\x1041e' -> '\x10446' + '\x1041f' -> '\x10447' + '\x10420' -> '\x10448' + '\x10421' -> '\x10449' + '\x10422' -> '\x1044a' + '\x10423' -> '\x1044b' + '\x10424' -> '\x1044c' + '\x10425' -> '\x1044d' + '\x10426' -> '\x1044e' + '\x10427' -> '\x1044f' + '\x104b0' -> '\x104d8' + '\x104b1' -> '\x104d9' + '\x104b2' -> '\x104da' + '\x104b3' -> '\x104db' + '\x104b4' -> '\x104dc' + '\x104b5' -> '\x104dd' + '\x104b6' -> '\x104de' + '\x104b7' -> '\x104df' + '\x104b8' -> '\x104e0' + '\x104b9' -> '\x104e1' + '\x104ba' -> '\x104e2' + '\x104bb' -> '\x104e3' + '\x104bc' -> '\x104e4' + '\x104bd' -> '\x104e5' + '\x104be' -> '\x104e6' + '\x104bf' -> '\x104e7' + '\x104c0' -> '\x104e8' + '\x104c1' -> '\x104e9' + '\x104c2' -> '\x104ea' + '\x104c3' -> '\x104eb' + '\x104c4' -> '\x104ec' + '\x104c5' -> '\x104ed' + '\x104c6' -> '\x104ee' + '\x104c7' -> '\x104ef' + '\x104c8' -> '\x104f0' + '\x104c9' -> '\x104f1' + '\x104ca' -> '\x104f2' + '\x104cb' -> '\x104f3' + '\x104cc' -> '\x104f4' + '\x104cd' -> '\x104f5' + '\x104ce' -> '\x104f6' + '\x104cf' -> '\x104f7' + '\x104d0' -> '\x104f8' + '\x104d1' -> '\x104f9' + '\x104d2' -> '\x104fa' + '\x104d3' -> '\x104fb' + '\x10570' -> '\x10597' + '\x10571' -> '\x10598' + '\x10572' -> '\x10599' + '\x10573' -> '\x1059a' + '\x10574' -> '\x1059b' + '\x10575' -> '\x1059c' + '\x10576' -> '\x1059d' + '\x10577' -> '\x1059e' + '\x10578' -> '\x1059f' + '\x10579' -> '\x105a0' + '\x1057a' -> '\x105a1' + '\x1057c' -> '\x105a3' + '\x1057d' -> '\x105a4' + '\x1057e' -> '\x105a5' + '\x1057f' -> '\x105a6' + '\x10580' -> '\x105a7' + '\x10581' -> '\x105a8' + '\x10582' -> '\x105a9' + '\x10583' -> '\x105aa' + '\x10584' -> '\x105ab' + '\x10585' -> '\x105ac' + '\x10586' -> '\x105ad' + '\x10587' -> '\x105ae' + '\x10588' -> '\x105af' + '\x10589' -> '\x105b0' + '\x1058a' -> '\x105b1' + '\x1058c' -> '\x105b3' + '\x1058d' -> '\x105b4' + '\x1058e' -> '\x105b5' + '\x1058f' -> '\x105b6' + '\x10590' -> '\x105b7' + '\x10591' -> '\x105b8' + '\x10592' -> '\x105b9' + '\x10594' -> '\x105bb' + '\x10595' -> '\x105bc' + '\x10c80' -> '\x10cc0' + '\x10c81' -> '\x10cc1' + '\x10c82' -> '\x10cc2' + '\x10c83' -> '\x10cc3' + '\x10c84' -> '\x10cc4' + '\x10c85' -> '\x10cc5' + '\x10c86' -> '\x10cc6' + '\x10c87' -> '\x10cc7' + '\x10c88' -> '\x10cc8' + '\x10c89' -> '\x10cc9' + '\x10c8a' -> '\x10cca' + '\x10c8b' -> '\x10ccb' + '\x10c8c' -> '\x10ccc' + '\x10c8d' -> '\x10ccd' + '\x10c8e' -> '\x10cce' + '\x10c8f' -> '\x10ccf' + '\x10c90' -> '\x10cd0' + '\x10c91' -> '\x10cd1' + '\x10c92' -> '\x10cd2' + '\x10c93' -> '\x10cd3' + '\x10c94' -> '\x10cd4' + '\x10c95' -> '\x10cd5' + '\x10c96' -> '\x10cd6' + '\x10c97' -> '\x10cd7' + '\x10c98' -> '\x10cd8' + '\x10c99' -> '\x10cd9' + '\x10c9a' -> '\x10cda' + '\x10c9b' -> '\x10cdb' + '\x10c9c' -> '\x10cdc' + '\x10c9d' -> '\x10cdd' + '\x10c9e' -> '\x10cde' + '\x10c9f' -> '\x10cdf' + '\x10ca0' -> '\x10ce0' + '\x10ca1' -> '\x10ce1' + '\x10ca2' -> '\x10ce2' + '\x10ca3' -> '\x10ce3' + '\x10ca4' -> '\x10ce4' + '\x10ca5' -> '\x10ce5' + '\x10ca6' -> '\x10ce6' + '\x10ca7' -> '\x10ce7' + '\x10ca8' -> '\x10ce8' + '\x10ca9' -> '\x10ce9' + '\x10caa' -> '\x10cea' + '\x10cab' -> '\x10ceb' + '\x10cac' -> '\x10cec' + '\x10cad' -> '\x10ced' + '\x10cae' -> '\x10cee' + '\x10caf' -> '\x10cef' + '\x10cb0' -> '\x10cf0' + '\x10cb1' -> '\x10cf1' + '\x10cb2' -> '\x10cf2' + '\x10d50' -> '\x10d70' + '\x10d51' -> '\x10d71' + '\x10d52' -> '\x10d72' + '\x10d53' -> '\x10d73' + '\x10d54' -> '\x10d74' + '\x10d55' -> '\x10d75' + '\x10d56' -> '\x10d76' + '\x10d57' -> '\x10d77' + '\x10d58' -> '\x10d78' + '\x10d59' -> '\x10d79' + '\x10d5a' -> '\x10d7a' + '\x10d5b' -> '\x10d7b' + '\x10d5c' -> '\x10d7c' + '\x10d5d' -> '\x10d7d' + '\x10d5e' -> '\x10d7e' + '\x10d5f' -> '\x10d7f' + '\x10d60' -> '\x10d80' + '\x10d61' -> '\x10d81' + '\x10d62' -> '\x10d82' + '\x10d63' -> '\x10d83' + '\x10d64' -> '\x10d84' + '\x10d65' -> '\x10d85' + '\x118a0' -> '\x118c0' + '\x118a1' -> '\x118c1' + '\x118a2' -> '\x118c2' + '\x118a3' -> '\x118c3' + '\x118a4' -> '\x118c4' + '\x118a5' -> '\x118c5' + '\x118a6' -> '\x118c6' + '\x118a7' -> '\x118c7' + '\x118a8' -> '\x118c8' + '\x118a9' -> '\x118c9' + '\x118aa' -> '\x118ca' + '\x118ab' -> '\x118cb' + '\x118ac' -> '\x118cc' + '\x118ad' -> '\x118cd' + '\x118ae' -> '\x118ce' + '\x118af' -> '\x118cf' + '\x118b0' -> '\x118d0' + '\x118b1' -> '\x118d1' + '\x118b2' -> '\x118d2' + '\x118b3' -> '\x118d3' + '\x118b4' -> '\x118d4' + '\x118b5' -> '\x118d5' + '\x118b6' -> '\x118d6' + '\x118b7' -> '\x118d7' + '\x118b8' -> '\x118d8' + '\x118b9' -> '\x118d9' + '\x118ba' -> '\x118da' + '\x118bb' -> '\x118db' + '\x118bc' -> '\x118dc' + '\x118bd' -> '\x118dd' + '\x118be' -> '\x118de' + '\x118bf' -> '\x118df' + '\x16e40' -> '\x16e60' + '\x16e41' -> '\x16e61' + '\x16e42' -> '\x16e62' + '\x16e43' -> '\x16e63' + '\x16e44' -> '\x16e64' + '\x16e45' -> '\x16e65' + '\x16e46' -> '\x16e66' + '\x16e47' -> '\x16e67' + '\x16e48' -> '\x16e68' + '\x16e49' -> '\x16e69' + '\x16e4a' -> '\x16e6a' + '\x16e4b' -> '\x16e6b' + '\x16e4c' -> '\x16e6c' + '\x16e4d' -> '\x16e6d' + '\x16e4e' -> '\x16e6e' + '\x16e4f' -> '\x16e6f' + '\x16e50' -> '\x16e70' + '\x16e51' -> '\x16e71' + '\x16e52' -> '\x16e72' + '\x16e53' -> '\x16e73' + '\x16e54' -> '\x16e74' + '\x16e55' -> '\x16e75' + '\x16e56' -> '\x16e76' + '\x16e57' -> '\x16e77' + '\x16e58' -> '\x16e78' + '\x16e59' -> '\x16e79' + '\x16e5a' -> '\x16e7a' + '\x16e5b' -> '\x16e7b' + '\x16e5c' -> '\x16e7c' + '\x16e5d' -> '\x16e7d' + '\x16e5e' -> '\x16e7e' + '\x16e5f' -> '\x16e7f' + '\x16ea0' -> '\x16ebb' + '\x16ea1' -> '\x16ebc' + '\x16ea2' -> '\x16ebd' + '\x16ea3' -> '\x16ebe' + '\x16ea4' -> '\x16ebf' + '\x16ea5' -> '\x16ec0' + '\x16ea6' -> '\x16ec1' + '\x16ea7' -> '\x16ec2' + '\x16ea8' -> '\x16ec3' + '\x16ea9' -> '\x16ec4' + '\x16eaa' -> '\x16ec5' + '\x16eab' -> '\x16ec6' + '\x16eac' -> '\x16ec7' + '\x16ead' -> '\x16ec8' + '\x16eae' -> '\x16ec9' + '\x16eaf' -> '\x16eca' + '\x16eb0' -> '\x16ecb' + '\x16eb1' -> '\x16ecc' + '\x16eb2' -> '\x16ecd' + '\x16eb3' -> '\x16ece' + '\x16eb4' -> '\x16ecf' + '\x16eb5' -> '\x16ed0' + '\x16eb6' -> '\x16ed1' + '\x16eb7' -> '\x16ed2' + '\x16eb8' -> '\x16ed3' + '\x1e900' -> '\x1e922' + '\x1e901' -> '\x1e923' + '\x1e902' -> '\x1e924' + '\x1e903' -> '\x1e925' + '\x1e904' -> '\x1e926' + '\x1e905' -> '\x1e927' + '\x1e906' -> '\x1e928' + '\x1e907' -> '\x1e929' + '\x1e908' -> '\x1e92a' + '\x1e909' -> '\x1e92b' + '\x1e90a' -> '\x1e92c' + '\x1e90b' -> '\x1e92d' + '\x1e90c' -> '\x1e92e' + '\x1e90d' -> '\x1e92f' + '\x1e90e' -> '\x1e930' + '\x1e90f' -> '\x1e931' + '\x1e910' -> '\x1e932' + '\x1e911' -> '\x1e933' + '\x1e912' -> '\x1e934' + '\x1e913' -> '\x1e935' + '\x1e914' -> '\x1e936' + '\x1e915' -> '\x1e937' + '\x1e916' -> '\x1e938' + '\x1e917' -> '\x1e939' + '\x1e918' -> '\x1e93a' + '\x1e919' -> '\x1e93b' + '\x1e91a' -> '\x1e93c' + '\x1e91b' -> '\x1e93d' + '\x1e91c' -> '\x1e93e' + '\x1e91d' -> '\x1e93f' + '\x1e91e' -> '\x1e940' + '\x1e91f' -> '\x1e941' + '\x1e920' -> '\x1e942' + '\x1e921' -> '\x1e943' + c -> c diff --git a/internal/Text/IDNA2008/Internal/Combining.hs b/internal/Text/IDNA2008/Internal/Combining.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Combining.hs @@ -0,0 +1,67 @@ +-- | +-- Module : Text.IDNA2008.Internal.Combining +-- Description : Combining-mark predicate for the RFC 5891 section +-- 4.2.3.2 leading-combining-mark check. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- RFC 5891 section 4.2.3.2 says: +-- +-- /The Unicode string MUST NOT begin with a combining mark or +-- combining character (see The Unicode Standard, Section 2.11 +-- for an exact definition)./ +-- +-- A combining mark is a codepoint whose @General_Category@ is one +-- of @Mn@ (Nonspacing_Mark), @Mc@ (Spacing_Mark), or @Me@ +-- (Enclosing_Mark). RFC 5892's disposition table classifies many +-- of those codepoints (Mn / Mc) as @PVALID@, so the per-codepoint +-- validator alone doesn't catch the "no combining mark at label +-- start" rule. This module is the dedicated lookup the validator +-- consults before walking the codepoint buffer. + +module Text.IDNA2008.Internal.Combining + ( isCombiningMark + ) where + +import Data.Bits (unsafeShiftR) +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word8, Word32) + +import Text.IDNA2008.Internal.Combining.Data + ( combiningRangeCount + , combiningRangeStarts + , combiningRangeTags + ) + +-- | True iff the codepoint has @General_Category@ in @{Mn, Mc, +-- Me}@. Out-of-range inputs return 'False'. +isCombiningMark :: Int -> Bool +isCombiningMark !cp + | cp < 0 || cp > 0x10FFFF = False + | otherwise = lookupTag cp /= 0 +{-# INLINE isCombiningMark #-} + +-- | Binary search for the largest index @i@ such that +-- @combiningRangeStarts[i] <= cp@; return that range's tag (0 or +-- 1). The table starts at @0x000000@ so a hit is guaranteed for +-- any @cp >= 0@. +{-# INLINE lookupTag #-} +lookupTag :: Int -> Word8 +lookupTag !cp = go 0 (combiningRangeCount - 1) + where + !target = fromIntegral cp :: Word32 + + go !lo !hi + | lo >= hi = readTag lo + | otherwise = + let !mid = (lo + hi + 1) `unsafeShiftR` 1 + !midKey = readStart mid + in if midKey <= target + then go mid hi + else go lo (mid - 1) + + readStart i = indexByteArray combiningRangeStarts i :: Word32 + readTag i = indexByteArray combiningRangeTags i :: Word8 diff --git a/internal/Text/IDNA2008/Internal/Combining/Data.hs b/internal/Text/IDNA2008/Internal/Combining/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Combining/Data.hs @@ -0,0 +1,156 @@ +-- | +-- Module : Text.IDNA2008.Internal.Combining.Data +-- Description : Codegen'd combining-mark table (gc in @{Mn, Mc, Me}@) +-- for the RFC 5891 section 4.2.3.2 leading-mark check. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED FROM UnicodeData.txt. Do not edit by +-- hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaCombining.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/UnicodeData.txt +-- Aligned with: Unicode 17.0.0 +-- Range count: 655 +-- +-- The table covers every Unicode codepoint in @[0, 0x10FFFF]@ with no +-- gaps: each entry says "from this start until the next entry's start +-- (or 0x10FFFF for the last), the resolved combining-mark status is +-- /T/". Tag 0 means the codepoint is /not/ a combining mark; tag 1 +-- means it is (gc in @{Mn, Mc, Me}@). Codepoints absent from +-- UnicodeData.txt are treated as non-combining. +module Text.IDNA2008.Internal.Combining.Data + ( combiningRangeCount + , combiningRangeStarts + , combiningRangeTags + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +-- | Number of (start, tag) entries. +combiningRangeCount :: Int +combiningRangeCount = 655 + +-- | Range starts; sorted ascending. +combiningRangeStarts :: ByteArray +combiningRangeStarts = PBA.byteArrayFromList + [ (0x000000 :: Word32), 0x000300, 0x000370, 0x000483, 0x00048A, 0x000591, 0x0005BE, 0x0005BF + , 0x0005C0, 0x0005C1, 0x0005C3, 0x0005C4, 0x0005C6, 0x0005C7, 0x0005C8, 0x000610 + , 0x00061B, 0x00064B, 0x000660, 0x000670, 0x000671, 0x0006D6, 0x0006DD, 0x0006DF + , 0x0006E5, 0x0006E7, 0x0006E9, 0x0006EA, 0x0006EE, 0x000711, 0x000712, 0x000730 + , 0x00074B, 0x0007A6, 0x0007B1, 0x0007EB, 0x0007F4, 0x0007FD, 0x0007FE, 0x000816 + , 0x00081A, 0x00081B, 0x000824, 0x000825, 0x000828, 0x000829, 0x00082E, 0x000859 + , 0x00085C, 0x000897, 0x0008A0, 0x0008CA, 0x0008E2, 0x0008E3, 0x000904, 0x00093A + , 0x00093D, 0x00093E, 0x000950, 0x000951, 0x000958, 0x000962, 0x000964, 0x000981 + , 0x000984, 0x0009BC, 0x0009BD, 0x0009BE, 0x0009C5, 0x0009C7, 0x0009C9, 0x0009CB + , 0x0009CE, 0x0009D7, 0x0009D8, 0x0009E2, 0x0009E4, 0x0009FE, 0x0009FF, 0x000A01 + , 0x000A04, 0x000A3C, 0x000A3D, 0x000A3E, 0x000A43, 0x000A47, 0x000A49, 0x000A4B + , 0x000A4E, 0x000A51, 0x000A52, 0x000A70, 0x000A72, 0x000A75, 0x000A76, 0x000A81 + , 0x000A84, 0x000ABC, 0x000ABD, 0x000ABE, 0x000AC6, 0x000AC7, 0x000ACA, 0x000ACB + , 0x000ACE, 0x000AE2, 0x000AE4, 0x000AFA, 0x000B00, 0x000B01, 0x000B04, 0x000B3C + , 0x000B3D, 0x000B3E, 0x000B45, 0x000B47, 0x000B49, 0x000B4B, 0x000B4E, 0x000B55 + , 0x000B58, 0x000B62, 0x000B64, 0x000B82, 0x000B83, 0x000BBE, 0x000BC3, 0x000BC6 + , 0x000BC9, 0x000BCA, 0x000BCE, 0x000BD7, 0x000BD8, 0x000C00, 0x000C05, 0x000C3C + , 0x000C3D, 0x000C3E, 0x000C45, 0x000C46, 0x000C49, 0x000C4A, 0x000C4E, 0x000C55 + , 0x000C57, 0x000C62, 0x000C64, 0x000C81, 0x000C84, 0x000CBC, 0x000CBD, 0x000CBE + , 0x000CC5, 0x000CC6, 0x000CC9, 0x000CCA, 0x000CCE, 0x000CD5, 0x000CD7, 0x000CE2 + , 0x000CE4, 0x000CF3, 0x000CF4, 0x000D00, 0x000D04, 0x000D3B, 0x000D3D, 0x000D3E + , 0x000D45, 0x000D46, 0x000D49, 0x000D4A, 0x000D4E, 0x000D57, 0x000D58, 0x000D62 + , 0x000D64, 0x000D81, 0x000D84, 0x000DCA, 0x000DCB, 0x000DCF, 0x000DD5, 0x000DD6 + , 0x000DD7, 0x000DD8, 0x000DE0, 0x000DF2, 0x000DF4, 0x000E31, 0x000E32, 0x000E34 + , 0x000E3B, 0x000E47, 0x000E4F, 0x000EB1, 0x000EB2, 0x000EB4, 0x000EBD, 0x000EC8 + , 0x000ECF, 0x000F18, 0x000F1A, 0x000F35, 0x000F36, 0x000F37, 0x000F38, 0x000F39 + , 0x000F3A, 0x000F3E, 0x000F40, 0x000F71, 0x000F85, 0x000F86, 0x000F88, 0x000F8D + , 0x000F98, 0x000F99, 0x000FBD, 0x000FC6, 0x000FC7, 0x00102B, 0x00103F, 0x001056 + , 0x00105A, 0x00105E, 0x001061, 0x001062, 0x001065, 0x001067, 0x00106E, 0x001071 + , 0x001075, 0x001082, 0x00108E, 0x00108F, 0x001090, 0x00109A, 0x00109E, 0x00135D + , 0x001360, 0x001712, 0x001716, 0x001732, 0x001735, 0x001752, 0x001754, 0x001772 + , 0x001774, 0x0017B4, 0x0017D4, 0x0017DD, 0x0017DE, 0x00180B, 0x00180E, 0x00180F + , 0x001810, 0x001885, 0x001887, 0x0018A9, 0x0018AA, 0x001920, 0x00192C, 0x001930 + , 0x00193C, 0x001A17, 0x001A1C, 0x001A55, 0x001A5F, 0x001A60, 0x001A7D, 0x001A7F + , 0x001A80, 0x001AB0, 0x001ADE, 0x001AE0, 0x001AEC, 0x001B00, 0x001B05, 0x001B34 + , 0x001B45, 0x001B6B, 0x001B74, 0x001B80, 0x001B83, 0x001BA1, 0x001BAE, 0x001BE6 + , 0x001BF4, 0x001C24, 0x001C38, 0x001CD0, 0x001CD3, 0x001CD4, 0x001CE9, 0x001CED + , 0x001CEE, 0x001CF4, 0x001CF5, 0x001CF7, 0x001CFA, 0x001DC0, 0x001E00, 0x0020D0 + , 0x0020F1, 0x002CEF, 0x002CF2, 0x002D7F, 0x002D80, 0x002DE0, 0x002E00, 0x00302A + , 0x003030, 0x003099, 0x00309B, 0x00A66F, 0x00A673, 0x00A674, 0x00A67E, 0x00A69E + , 0x00A6A0, 0x00A6F0, 0x00A6F2, 0x00A802, 0x00A803, 0x00A806, 0x00A807, 0x00A80B + , 0x00A80C, 0x00A823, 0x00A828, 0x00A82C, 0x00A82D, 0x00A880, 0x00A882, 0x00A8B4 + , 0x00A8C6, 0x00A8E0, 0x00A8F2, 0x00A8FF, 0x00A900, 0x00A926, 0x00A92E, 0x00A947 + , 0x00A954, 0x00A980, 0x00A984, 0x00A9B3, 0x00A9C1, 0x00A9E5, 0x00A9E6, 0x00AA29 + , 0x00AA37, 0x00AA43, 0x00AA44, 0x00AA4C, 0x00AA4E, 0x00AA7B, 0x00AA7E, 0x00AAB0 + , 0x00AAB1, 0x00AAB2, 0x00AAB5, 0x00AAB7, 0x00AAB9, 0x00AABE, 0x00AAC0, 0x00AAC1 + , 0x00AAC2, 0x00AAEB, 0x00AAF0, 0x00AAF5, 0x00AAF7, 0x00ABE3, 0x00ABEB, 0x00ABEC + , 0x00ABEE, 0x00FB1E, 0x00FB1F, 0x00FE00, 0x00FE10, 0x00FE20, 0x00FE30, 0x0101FD + , 0x0101FE, 0x0102E0, 0x0102E1, 0x010376, 0x01037B, 0x010A01, 0x010A04, 0x010A05 + , 0x010A07, 0x010A0C, 0x010A10, 0x010A38, 0x010A3B, 0x010A3F, 0x010A40, 0x010AE5 + , 0x010AE7, 0x010D24, 0x010D28, 0x010D69, 0x010D6E, 0x010EAB, 0x010EAD, 0x010EFA + , 0x010F00, 0x010F46, 0x010F51, 0x010F82, 0x010F86, 0x011000, 0x011003, 0x011038 + , 0x011047, 0x011070, 0x011071, 0x011073, 0x011075, 0x01107F, 0x011083, 0x0110B0 + , 0x0110BB, 0x0110C2, 0x0110C3, 0x011100, 0x011103, 0x011127, 0x011135, 0x011145 + , 0x011147, 0x011173, 0x011174, 0x011180, 0x011183, 0x0111B3, 0x0111C1, 0x0111C9 + , 0x0111CD, 0x0111CE, 0x0111D0, 0x01122C, 0x011238, 0x01123E, 0x01123F, 0x011241 + , 0x011242, 0x0112DF, 0x0112EB, 0x011300, 0x011304, 0x01133B, 0x01133D, 0x01133E + , 0x011345, 0x011347, 0x011349, 0x01134B, 0x01134E, 0x011357, 0x011358, 0x011362 + , 0x011364, 0x011366, 0x01136D, 0x011370, 0x011375, 0x0113B8, 0x0113C1, 0x0113C2 + , 0x0113C3, 0x0113C5, 0x0113C6, 0x0113C7, 0x0113CB, 0x0113CC, 0x0113D1, 0x0113D2 + , 0x0113D3, 0x0113E1, 0x0113E3, 0x011435, 0x011447, 0x01145E, 0x01145F, 0x0114B0 + , 0x0114C4, 0x0115AF, 0x0115B6, 0x0115B8, 0x0115C1, 0x0115DC, 0x0115DE, 0x011630 + , 0x011641, 0x0116AB, 0x0116B8, 0x01171D, 0x01172C, 0x01182C, 0x01183B, 0x011930 + , 0x011936, 0x011937, 0x011939, 0x01193B, 0x01193F, 0x011940, 0x011941, 0x011942 + , 0x011944, 0x0119D1, 0x0119D8, 0x0119DA, 0x0119E1, 0x0119E4, 0x0119E5, 0x011A01 + , 0x011A0B, 0x011A33, 0x011A3A, 0x011A3B, 0x011A3F, 0x011A47, 0x011A48, 0x011A51 + , 0x011A5C, 0x011A8A, 0x011A9A, 0x011B60, 0x011B68, 0x011C2F, 0x011C37, 0x011C38 + , 0x011C40, 0x011C92, 0x011CA8, 0x011CA9, 0x011CB7, 0x011D31, 0x011D37, 0x011D3A + , 0x011D3B, 0x011D3C, 0x011D3E, 0x011D3F, 0x011D46, 0x011D47, 0x011D48, 0x011D8A + , 0x011D8F, 0x011D90, 0x011D92, 0x011D93, 0x011D98, 0x011EF3, 0x011EF7, 0x011F00 + , 0x011F02, 0x011F03, 0x011F04, 0x011F34, 0x011F3B, 0x011F3E, 0x011F43, 0x011F5A + , 0x011F5B, 0x013440, 0x013441, 0x013447, 0x013456, 0x01611E, 0x016130, 0x016AF0 + , 0x016AF5, 0x016B30, 0x016B37, 0x016F4F, 0x016F50, 0x016F51, 0x016F88, 0x016F8F + , 0x016F93, 0x016FE4, 0x016FE5, 0x016FF0, 0x016FF2, 0x01BC9D, 0x01BC9F, 0x01CF00 + , 0x01CF2E, 0x01CF30, 0x01CF47, 0x01D165, 0x01D16A, 0x01D16D, 0x01D173, 0x01D17B + , 0x01D183, 0x01D185, 0x01D18C, 0x01D1AA, 0x01D1AE, 0x01D242, 0x01D245, 0x01DA00 + , 0x01DA37, 0x01DA3B, 0x01DA6D, 0x01DA75, 0x01DA76, 0x01DA84, 0x01DA85, 0x01DA9B + , 0x01DAA0, 0x01DAA1, 0x01DAB0, 0x01E000, 0x01E007, 0x01E008, 0x01E019, 0x01E01B + , 0x01E022, 0x01E023, 0x01E025, 0x01E026, 0x01E02B, 0x01E08F, 0x01E090, 0x01E130 + , 0x01E137, 0x01E2AE, 0x01E2AF, 0x01E2EC, 0x01E2F0, 0x01E4EC, 0x01E4F0, 0x01E5EE + , 0x01E5F0, 0x01E6E3, 0x01E6E4, 0x01E6E6, 0x01E6E7, 0x01E6EE, 0x01E6F0, 0x01E6F5 + , 0x01E6F6, 0x01E8D0, 0x01E8D7, 0x01E944, 0x01E94B, 0x0E0100, 0x0E01F0 + ] + +-- | Combining-mark tags; parallel to +-- 'combiningRangeStarts'. +-- +-- * 0 -- not a combining mark +-- * 1 -- combining mark (gc in @{Mn, Mc, Me}@) +combiningRangeTags :: ByteArray +combiningRangeTags = PBA.byteArrayFromList @Word8 + [ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + , 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 + ] diff --git a/internal/Text/IDNA2008/Internal/Emoji.hs b/internal/Text/IDNA2008/Internal/Emoji.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Emoji.hs @@ -0,0 +1,56 @@ +-- | +-- Module : Text.IDNA2008.Internal.Emoji +-- Description : Typed wrappers over the Unicode @Emoji=Yes@ range +-- table and its UTS #46-mapped subset, consumed by +-- the @EMOJIOK@ option. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Two lookups, 'isEmoji' and 'isMappedEmoji', both implemented as +-- partial applications of "Text.IDNA2008.Internal.Ranges"'s +-- 'inRanges' helper over the coalesced range tables in +-- "Text.IDNA2008.Internal.Emoji.Data". +-- +-- 'isEmoji' is the Unicode @Emoji=Yes@ property predicate; +-- 'isMappedEmoji' is the subset whose UTS #46 status is also +-- @mapped@ -- i.e. the ambiguous-across-tools subset (browsers +-- following UTS #46 mapping reach the fold target; admit-as-is +-- tooling would reach the unmapped form). This library +-- declines to pick either interpretation: codepoints in the +-- mapped-emoji subset remain @DISALLOWED@ even under @EMOJIOK@. +-- +-- The IDNA U-label validator combines the two predicates: under +-- @EMOJIOK@ a non-ASCII @DISALLOWED@ codepoint is admitted iff +-- @'isEmoji' cp && not ('isMappedEmoji' cp)@. Both predicates +-- are dead code when @EMOJIOK@ is not set. + +module Text.IDNA2008.Internal.Emoji + ( isEmoji + , isMappedEmoji + ) where + +import Text.IDNA2008.Internal.Ranges (inRanges) +import Text.IDNA2008.Internal.Emoji.Data + ( emojiRangeCount + , emojiRanges + , mappedEmojiRangeCount + , mappedEmojiRanges + ) + +-- | Test whether @cp@ has the Unicode @Emoji=Yes@ property. +-- Codepoints absent from the range table are not emoji. +isEmoji :: Int -> Bool +isEmoji = inRanges emojiRanges emojiRangeCount +{-# INLINE isEmoji #-} + +-- | Test whether @cp@ is both @Emoji=Yes@ and folded by UTS #46. +-- These codepoints are excluded from the @EMOJIOK@ relaxation +-- because they resolve ambiguously across the ecosystem. See +-- the module haddock in "Text.IDNA2008.Internal.Emoji.Data" for +-- the empirical rationale. +isMappedEmoji :: Int -> Bool +isMappedEmoji = inRanges mappedEmojiRanges mappedEmojiRangeCount +{-# INLINE isMappedEmoji #-} diff --git a/internal/Text/IDNA2008/Internal/Emoji/Data.hs b/internal/Text/IDNA2008/Internal/Emoji/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Emoji/Data.hs @@ -0,0 +1,392 @@ +-- | +-- Module : Text.IDNA2008.Internal.Emoji.Data +-- Description : Compact codepoint range tables for the Unicode +-- @Emoji@ property and its UTS #46-mapped subset. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaEmoji.py \\ +-- +-- +-- Source (Unicode 12.0): https://www.unicode.org/Public/emoji/12.0/emoji-data.txt +-- Source (Unicode 13.0+): https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt +-- Source (Unicode 17.0+, idna): https://www.unicode.org/Public/17.0.0/idna/IdnaMappingTable.txt +-- Aligned with: Unicode 17.0.0 +-- Emoji ranges: 151 +-- Mapped-emoji ranges: 12 +-- +-- Two range tables, both with the same @Word32@ start\/end layout +-- (sorted by start, abutting ranges coalesced): +-- +-- * 'emojiRanges' -- codepoints with Unicode property @Emoji=Yes@. +-- +-- * 'mappedEmojiRanges' -- the subset of 'emojiRanges' whose +-- UTS #46 status is @mapped@; cross-tool ambiguous. +-- +-- Both tables feed the @EMOJIOK@ relaxation in +-- "Text.IDNA2008.Internal.Parse"; see the @EMOJIOK@ pattern in +-- "Text.IDNA2008.Internal.Flags" for the policy rationale. +module Text.IDNA2008.Internal.Emoji.Data + ( emojiRangeCount + , emojiRanges + , mappedEmojiRangeCount + , mappedEmojiRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +-- | Number of (start, end) range pairs in 'emojiRanges'. +emojiRangeCount :: Int +emojiRangeCount = 151 + +-- | Codepoints with Unicode @Emoji=Yes@, alternating +-- start\/end. +emojiRanges :: ByteArray +emojiRanges = PBA.byteArrayFromList + [ (0x000023 :: Word32) + , 0x000023 + , 0x00002A + , 0x00002A + , 0x000030 + , 0x000039 + , 0x0000A9 + , 0x0000A9 + , 0x0000AE + , 0x0000AE + , 0x00203C + , 0x00203C + , 0x002049 + , 0x002049 + , 0x002122 + , 0x002122 + , 0x002139 + , 0x002139 + , 0x002194 + , 0x002199 + , 0x0021A9 + , 0x0021AA + , 0x00231A + , 0x00231B + , 0x002328 + , 0x002328 + , 0x0023CF + , 0x0023CF + , 0x0023E9 + , 0x0023F3 + , 0x0023F8 + , 0x0023FA + , 0x0024C2 + , 0x0024C2 + , 0x0025AA + , 0x0025AB + , 0x0025B6 + , 0x0025B6 + , 0x0025C0 + , 0x0025C0 + , 0x0025FB + , 0x0025FE + , 0x002600 + , 0x002604 + , 0x00260E + , 0x00260E + , 0x002611 + , 0x002611 + , 0x002614 + , 0x002615 + , 0x002618 + , 0x002618 + , 0x00261D + , 0x00261D + , 0x002620 + , 0x002620 + , 0x002622 + , 0x002623 + , 0x002626 + , 0x002626 + , 0x00262A + , 0x00262A + , 0x00262E + , 0x00262F + , 0x002638 + , 0x00263A + , 0x002640 + , 0x002640 + , 0x002642 + , 0x002642 + , 0x002648 + , 0x002653 + , 0x00265F + , 0x002660 + , 0x002663 + , 0x002663 + , 0x002665 + , 0x002666 + , 0x002668 + , 0x002668 + , 0x00267B + , 0x00267B + , 0x00267E + , 0x00267F + , 0x002692 + , 0x002697 + , 0x002699 + , 0x002699 + , 0x00269B + , 0x00269C + , 0x0026A0 + , 0x0026A1 + , 0x0026A7 + , 0x0026A7 + , 0x0026AA + , 0x0026AB + , 0x0026B0 + , 0x0026B1 + , 0x0026BD + , 0x0026BE + , 0x0026C4 + , 0x0026C5 + , 0x0026C8 + , 0x0026C8 + , 0x0026CE + , 0x0026CF + , 0x0026D1 + , 0x0026D1 + , 0x0026D3 + , 0x0026D4 + , 0x0026E9 + , 0x0026EA + , 0x0026F0 + , 0x0026F5 + , 0x0026F7 + , 0x0026FA + , 0x0026FD + , 0x0026FD + , 0x002702 + , 0x002702 + , 0x002705 + , 0x002705 + , 0x002708 + , 0x00270D + , 0x00270F + , 0x00270F + , 0x002712 + , 0x002712 + , 0x002714 + , 0x002714 + , 0x002716 + , 0x002716 + , 0x00271D + , 0x00271D + , 0x002721 + , 0x002721 + , 0x002728 + , 0x002728 + , 0x002733 + , 0x002734 + , 0x002744 + , 0x002744 + , 0x002747 + , 0x002747 + , 0x00274C + , 0x00274C + , 0x00274E + , 0x00274E + , 0x002753 + , 0x002755 + , 0x002757 + , 0x002757 + , 0x002763 + , 0x002764 + , 0x002795 + , 0x002797 + , 0x0027A1 + , 0x0027A1 + , 0x0027B0 + , 0x0027B0 + , 0x0027BF + , 0x0027BF + , 0x002934 + , 0x002935 + , 0x002B05 + , 0x002B07 + , 0x002B1B + , 0x002B1C + , 0x002B50 + , 0x002B50 + , 0x002B55 + , 0x002B55 + , 0x003030 + , 0x003030 + , 0x00303D + , 0x00303D + , 0x003297 + , 0x003297 + , 0x003299 + , 0x003299 + , 0x01F004 + , 0x01F004 + , 0x01F0CF + , 0x01F0CF + , 0x01F170 + , 0x01F171 + , 0x01F17E + , 0x01F17F + , 0x01F18E + , 0x01F18E + , 0x01F191 + , 0x01F19A + , 0x01F1E6 + , 0x01F1FF + , 0x01F201 + , 0x01F202 + , 0x01F21A + , 0x01F21A + , 0x01F22F + , 0x01F22F + , 0x01F232 + , 0x01F23A + , 0x01F250 + , 0x01F251 + , 0x01F300 + , 0x01F321 + , 0x01F324 + , 0x01F393 + , 0x01F396 + , 0x01F397 + , 0x01F399 + , 0x01F39B + , 0x01F39E + , 0x01F3F0 + , 0x01F3F3 + , 0x01F3F5 + , 0x01F3F7 + , 0x01F4FD + , 0x01F4FF + , 0x01F53D + , 0x01F549 + , 0x01F54E + , 0x01F550 + , 0x01F567 + , 0x01F56F + , 0x01F570 + , 0x01F573 + , 0x01F57A + , 0x01F587 + , 0x01F587 + , 0x01F58A + , 0x01F58D + , 0x01F590 + , 0x01F590 + , 0x01F595 + , 0x01F596 + , 0x01F5A4 + , 0x01F5A5 + , 0x01F5A8 + , 0x01F5A8 + , 0x01F5B1 + , 0x01F5B2 + , 0x01F5BC + , 0x01F5BC + , 0x01F5C2 + , 0x01F5C4 + , 0x01F5D1 + , 0x01F5D3 + , 0x01F5DC + , 0x01F5DE + , 0x01F5E1 + , 0x01F5E1 + , 0x01F5E3 + , 0x01F5E3 + , 0x01F5E8 + , 0x01F5E8 + , 0x01F5EF + , 0x01F5EF + , 0x01F5F3 + , 0x01F5F3 + , 0x01F5FA + , 0x01F64F + , 0x01F680 + , 0x01F6C5 + , 0x01F6CB + , 0x01F6D2 + , 0x01F6D5 + , 0x01F6D8 + , 0x01F6DC + , 0x01F6E5 + , 0x01F6E9 + , 0x01F6E9 + , 0x01F6EB + , 0x01F6EC + , 0x01F6F0 + , 0x01F6F0 + , 0x01F6F3 + , 0x01F6FC + , 0x01F7E0 + , 0x01F7EB + , 0x01F7F0 + , 0x01F7F0 + , 0x01F90C + , 0x01F93A + , 0x01F93C + , 0x01F945 + , 0x01F947 + , 0x01F9FF + , 0x01FA70 + , 0x01FA7C + , 0x01FA80 + , 0x01FA8A + , 0x01FA8E + , 0x01FAC6 + , 0x01FAC8 + , 0x01FAC8 + , 0x01FACD + , 0x01FADC + , 0x01FADF + , 0x01FAEA + , 0x01FAEF + , 0x01FAF8 + ] + +-- | Number of (start, end) range pairs in +-- 'mappedEmojiRanges'. +mappedEmojiRangeCount :: Int +mappedEmojiRangeCount = 12 + +-- | Subset of 'emojiRanges' whose UTS #46 status is +-- @mapped@, alternating start\/end. Excluded from +-- the @EMOJIOK@ relaxation; see module haddock for +-- the rationale. +mappedEmojiRanges :: ByteArray +mappedEmojiRanges = PBA.byteArrayFromList + [ (0x00203C :: Word32) + , 0x00203C + , 0x002049 + , 0x002049 + , 0x002122 + , 0x002122 + , 0x002139 + , 0x002139 + , 0x0024C2 + , 0x0024C2 + , 0x003297 + , 0x003297 + , 0x003299 + , 0x003299 + , 0x01F201 + , 0x01F202 + , 0x01F21A + , 0x01F21A + , 0x01F22F + , 0x01F22F + , 0x01F232 + , 0x01F23A + , 0x01F250 + , 0x01F251 + ] diff --git a/internal/Text/IDNA2008/Internal/Error.hs b/internal/Text/IDNA2008/Internal/Error.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Error.hs @@ -0,0 +1,171 @@ +-- | +-- Module : Text.IDNA2008.Internal.Error +-- Description : Error type returned by the parser. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Errors raised by the IDNA-aware domain-name parser. Failures +-- are reported with enough context (label index, byte offset +-- where useful) for callers to pinpoint the offending part of +-- the input without the library having to retain or echo back +-- the raw input bytes. +module Text.IDNA2008.Internal.Error + ( IdnaError(..) + , LabelReason(..) + , AceReason(..) + , BidiRuleViolation(..) + ) where + +import Text.IDNA2008.Internal.LabelForm (LabelForm) + +-- | Possible failure modes when parsing a presentation-form +-- domain name into wire form, including IDN encoding / +-- decoding errors. +data IdnaError + = -- | An empty (non-final) label was encountered, e.g. @\"foo..bar\"@. + ErrEmptyLabel + !Int -- ^ Label index (0-based) + | -- | A label exceeds 63 wire octets. The 'Int' is the actual length. + ErrLabelTooLong + !Int -- ^ Label index + !Int -- ^ Actual length + | -- | The domain's total wire length would exceed 255 bytes. + ErrNameTooLong + !Int -- ^ Actual length. + | -- | A backslash was followed by an invalid escape sequence. + ErrBadEscape + !Int -- ^ Label index + !(Maybe Int) -- ^ The zero-based byte offset into the original input + -- where the offending escape begins, when known. + | -- | The input contained an ill-formed UTF-8 byte sequence. + ErrInvalidUtf8 + !Int -- ^ Label index + !(Maybe Int) -- ^ The byte offset into the original input where the + -- ill-formed byte sequence begins, when known. The label + -- index is often @-1@ because the error is detected + -- before any label boundary is recognised. + | -- | The input includes a label that has no coherent presentation form: it + -- mixes raw-byte escape syntax (@\'\\DDD\'@ or @\'\\X\'@, only allowed + -- in @OCTET@ labels) with a codepoint outside the 8-bit range (which an + -- @OCTET@ label cannot carry). + ErrUnpresentableLabel + !Int -- ^ Label index + | -- | A codepoint outside the legal Unicode range (above @U+10FFFF@) or + -- in the surrogate range was observed. + ErrCodepointTooLarge + !Int -- ^ Label index. + !Int -- ^ Codepoint. + | -- | The label was successfully classified, but its 'LabelForm' is not + -- in the caller-supplied set of permitted forms. + ErrFormNotAllowed + !Int -- ^ Label index. + !LabelForm -- ^ The form of the rejected label. + | -- | A label that took the validation path failed it; see + ErrLabelInvalid + !Int -- ^ Label index. + !LabelReason -- ^ The specific cause. + | -- | A label looked like an ACE label (@\"xn--...\"@) but + -- failed validation: Punycode decode failed, decoded form + -- is not a valid IDN label, or re-encoding does not match. + ErrAceInvalid + !Int -- ^ Label index. + !AceReason -- ^ Reason + | -- | The label violated a cross-label bidirectional-text rule. + -- Raised when one or more labels in the name contains right-to-left + -- content; the rules then apply to /every/ label in the + -- name, including pure left-to-right siblings. + ErrCrossLabelBidi + !Int -- ^ Label index. + !BidiRuleViolation -- ^ Actual problem + | -- | Internal Punycode arithmetic overflow during encode or + -- decode. Should not occur for compliant inputs of bounded + -- length. + ErrPunycodeOverflow + !Int -- ^ Label index. + deriving (Eq, Show) + +-- | Why a label failed validation. +data LabelReason + = DisallowedCodepoint !Int + -- ^ A codepoint not permitted in any IDN label. Carries + -- the offending codepoint regardless of whether it's + -- ASCII (e.g. an uppercase letter, an underscore) or + -- non-ASCII (a symbol, dingbat, etc.). + | ContextRule !Int + -- ^ A codepoint admissible only in specific contexts (a + -- joiner, an Arabic-Indic digit, etc.) appeared in a + -- context where its rule isn't satisfied. + | NotNFC + -- ^ The label is not in Unicode Normalization Form C. + | LabelBidi !BidiRuleViolation + -- ^ A bidirectional-text rule was violated within the + -- label. See 'BidiRuleViolation' for the specific rule. + | HyphenViolation + -- ^ Per RFC 5891 section 4.2.3.1: the label has a leading + -- hyphen, a trailing hyphen, or hyphens at both positions + -- 3 and 4. The latter form reserves those slots for the + -- ACE prefix (@\"xn--\"@), so a U-label that contains + -- @\"--\"@ at positions 3-4 would visually mimic an + -- A-label without being one and is rejected. + | LeadingCombiningMark !Int + -- ^ The first codepoint of the label has + -- @General_Category@ in @{Mn, Mc, Me}@. Per RFC 5891 + -- section 4.2.3.2, a U-label must not begin with a + -- combining mark. Carries the offending codepoint. + deriving (Eq, Show) + +-- | Why an ACE-prefixed (@\"xn--\"@) label failed validation. +data AceReason + = BadPunycode + -- ^ Punycode body could not be decoded (truncated, bad + -- delimiter, non-base-36 digit). + | DecodedInvalid !LabelReason + -- ^ Decoding succeeded but the result is not a valid IDN + -- label. See 'LabelReason' for the specific cause. + | RoundTripMismatch + -- ^ Decoded label re-encoded to a form that does not match + -- the input ACE bytes. The round-trip check is + -- case-insensitive. + deriving (Eq, Show) + +-- | Which bidirectional-text rule a label violates, with enough +-- granularity for a caller to format a useful diagnostic +-- without having to consult the rule text. Used as the +-- payload of 'LabelBidi' (per-label check) and +-- 'ErrCrossLabelBidi' (cross-label check). +-- +-- The rules apply when a label contains right-to-left text +-- (Hebrew, Arabic, ...) or an Arabic-Indic digit. +data BidiRuleViolation + = BidiRule1FirstNotLRAL + -- ^ The first codepoint of a bidirectional label + -- was not a letter from a known left-to-right or + -- right-to-left script. Most often catches digit-, mark-, + -- or punctuation-leading labels in a name that also has a + -- right-to-left component (e.g. @\"_tcp\"@, @\"123\"@). + | BidiRule2RTLDisallowed + -- ^ A right-to-left label (one whose first codepoint is a + -- right-to-left letter) contained a codepoint that is not + -- allowed in such a label — typically a left-to-right + -- letter, or an embedding\/override\/isolate format + -- character. + | BidiRule3RTLBadEnd + -- ^ A right-to-left label's last non-mark codepoint was + -- not a letter or digit. + | BidiRule4ENANMix + -- ^ A right-to-left label mixed European-style digits + -- (ASCII or extended Arabic-Indic) with Arabic-Indic + -- digits. The two digit families cannot coexist in the + -- same label. + | BidiRule5LTRDisallowed + -- ^ A left-to-right label (one whose first codepoint is a + -- left-to-right letter) contained a codepoint that is not + -- allowed in such a label — typically a right-to-left + -- letter or an Arabic-Indic digit. + | BidiRule6LTRBadEnd + -- ^ A left-to-right label's last non-mark codepoint was + -- not a letter or a European-style digit. + deriving (Eq, Show) diff --git a/internal/Text/IDNA2008/Internal/Flags.hs b/internal/Text/IDNA2008/Internal/Flags.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Flags.hs @@ -0,0 +1,320 @@ +-- | +-- Module : Text.IDNA2008.Internal.Flags +-- Description : Parser and presentation flags. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Knobs that control how strictly the parser validates a domain +-- name and how leniently it cleans up its input. Composed with +-- @('<>')@; tested with 'meetsIdnaFlags'. The data constructor +-- is intentionally not exposed; callers compose values from the +-- singleton patterns. +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} + +module Text.IDNA2008.Internal.Flags + ( -- * Flag set with bundled singleton patterns + IdnaFlags( ALABELCHECK + , NFCCHECK + , EMOJIOK + , MAPDOTS + , MAPNFC + , MAPCASE + , MAPWIDTH + , BIDICHECK + , ASCIIFALLBACK + , MAPUTS46 + ) + , allIdnaMappings + , defaultIdnaFlags + , effectiveIdnaFlags + , meetsIdnaFlags + , withoutIdnaFlags + + -- * Command-line option syntax + , idnaFlagsTokens + , idnaFlagsPresets + , parseIdnaFlags + , parseIdnaFlagsStr + ) where + +import Control.Monad ((>=>)) +import Data.Bits ((.|.), (.&.), complement, unsafeShiftR) +import Data.ByteString (ByteString) +import Data.List (intercalate) +import Data.Word (Word16) + +import Text.IDNA2008.Internal.Tokens + ( TokenEntry(..), Preset, asciiByteString, parseTokens ) + +-- | Flag set for the parser, stored as a bitmask of one-bit +-- pattern singletons. +newtype IdnaFlags = IdnaFlags Word16 + deriving Eq + +instance Semigroup IdnaFlags where + IdnaFlags a <> IdnaFlags b = IdnaFlags (a .|. b) + {-# INLINE (<>) #-} + +instance Monoid IdnaFlags where + mempty = IdnaFlags 0 + {-# INLINE mempty #-} + +instance Show IdnaFlags where + show o = case flagNames o of + [] -> "mempty" + ns -> intercalate "," ns + +flagNames :: IdnaFlags -> [String] +flagNames (IdnaFlags w) = go names w + where + names = [ "alabel-check", "nfc-check", "emoji-ok", + "map-dots", "map-nfc", "map-case", "map-width", + "bidi-check", "ascii-fallback", + "map-uts46" ] + go _ 0 = [] + go [] _ = [] + go (s:ss) n + | odd n = s : go ss (n `unsafeShiftR` 1) + | otherwise = go ss (n `unsafeShiftR` 1) + +---------------------------------------------------------------------- +-- Pattern singletons +---------------------------------------------------------------------- + +-- | Strict A-label check. An ACE-prefixed label whose +-- Punycode body doesn't decode to a valid IDN label, or whose +-- re-encoding doesn't match the input bytes, classifies as +-- 'Text.IDNA2008.Internal.LabelForm.FAKEA' rather than +-- 'Text.IDNA2008.Internal.LabelForm.ALABEL'. Without this +-- flag, every well-formed ACE-prefixed LDH label is reported as +-- 'Text.IDNA2008.Internal.LabelForm.ALABEL'. On in +-- 'defaultIdnaFlags'. +pattern ALABELCHECK :: IdnaFlags +pattern ALABELCHECK = IdnaFlags 0x0001 + +-- | Require Unicode Normalization Form C on labels with non-ASCII +-- content. Labels with combining marks in non-canonical order, +-- or decomposed sequences with a precomposed equivalent, are +-- rejected. Without this flag no normalization check is +-- performed, and the same display string can map to multiple +-- distinct A-labels on the wire. On in 'defaultIdnaFlags'. +pattern NFCCHECK :: IdnaFlags +pattern NFCCHECK = IdnaFlags 0x0002 + +-- | Diagnostic relaxation: codepoints that carry the Unicode +-- @Emoji=Yes@ property are admitted even if they're otherwise +-- rejected by the IDN allowed-codepoint rules. Useful when +-- analysing real-world IDN data that includes emoji +-- registrations. Off by default. +-- +-- /Exclusion/: emoji codepoints whose UTS #46 status is also +-- @mapped@ are /not/ admitted by this flag. These codepoints +-- resolve ambiguously across the ecosystem: browsers following +-- UTS #46 lookup processing apply the fold and reach the mapped +-- target, while admit-as-is tooling reaches the unmapped form. +pattern EMOJIOK :: IdnaFlags +pattern EMOJIOK = IdnaFlags 0x0004 + +-- | Treat the East Asian period characters (the ideographic +-- period @U+3002@ and its fullwidth and halfwidth variants) as +-- label separators, just like @\'.\'@ (@U+002E@). Off by +-- default; some user-input contexts need this. +pattern MAPDOTS :: IdnaFlags +pattern MAPDOTS = IdnaFlags 0x0008 + +-- | Combine letter-and-accent sequences into single characters +-- where possible, before validation. An input with decomposed +-- letter+combining-mark sequences is normalised to its +-- precomposed equivalent. Affects only labels with non-ASCII +-- content. +pattern MAPNFC :: IdnaFlags +pattern MAPNFC = IdnaFlags 0x0010 + +-- | Lowercase the input. Without this flag the parser +-- preserves input case verbatim, so uppercase ASCII in a +-- pure-LDH label survives into the wire form, and an +-- uppercase letter in a label with non-ASCII content is +-- rejected as a @DISALLOWED@ codepoint under strict +-- IDNA2008. With the flag set: ASCII A-Z is lowercased +-- before classification, and any other letter with a known +-- Unicode lowercase form is lowercased on the U-label path. +-- After mapping, validation runs against the lowercased +-- form. +pattern MAPCASE :: IdnaFlags +pattern MAPCASE = IdnaFlags 0x0020 + +-- | Convert fullwidth and halfwidth characters to their normal- +-- width form before validation. Fullwidth Latin letters become +-- ordinary ASCII letters, halfwidth katakana becomes regular +-- katakana, etc. Implies 'MAPDOTS', because two of the +-- fullwidth\/halfwidth forms are label-separator characters that +-- must be recognised before per-label processing. +pattern MAPWIDTH :: IdnaFlags +pattern MAPWIDTH = IdnaFlags 0x0040 + +-- | Apply the bidirectional-text rules. When right-to-left +-- scripts (Hebrew, Arabic, ...) appear in a domain name, this +-- check enforces a small set of constraints (per label and +-- across labels) designed to prevent visual confusion in +-- mixed-direction text. On in 'defaultIdnaFlags'. +pattern BIDICHECK :: IdnaFlags +pattern BIDICHECK = IdnaFlags 0x0080 + +-- | Presentation-time policy: when 'BIDICHECK' would reject a +-- domain at presentation time (because the cross-label +-- constraints fail), render it as ASCII anyway, with each label +-- in its ACE-prefixed form. Useful for displaying domain +-- names as part of running text where the goal is \"show the user +-- something readable\" and a Bidi-violating name should degrade +-- to its ACE-prefixed form rather than produce an error. +-- Implies 'BIDICHECK'. +pattern ASCIIFALLBACK :: IdnaFlags +pattern ASCIIFALLBACK = IdnaFlags 0x0100 + +-- | Apply a small hand-curated subset of UTS #46 character mappings +-- before validation, beyond the four RFC 5895 input mappings +-- (@MAPDOTS@, @MAPCASE@, @MAPNFC@, @MAPWIDTH@). The adopted set is +-- the in\\/out classification documented at the top of +-- "Text.IDNA2008.Internal.UTS46"; in summary it covers the +-- Japanese era IME shortcuts (five codepoints at @U+32FF@ and +-- @U+337B..U+337E@) and the circled-CJK ideographs at +-- @U+3244..U+3247@ and @U+3280..U+32B0@ (with @U+3297@ and +-- @U+3299@ carved out) that fold to a single base ideograph each. +-- Every other UTS #46 mapping is rejected on principle; see the +-- rationale block for the OUT list. +-- +-- /Implies/ 'allIdnaMappings': UTS #46 preprocessing subsumes the +-- RFC 5895 input mappings, so requesting @MAPUTS46@ alone enables +-- 'MAPNFC', 'MAPDOTS', 'MAPCASE', and 'MAPWIDTH' as well. +-- +-- Beyond strict IDNA2008. /Not/ included in 'allIdnaMappings'; +-- callers must request it explicitly. +pattern MAPUTS46 :: IdnaFlags +pattern MAPUTS46 = IdnaFlags 0x0200 + +---------------------------------------------------------------------- +-- Defaults and implications +---------------------------------------------------------------------- + +-- | Default flag set: 'ALABELCHECK', 'NFCCHECK', 'BIDICHECK'. +-- The spec-conformant choices: check whether ACE-prefixed +-- LDH-labels are actually A-labels, or just /fake/ A-labels, +-- require NFC form in U-labels, and enforce BIDI rules. Callers +-- wanting looser semantics pass 'mempty' or a custom subset. +defaultIdnaFlags :: IdnaFlags +defaultIdnaFlags = ALABELCHECK <> NFCCHECK <> BIDICHECK + +-- | All four optional RFC 5895 mappings. To be used only after +-- reading the caveats in the +-- [Introduction](https://www.rfc-editor.org/rfc/rfc5895.html#section-1) +-- +-- This expands to the union (@'mconcat'@) of 'MAPNFC', 'MAPDOTS', +-- 'MAPCASE', and 'MAPWIDTH'. These mappings are outside the core +-- IDNA2008 specification but are part of RFC 5895; they are not +-- applied by default. +-- +-- Deliberately /excludes/ 'MAPUTS46', which is a separate +-- beyond-IDNA2008 opt-in covering a hand-curated subset of UTS #46 +-- compatibility mappings, on the same footing as 'EMOJIOK' rather +-- than the RFC 5895 set. +allIdnaMappings :: IdnaFlags +allIdnaMappings = MAPNFC <> MAPDOTS <> MAPCASE <> MAPWIDTH + +-- | Lift implied bits in an 'IdnaFlags' set: +-- +-- * 'MAPUTS46' implies 'allIdnaMappings' ('MAPNFC', 'MAPDOTS', +-- 'MAPCASE', 'MAPWIDTH'). UTS #46-style preprocessing +-- subsumes the four RFC 5895 input mappings — a user +-- opting into the UTS #46 carve-out implicitly wants +-- case-folded, width-folded, NFC-normalised input with +-- East-Asian dot recognition; running MAPUTS46 without +-- those would leave the input partially pre-processed and +-- produce inconsistent fold outcomes. +-- * 'MAPWIDTH' implies 'MAPDOTS', because the wide\/narrow +-- decompositions of label separators (e.g. @U+FF0E@, +-- @U+FF61@) need to be recognised at split time, not after. +-- * 'ASCIIFALLBACK' implies 'BIDICHECK', because the fallback +-- policy only kicks in when a check has flagged a violation. +effectiveIdnaFlags :: IdnaFlags -> IdnaFlags +effectiveIdnaFlags !flags = + let !f0 = if MAPUTS46 `meetsIdnaFlags` flags + then flags <> allIdnaMappings else flags + !f1 = if MAPWIDTH `meetsIdnaFlags` f0 + then f0 <> MAPDOTS else f0 + !f2 = if ASCIIFALLBACK `meetsIdnaFlags` f1 + then f1 <> BIDICHECK else f1 + in f2 +{-# INLINE effectiveIdnaFlags #-} + +-- | Does the first 'IdnaFlags' value satisfy the requirement +-- expressed by the second? Reads as +-- @flag \`meetsIdnaFlags\` flags@. +meetsIdnaFlags :: IdnaFlags -> IdnaFlags -> Bool +IdnaFlags a `meetsIdnaFlags` IdnaFlags b = a == a .&. b +{-# INLINE meetsIdnaFlags #-} + +-- | @set \`withoutIdnaFlags\` flag@ removes every bit of @flag@ +-- from @set@. +withoutIdnaFlags :: IdnaFlags -> IdnaFlags -> IdnaFlags +IdnaFlags a `withoutIdnaFlags` IdnaFlags b = + IdnaFlags (a .&. complement b) +{-# INLINE withoutIdnaFlags #-} + +---------------------------------------------------------------------- +-- Command-line option syntax +---------------------------------------------------------------------- + +-- | Single-bit token table for the command-line option syntax. +idnaFlagsTokens :: [TokenEntry IdnaFlags] +idnaFlagsTokens = + [ TokenEntry ALABELCHECK "alabel-check" ["xncheck"] + , TokenEntry NFCCHECK "nfc-check" [] + , TokenEntry EMOJIOK "emoji-ok" [] + , TokenEntry MAPDOTS "map-dots" ["dmap"] + , TokenEntry MAPNFC "map-nfc" ["nmap"] + , TokenEntry MAPCASE "map-case" ["cmap"] + , TokenEntry MAPWIDTH "map-width" ["wmap"] + , TokenEntry BIDICHECK "bidi-check" [] + , TokenEntry ASCIIFALLBACK "ascii-fallback" [] + , TokenEntry MAPUTS46 "map-uts46" [] + ] + +-- | Preset names that expand to multi-bit flag sets. Useful as +-- command line option shorthands. +idnaFlagsPresets :: [Preset IdnaFlags] +idnaFlagsPresets = + [ ("map", allIdnaMappings) + , ("umap", allIdnaMappings <> MAPUTS46) + ] + +-- | Parse a comma-separated CLI value into an 'IdnaFlags' set. +-- The first argument is the application-specific default value. +-- +-- Each comma-separated token may optionally be prefixed with +-- @\'+\'@ (additive, the default) or @\'-\'@ (subtractive). +-- If the first token has a sign prefix, the running result is +-- seeded with the supplied default; otherwise the running +-- result starts empty and the tokens replace the default +-- cleanly. +-- +-- Tokens are matched case-insensitively, with unambiguous prefix +-- matching (3-character minimum) as a fallback when there's no +-- exact match. The token @default@ or an unambiguous prefix +-- matches the supplied default value. +parseIdnaFlags :: IdnaFlags -- ^ Application-specific default + -> ByteString -- ^ Tokens to parse + -> Either String IdnaFlags +parseIdnaFlags !defF = + parseTokens idnaFlagsTokens + (("default", defF) : idnaFlagsPresets) + defF + withoutIdnaFlags + +-- | 'String'-based wrapper for 'parseIdnaFlags': validates that +-- the input is pure ASCII. +parseIdnaFlagsStr :: IdnaFlags -> String -> Either String IdnaFlags +parseIdnaFlagsStr !defF = asciiByteString >=> parseIdnaFlags defF diff --git a/internal/Text/IDNA2008/Internal/Joining.hs b/internal/Text/IDNA2008/Internal/Joining.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Joining.hs @@ -0,0 +1,120 @@ +-- | +-- Module : Text.IDNA2008.Internal.Joining +-- Description : Joining_Type and Virama (Canonical_Combining_Class +-- = 9) predicates for the CONTEXTJ contextual rules +-- (RFC 5892 A.1, A.2). +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Predicates over the Unicode Joining_Type and +-- Canonical_Combining_Class properties consumed by the CONTEXTJ +-- contextual rules. Backed by the compact range tables in +-- "Text.IDNA2008.Internal.Joining.Data". +{-# LANGUAGE BangPatterns #-} + +module Text.IDNA2008.Internal.Joining + ( -- * Joining_Type predicates + jtIsLeftOrDual + , jtIsRightOrDual + , jtIsTransparent + + -- * Canonical_Combining_Class + , isVirama + ) where + +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word8, Word32) + +import Text.IDNA2008.Internal.Ranges (inRanges) +import Text.IDNA2008.Internal.Joining.Data + ( jtRangeCount + , jtRangeStarts + , jtRangeEnds + , jtRangeTags + , jtTagL + , jtTagR + , jtTagD + , jtTagT + , viramaRangeCount + , viramaRanges + ) + +-- | Is @cp@'s Joining_Type either @L@ (Left_Joining) or @D@ +-- (Dual_Joining)? RFC 5892 A.1 consults this on the codepoint +-- to the left of @U+200C@ (after skipping any 'jtIsTransparent' +-- run). +jtIsLeftOrDual :: Int -> Bool +jtIsLeftOrDual !cp = case lookupJt cp of + Nothing -> False + Just t -> t == jtTagL || t == jtTagD +{-# INLINE jtIsLeftOrDual #-} + +-- | Is @cp@'s Joining_Type either @R@ (Right_Joining) or @D@ +-- (Dual_Joining)? RFC 5892 A.1 consults this on the codepoint +-- to the right of @U+200C@ (after skipping any 'jtIsTransparent' +-- run). +jtIsRightOrDual :: Int -> Bool +jtIsRightOrDual !cp = case lookupJt cp of + Nothing -> False + Just t -> t == jtTagR || t == jtTagD +{-# INLINE jtIsRightOrDual #-} + +-- | Is @cp@'s Joining_Type @T@ (Transparent)? Transparent +-- codepoints (combining marks etc.) are skipped over by the +-- CONTEXTJ A.1 regex. +jtIsTransparent :: Int -> Bool +jtIsTransparent !cp = case lookupJt cp of + Nothing -> False + Just t -> t == jtTagT +{-# INLINE jtIsTransparent #-} + +-- | Is @cp@'s Canonical_Combining_Class equal to 9 (Virama)? +-- Both A.1 (ZWNJ) and A.2 (ZWJ) admit their codepoint when the +-- immediately preceding codepoint is a Virama. +isVirama :: Int -> Bool +isVirama = inRanges viramaRanges viramaRangeCount +{-# INLINE isVirama #-} + +---------------------------------------------------------------------- +-- Lookup helpers +---------------------------------------------------------------------- + +-- | Look up the Joining_Type tag for @cp@ in the (start, end, tag) +-- triple table. Returns 'Nothing' for codepoints whose +-- Joining_Type is not in the relevant set @{L, R, D, T}@ (which +-- includes @C@, @U@, and unassigned codepoints). +lookupJt :: Int -> Maybe Word8 +lookupJt !cp + | jtRangeCount == 0 = Nothing + | cp < startAt 0 = Nothing + | otherwise = + let !idx = bsearch 0 (jtRangeCount - 1) + in if cp <= endAt idx + then Just (tagAt idx) + else Nothing + where + startAt :: Int -> Int + startAt !i = fromIntegral (indexByteArray jtRangeStarts i :: Word32) + {-# INLINE startAt #-} + + endAt :: Int -> Int + endAt !i = fromIntegral (indexByteArray jtRangeEnds i :: Word32) + {-# INLINE endAt #-} + + tagAt :: Int -> Word8 + tagAt !i = indexByteArray jtRangeTags i + {-# INLINE tagAt #-} + + bsearch :: Int -> Int -> Int + bsearch !lo !hi + | lo == hi = lo + | otherwise = + let !mid = (lo + hi + 1) `quot` 2 + in if startAt mid <= cp + then bsearch mid hi + else bsearch lo (mid - 1) +{-# INLINE lookupJt #-} + diff --git a/internal/Text/IDNA2008/Internal/Joining/Data.hs b/internal/Text/IDNA2008/Internal/Joining/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Joining/Data.hs @@ -0,0 +1,1797 @@ +-- | +-- Module : Text.IDNA2008.Internal.Joining.Data +-- Description : Compact codepoint range tables for the Unicode +-- Joining_Type and Virama (Canonical_Combining_Class +-- = 9) properties consulted by the CONTEXTJ +-- contextual rules. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaJoining.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/extracted/ +-- Aligned with: Unicode 17.0.0 +-- Joining_Type ranges: 524 +-- Virama ranges: 58 +-- +-- The four Joining_Type values relevant to RFC 5892 A.1 are encoded +-- as a 'Word8' tag per range: +-- +-- * 1 = L (Left_Joining) +-- * 2 = R (Right_Joining) +-- * 3 = D (Dual_Joining) +-- * 4 = T (Transparent) +-- +-- Joining_Type=C (Join_Causing, including ZWJ\/ZWNJ themselves and +-- Arabic Tatweel @U+0640@) and Joining_Type=U (Non_Joining) are not +-- encoded: at any position a codepoint with one of those types is a +-- \"stop\" for the regex. +module Text.IDNA2008.Internal.Joining.Data + ( -- * Joining_Type + jtRangeCount + , jtRangeStarts + , jtRangeEnds + , jtRangeTags + + -- * Joining_Type tag values + , jtTagL + , jtTagR + , jtTagD + , jtTagT + + -- * Virama (CCC = 9) + , viramaRangeCount + , viramaRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- Joining_Type +---------------------------------------------------------------------- + +-- | Tag value for Joining_Type=L (Left_Joining). +jtTagL :: Word8 +jtTagL = 1 + +-- | Tag value for Joining_Type=R (Right_Joining). +jtTagR :: Word8 +jtTagR = 2 + +-- | Tag value for Joining_Type=D (Dual_Joining). +jtTagD :: Word8 +jtTagD = 3 + +-- | Tag value for Joining_Type=T (Transparent). +jtTagT :: Word8 +jtTagT = 4 + +-- | Number of joining-type ranges. +jtRangeCount :: Int +jtRangeCount = 524 + +-- | Inclusive range starts, sorted ascending. +jtRangeStarts :: ByteArray +jtRangeStarts = PBA.byteArrayFromList + [ (0x0000AD :: Word32) + , 0x000300 + , 0x000483 + , 0x000591 + , 0x0005BF + , 0x0005C1 + , 0x0005C4 + , 0x0005C7 + , 0x000610 + , 0x00061C + , 0x000620 + , 0x000622 + , 0x000626 + , 0x000627 + , 0x000628 + , 0x000629 + , 0x00062A + , 0x00062F + , 0x000633 + , 0x000641 + , 0x000648 + , 0x000649 + , 0x00064B + , 0x00066E + , 0x000670 + , 0x000671 + , 0x000675 + , 0x000678 + , 0x000688 + , 0x00069A + , 0x0006C0 + , 0x0006C1 + , 0x0006C3 + , 0x0006CC + , 0x0006CD + , 0x0006CE + , 0x0006CF + , 0x0006D0 + , 0x0006D2 + , 0x0006D5 + , 0x0006D6 + , 0x0006DF + , 0x0006E7 + , 0x0006EA + , 0x0006EE + , 0x0006FA + , 0x0006FF + , 0x00070F + , 0x000710 + , 0x000711 + , 0x000712 + , 0x000715 + , 0x00071A + , 0x00071E + , 0x00071F + , 0x000728 + , 0x000729 + , 0x00072A + , 0x00072B + , 0x00072C + , 0x00072D + , 0x00072F + , 0x000730 + , 0x00074D + , 0x00074E + , 0x000759 + , 0x00075C + , 0x00076B + , 0x00076D + , 0x000771 + , 0x000772 + , 0x000773 + , 0x000775 + , 0x000778 + , 0x00077A + , 0x0007A6 + , 0x0007CA + , 0x0007EB + , 0x0007FD + , 0x000816 + , 0x00081B + , 0x000825 + , 0x000829 + , 0x000840 + , 0x000841 + , 0x000846 + , 0x000848 + , 0x000849 + , 0x00084A + , 0x000854 + , 0x000855 + , 0x000856 + , 0x000859 + , 0x000860 + , 0x000862 + , 0x000867 + , 0x000868 + , 0x000869 + , 0x000870 + , 0x000886 + , 0x000889 + , 0x00088E + , 0x00088F + , 0x000897 + , 0x0008A0 + , 0x0008AA + , 0x0008AE + , 0x0008AF + , 0x0008B1 + , 0x0008B3 + , 0x0008B9 + , 0x0008BA + , 0x0008CA + , 0x0008E3 + , 0x00093A + , 0x00093C + , 0x000941 + , 0x00094D + , 0x000951 + , 0x000962 + , 0x000981 + , 0x0009BC + , 0x0009C1 + , 0x0009CD + , 0x0009E2 + , 0x0009FE + , 0x000A01 + , 0x000A3C + , 0x000A41 + , 0x000A47 + , 0x000A4B + , 0x000A51 + , 0x000A70 + , 0x000A75 + , 0x000A81 + , 0x000ABC + , 0x000AC1 + , 0x000AC7 + , 0x000ACD + , 0x000AE2 + , 0x000AFA + , 0x000B01 + , 0x000B3C + , 0x000B3F + , 0x000B41 + , 0x000B4D + , 0x000B55 + , 0x000B62 + , 0x000B82 + , 0x000BC0 + , 0x000BCD + , 0x000C00 + , 0x000C04 + , 0x000C3C + , 0x000C3E + , 0x000C46 + , 0x000C4A + , 0x000C55 + , 0x000C62 + , 0x000C81 + , 0x000CBC + , 0x000CBF + , 0x000CC6 + , 0x000CCC + , 0x000CE2 + , 0x000D00 + , 0x000D3B + , 0x000D41 + , 0x000D4D + , 0x000D62 + , 0x000D81 + , 0x000DCA + , 0x000DD2 + , 0x000DD6 + , 0x000E31 + , 0x000E34 + , 0x000E47 + , 0x000EB1 + , 0x000EB4 + , 0x000EC8 + , 0x000F18 + , 0x000F35 + , 0x000F37 + , 0x000F39 + , 0x000F71 + , 0x000F80 + , 0x000F86 + , 0x000F8D + , 0x000F99 + , 0x000FC6 + , 0x00102D + , 0x001032 + , 0x001039 + , 0x00103D + , 0x001058 + , 0x00105E + , 0x001071 + , 0x001082 + , 0x001085 + , 0x00108D + , 0x00109D + , 0x00135D + , 0x001712 + , 0x001732 + , 0x001752 + , 0x001772 + , 0x0017B4 + , 0x0017B7 + , 0x0017C6 + , 0x0017C9 + , 0x0017DD + , 0x001807 + , 0x00180B + , 0x00180F + , 0x001820 + , 0x001885 + , 0x001887 + , 0x0018A9 + , 0x0018AA + , 0x001920 + , 0x001927 + , 0x001932 + , 0x001939 + , 0x001A17 + , 0x001A1B + , 0x001A56 + , 0x001A58 + , 0x001A60 + , 0x001A62 + , 0x001A65 + , 0x001A73 + , 0x001A7F + , 0x001AB0 + , 0x001AE0 + , 0x001B00 + , 0x001B34 + , 0x001B36 + , 0x001B3C + , 0x001B42 + , 0x001B6B + , 0x001B80 + , 0x001BA2 + , 0x001BA8 + , 0x001BAB + , 0x001BE6 + , 0x001BE8 + , 0x001BED + , 0x001BEF + , 0x001C2C + , 0x001C36 + , 0x001CD0 + , 0x001CD4 + , 0x001CE2 + , 0x001CED + , 0x001CF4 + , 0x001CF8 + , 0x001DC0 + , 0x00200B + , 0x00200E + , 0x00202A + , 0x002060 + , 0x00206A + , 0x0020D0 + , 0x002CEF + , 0x002D7F + , 0x002DE0 + , 0x00302A + , 0x003099 + , 0x00A66F + , 0x00A674 + , 0x00A69E + , 0x00A6F0 + , 0x00A802 + , 0x00A806 + , 0x00A80B + , 0x00A825 + , 0x00A82C + , 0x00A840 + , 0x00A872 + , 0x00A8C4 + , 0x00A8E0 + , 0x00A8FF + , 0x00A926 + , 0x00A947 + , 0x00A980 + , 0x00A9B3 + , 0x00A9B6 + , 0x00A9BC + , 0x00A9E5 + , 0x00AA29 + , 0x00AA31 + , 0x00AA35 + , 0x00AA43 + , 0x00AA4C + , 0x00AA7C + , 0x00AAB0 + , 0x00AAB2 + , 0x00AAB7 + , 0x00AABE + , 0x00AAC1 + , 0x00AAEC + , 0x00AAF6 + , 0x00ABE5 + , 0x00ABE8 + , 0x00ABED + , 0x00FB1E + , 0x00FE00 + , 0x00FE20 + , 0x00FEFF + , 0x00FFF9 + , 0x0101FD + , 0x0102E0 + , 0x010376 + , 0x010A01 + , 0x010A05 + , 0x010A0C + , 0x010A38 + , 0x010A3F + , 0x010AC0 + , 0x010AC5 + , 0x010AC7 + , 0x010AC9 + , 0x010ACD + , 0x010ACE + , 0x010AD3 + , 0x010AD7 + , 0x010AD8 + , 0x010ADD + , 0x010ADE + , 0x010AE1 + , 0x010AE4 + , 0x010AE5 + , 0x010AEB + , 0x010AEF + , 0x010B80 + , 0x010B81 + , 0x010B82 + , 0x010B83 + , 0x010B86 + , 0x010B89 + , 0x010B8A + , 0x010B8C + , 0x010B8D + , 0x010B8E + , 0x010B90 + , 0x010B91 + , 0x010BA9 + , 0x010BAD + , 0x010D00 + , 0x010D01 + , 0x010D22 + , 0x010D23 + , 0x010D24 + , 0x010D69 + , 0x010EAB + , 0x010EC2 + , 0x010EC3 + , 0x010EC6 + , 0x010EFA + , 0x010F30 + , 0x010F33 + , 0x010F34 + , 0x010F46 + , 0x010F51 + , 0x010F54 + , 0x010F70 + , 0x010F74 + , 0x010F76 + , 0x010F82 + , 0x010FB0 + , 0x010FB2 + , 0x010FB4 + , 0x010FB8 + , 0x010FB9 + , 0x010FBB + , 0x010FBD + , 0x010FBE + , 0x010FC1 + , 0x010FC2 + , 0x010FC4 + , 0x010FC9 + , 0x010FCA + , 0x010FCB + , 0x011001 + , 0x011038 + , 0x011070 + , 0x011073 + , 0x01107F + , 0x0110B3 + , 0x0110B9 + , 0x0110C2 + , 0x011100 + , 0x011127 + , 0x01112D + , 0x011173 + , 0x011180 + , 0x0111B6 + , 0x0111C9 + , 0x0111CF + , 0x01122F + , 0x011234 + , 0x011236 + , 0x01123E + , 0x011241 + , 0x0112DF + , 0x0112E3 + , 0x011300 + , 0x01133B + , 0x011340 + , 0x011366 + , 0x011370 + , 0x0113BB + , 0x0113CE + , 0x0113D0 + , 0x0113D2 + , 0x0113E1 + , 0x011438 + , 0x011442 + , 0x011446 + , 0x01145E + , 0x0114B3 + , 0x0114BA + , 0x0114BF + , 0x0114C2 + , 0x0115B2 + , 0x0115BC + , 0x0115BF + , 0x0115DC + , 0x011633 + , 0x01163D + , 0x01163F + , 0x0116AB + , 0x0116AD + , 0x0116B0 + , 0x0116B7 + , 0x01171D + , 0x01171F + , 0x011722 + , 0x011727 + , 0x01182F + , 0x011839 + , 0x01193B + , 0x01193E + , 0x011943 + , 0x0119D4 + , 0x0119DA + , 0x0119E0 + , 0x011A01 + , 0x011A33 + , 0x011A3B + , 0x011A47 + , 0x011A51 + , 0x011A59 + , 0x011A8A + , 0x011A98 + , 0x011B60 + , 0x011B62 + , 0x011B66 + , 0x011C30 + , 0x011C38 + , 0x011C3F + , 0x011C92 + , 0x011CAA + , 0x011CB2 + , 0x011CB5 + , 0x011D31 + , 0x011D3A + , 0x011D3C + , 0x011D3F + , 0x011D47 + , 0x011D90 + , 0x011D95 + , 0x011D97 + , 0x011EF3 + , 0x011F00 + , 0x011F36 + , 0x011F40 + , 0x011F42 + , 0x011F5A + , 0x013430 + , 0x013447 + , 0x01611E + , 0x01612D + , 0x016AF0 + , 0x016B30 + , 0x016F4F + , 0x016F8F + , 0x016FE4 + , 0x01BC9D + , 0x01BCA0 + , 0x01CF00 + , 0x01CF30 + , 0x01D167 + , 0x01D173 + , 0x01D185 + , 0x01D1AA + , 0x01D242 + , 0x01DA00 + , 0x01DA3B + , 0x01DA75 + , 0x01DA84 + , 0x01DA9B + , 0x01DAA1 + , 0x01E000 + , 0x01E008 + , 0x01E01B + , 0x01E023 + , 0x01E026 + , 0x01E08F + , 0x01E130 + , 0x01E2AE + , 0x01E2EC + , 0x01E4EC + , 0x01E5EE + , 0x01E6E3 + , 0x01E6E6 + , 0x01E6EE + , 0x01E6F5 + , 0x01E8D0 + , 0x01E900 + , 0x01E944 + , 0x0E0001 + , 0x0E0020 + , 0x0E0100 + ] + +-- | Inclusive range ends, parallel to 'jtRangeStarts'. +jtRangeEnds :: ByteArray +jtRangeEnds = PBA.byteArrayFromList + [ (0x0000AD :: Word32) + , 0x00036F + , 0x000489 + , 0x0005BD + , 0x0005BF + , 0x0005C2 + , 0x0005C5 + , 0x0005C7 + , 0x00061A + , 0x00061C + , 0x000620 + , 0x000625 + , 0x000626 + , 0x000627 + , 0x000628 + , 0x000629 + , 0x00062E + , 0x000632 + , 0x00063F + , 0x000647 + , 0x000648 + , 0x00064A + , 0x00065F + , 0x00066F + , 0x000670 + , 0x000673 + , 0x000677 + , 0x000687 + , 0x000699 + , 0x0006BF + , 0x0006C0 + , 0x0006C2 + , 0x0006CB + , 0x0006CC + , 0x0006CD + , 0x0006CE + , 0x0006CF + , 0x0006D1 + , 0x0006D3 + , 0x0006D5 + , 0x0006DC + , 0x0006E4 + , 0x0006E8 + , 0x0006ED + , 0x0006EF + , 0x0006FC + , 0x0006FF + , 0x00070F + , 0x000710 + , 0x000711 + , 0x000714 + , 0x000719 + , 0x00071D + , 0x00071E + , 0x000727 + , 0x000728 + , 0x000729 + , 0x00072A + , 0x00072B + , 0x00072C + , 0x00072E + , 0x00072F + , 0x00074A + , 0x00074D + , 0x000758 + , 0x00075B + , 0x00076A + , 0x00076C + , 0x000770 + , 0x000771 + , 0x000772 + , 0x000774 + , 0x000777 + , 0x000779 + , 0x00077F + , 0x0007B0 + , 0x0007EA + , 0x0007F3 + , 0x0007FD + , 0x000819 + , 0x000823 + , 0x000827 + , 0x00082D + , 0x000840 + , 0x000845 + , 0x000847 + , 0x000848 + , 0x000849 + , 0x000853 + , 0x000854 + , 0x000855 + , 0x000858 + , 0x00085B + , 0x000860 + , 0x000865 + , 0x000867 + , 0x000868 + , 0x00086A + , 0x000882 + , 0x000886 + , 0x00088D + , 0x00088E + , 0x00088F + , 0x00089F + , 0x0008A9 + , 0x0008AC + , 0x0008AE + , 0x0008B0 + , 0x0008B2 + , 0x0008B8 + , 0x0008B9 + , 0x0008C8 + , 0x0008E1 + , 0x000902 + , 0x00093A + , 0x00093C + , 0x000948 + , 0x00094D + , 0x000957 + , 0x000963 + , 0x000981 + , 0x0009BC + , 0x0009C4 + , 0x0009CD + , 0x0009E3 + , 0x0009FE + , 0x000A02 + , 0x000A3C + , 0x000A42 + , 0x000A48 + , 0x000A4D + , 0x000A51 + , 0x000A71 + , 0x000A75 + , 0x000A82 + , 0x000ABC + , 0x000AC5 + , 0x000AC8 + , 0x000ACD + , 0x000AE3 + , 0x000AFF + , 0x000B01 + , 0x000B3C + , 0x000B3F + , 0x000B44 + , 0x000B4D + , 0x000B56 + , 0x000B63 + , 0x000B82 + , 0x000BC0 + , 0x000BCD + , 0x000C00 + , 0x000C04 + , 0x000C3C + , 0x000C40 + , 0x000C48 + , 0x000C4D + , 0x000C56 + , 0x000C63 + , 0x000C81 + , 0x000CBC + , 0x000CBF + , 0x000CC6 + , 0x000CCD + , 0x000CE3 + , 0x000D01 + , 0x000D3C + , 0x000D44 + , 0x000D4D + , 0x000D63 + , 0x000D81 + , 0x000DCA + , 0x000DD4 + , 0x000DD6 + , 0x000E31 + , 0x000E3A + , 0x000E4E + , 0x000EB1 + , 0x000EBC + , 0x000ECE + , 0x000F19 + , 0x000F35 + , 0x000F37 + , 0x000F39 + , 0x000F7E + , 0x000F84 + , 0x000F87 + , 0x000F97 + , 0x000FBC + , 0x000FC6 + , 0x001030 + , 0x001037 + , 0x00103A + , 0x00103E + , 0x001059 + , 0x001060 + , 0x001074 + , 0x001082 + , 0x001086 + , 0x00108D + , 0x00109D + , 0x00135F + , 0x001714 + , 0x001733 + , 0x001753 + , 0x001773 + , 0x0017B5 + , 0x0017BD + , 0x0017C6 + , 0x0017D3 + , 0x0017DD + , 0x001807 + , 0x00180D + , 0x00180F + , 0x001878 + , 0x001886 + , 0x0018A8 + , 0x0018A9 + , 0x0018AA + , 0x001922 + , 0x001928 + , 0x001932 + , 0x00193B + , 0x001A18 + , 0x001A1B + , 0x001A56 + , 0x001A5E + , 0x001A60 + , 0x001A62 + , 0x001A6C + , 0x001A7C + , 0x001A7F + , 0x001ADD + , 0x001AEB + , 0x001B03 + , 0x001B34 + , 0x001B3A + , 0x001B3C + , 0x001B42 + , 0x001B73 + , 0x001B81 + , 0x001BA5 + , 0x001BA9 + , 0x001BAD + , 0x001BE6 + , 0x001BE9 + , 0x001BED + , 0x001BF1 + , 0x001C33 + , 0x001C37 + , 0x001CD2 + , 0x001CE0 + , 0x001CE8 + , 0x001CED + , 0x001CF4 + , 0x001CF9 + , 0x001DFF + , 0x00200B + , 0x00200F + , 0x00202E + , 0x002064 + , 0x00206F + , 0x0020F0 + , 0x002CF1 + , 0x002D7F + , 0x002DFF + , 0x00302D + , 0x00309A + , 0x00A672 + , 0x00A67D + , 0x00A69F + , 0x00A6F1 + , 0x00A802 + , 0x00A806 + , 0x00A80B + , 0x00A826 + , 0x00A82C + , 0x00A871 + , 0x00A872 + , 0x00A8C5 + , 0x00A8F1 + , 0x00A8FF + , 0x00A92D + , 0x00A951 + , 0x00A982 + , 0x00A9B3 + , 0x00A9B9 + , 0x00A9BD + , 0x00A9E5 + , 0x00AA2E + , 0x00AA32 + , 0x00AA36 + , 0x00AA43 + , 0x00AA4C + , 0x00AA7C + , 0x00AAB0 + , 0x00AAB4 + , 0x00AAB8 + , 0x00AABF + , 0x00AAC1 + , 0x00AAED + , 0x00AAF6 + , 0x00ABE5 + , 0x00ABE8 + , 0x00ABED + , 0x00FB1E + , 0x00FE0F + , 0x00FE2F + , 0x00FEFF + , 0x00FFFB + , 0x0101FD + , 0x0102E0 + , 0x01037A + , 0x010A03 + , 0x010A06 + , 0x010A0F + , 0x010A3A + , 0x010A3F + , 0x010AC4 + , 0x010AC5 + , 0x010AC7 + , 0x010ACA + , 0x010ACD + , 0x010AD2 + , 0x010AD6 + , 0x010AD7 + , 0x010ADC + , 0x010ADD + , 0x010AE0 + , 0x010AE1 + , 0x010AE4 + , 0x010AE6 + , 0x010AEE + , 0x010AEF + , 0x010B80 + , 0x010B81 + , 0x010B82 + , 0x010B85 + , 0x010B88 + , 0x010B89 + , 0x010B8B + , 0x010B8C + , 0x010B8D + , 0x010B8F + , 0x010B90 + , 0x010B91 + , 0x010BAC + , 0x010BAE + , 0x010D00 + , 0x010D21 + , 0x010D22 + , 0x010D23 + , 0x010D27 + , 0x010D6D + , 0x010EAC + , 0x010EC2 + , 0x010EC4 + , 0x010EC7 + , 0x010EFF + , 0x010F32 + , 0x010F33 + , 0x010F44 + , 0x010F50 + , 0x010F53 + , 0x010F54 + , 0x010F73 + , 0x010F75 + , 0x010F81 + , 0x010F85 + , 0x010FB0 + , 0x010FB3 + , 0x010FB6 + , 0x010FB8 + , 0x010FBA + , 0x010FBC + , 0x010FBD + , 0x010FBF + , 0x010FC1 + , 0x010FC3 + , 0x010FC4 + , 0x010FC9 + , 0x010FCA + , 0x010FCB + , 0x011001 + , 0x011046 + , 0x011070 + , 0x011074 + , 0x011081 + , 0x0110B6 + , 0x0110BA + , 0x0110C2 + , 0x011102 + , 0x01112B + , 0x011134 + , 0x011173 + , 0x011181 + , 0x0111BE + , 0x0111CC + , 0x0111CF + , 0x011231 + , 0x011234 + , 0x011237 + , 0x01123E + , 0x011241 + , 0x0112DF + , 0x0112EA + , 0x011301 + , 0x01133C + , 0x011340 + , 0x01136C + , 0x011374 + , 0x0113C0 + , 0x0113CE + , 0x0113D0 + , 0x0113D2 + , 0x0113E2 + , 0x01143F + , 0x011444 + , 0x011446 + , 0x01145E + , 0x0114B8 + , 0x0114BA + , 0x0114C0 + , 0x0114C3 + , 0x0115B5 + , 0x0115BD + , 0x0115C0 + , 0x0115DD + , 0x01163A + , 0x01163D + , 0x011640 + , 0x0116AB + , 0x0116AD + , 0x0116B5 + , 0x0116B7 + , 0x01171D + , 0x01171F + , 0x011725 + , 0x01172B + , 0x011837 + , 0x01183A + , 0x01193C + , 0x01193E + , 0x011943 + , 0x0119D7 + , 0x0119DB + , 0x0119E0 + , 0x011A0A + , 0x011A38 + , 0x011A3E + , 0x011A47 + , 0x011A56 + , 0x011A5B + , 0x011A96 + , 0x011A99 + , 0x011B60 + , 0x011B64 + , 0x011B66 + , 0x011C36 + , 0x011C3D + , 0x011C3F + , 0x011CA7 + , 0x011CB0 + , 0x011CB3 + , 0x011CB6 + , 0x011D36 + , 0x011D3A + , 0x011D3D + , 0x011D45 + , 0x011D47 + , 0x011D91 + , 0x011D95 + , 0x011D97 + , 0x011EF4 + , 0x011F01 + , 0x011F3A + , 0x011F40 + , 0x011F42 + , 0x011F5A + , 0x013440 + , 0x013455 + , 0x016129 + , 0x01612F + , 0x016AF4 + , 0x016B36 + , 0x016F4F + , 0x016F92 + , 0x016FE4 + , 0x01BC9E + , 0x01BCA3 + , 0x01CF2D + , 0x01CF46 + , 0x01D169 + , 0x01D182 + , 0x01D18B + , 0x01D1AD + , 0x01D244 + , 0x01DA36 + , 0x01DA6C + , 0x01DA75 + , 0x01DA84 + , 0x01DA9F + , 0x01DAAF + , 0x01E006 + , 0x01E018 + , 0x01E021 + , 0x01E024 + , 0x01E02A + , 0x01E08F + , 0x01E136 + , 0x01E2AE + , 0x01E2EF + , 0x01E4EF + , 0x01E5EF + , 0x01E6E3 + , 0x01E6E6 + , 0x01E6EF + , 0x01E6F5 + , 0x01E8D6 + , 0x01E943 + , 0x01E94B + , 0x0E0001 + , 0x0E007F + , 0x0E01EF + ] + +-- | Joining_Type tag for each range, parallel to +-- 'jtRangeStarts' and 'jtRangeEnds'. +jtRangeTags :: ByteArray +jtRangeTags = PBA.byteArrayFromList + [ (4 :: Word8) + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 3 + , 2 + , 3 + , 4 + , 3 + , 4 + , 2 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 2 + , 4 + , 4 + , 4 + , 4 + , 2 + , 3 + , 3 + , 4 + , 2 + , 4 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 4 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 4 + , 3 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 4 + , 3 + , 3 + , 2 + , 3 + , 2 + , 2 + , 3 + , 3 + , 2 + , 3 + , 4 + , 3 + , 2 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 3 + , 4 + , 4 + , 3 + , 4 + , 3 + , 4 + , 3 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 3 + , 1 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 3 + , 2 + , 2 + , 2 + , 1 + , 2 + , 3 + , 1 + , 3 + , 2 + , 3 + , 2 + , 2 + , 4 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 2 + , 3 + , 1 + , 3 + , 2 + , 3 + , 4 + , 4 + , 4 + , 2 + , 3 + , 3 + , 4 + , 3 + , 2 + , 3 + , 4 + , 3 + , 2 + , 3 + , 2 + , 3 + , 4 + , 3 + , 3 + , 2 + , 3 + , 2 + , 3 + , 2 + , 3 + , 3 + , 2 + , 3 + , 2 + , 3 + , 1 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 4 + , 3 + , 4 + , 4 + , 4 + , 4 + ] + +---------------------------------------------------------------------- +-- Virama (Canonical_Combining_Class = 9) +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'viramaRanges'. +viramaRangeCount :: Int +viramaRangeCount = 58 + +-- | Codepoints with Canonical_Combining_Class = 9 +-- (Virama), alternating start\/end. +viramaRanges :: ByteArray +viramaRanges = PBA.byteArrayFromList + [ (0x00094D :: Word32) + , 0x00094D + , 0x0009CD + , 0x0009CD + , 0x000A4D + , 0x000A4D + , 0x000ACD + , 0x000ACD + , 0x000B4D + , 0x000B4D + , 0x000BCD + , 0x000BCD + , 0x000C4D + , 0x000C4D + , 0x000CCD + , 0x000CCD + , 0x000D3B + , 0x000D3C + , 0x000D4D + , 0x000D4D + , 0x000DCA + , 0x000DCA + , 0x000E3A + , 0x000E3A + , 0x000EBA + , 0x000EBA + , 0x000F84 + , 0x000F84 + , 0x001039 + , 0x00103A + , 0x001714 + , 0x001715 + , 0x001734 + , 0x001734 + , 0x0017D2 + , 0x0017D2 + , 0x001A60 + , 0x001A60 + , 0x001B44 + , 0x001B44 + , 0x001BAA + , 0x001BAB + , 0x001BF2 + , 0x001BF3 + , 0x002D7F + , 0x002D7F + , 0x00A806 + , 0x00A806 + , 0x00A82C + , 0x00A82C + , 0x00A8C4 + , 0x00A8C4 + , 0x00A953 + , 0x00A953 + , 0x00A9C0 + , 0x00A9C0 + , 0x00AAF6 + , 0x00AAF6 + , 0x00ABED + , 0x00ABED + , 0x010A3F + , 0x010A3F + , 0x011046 + , 0x011046 + , 0x011070 + , 0x011070 + , 0x01107F + , 0x01107F + , 0x0110B9 + , 0x0110B9 + , 0x011133 + , 0x011134 + , 0x0111C0 + , 0x0111C0 + , 0x011235 + , 0x011235 + , 0x0112EA + , 0x0112EA + , 0x01134D + , 0x01134D + , 0x0113CE + , 0x0113D0 + , 0x011442 + , 0x011442 + , 0x0114C2 + , 0x0114C2 + , 0x0115BF + , 0x0115BF + , 0x01163F + , 0x01163F + , 0x0116B6 + , 0x0116B6 + , 0x01172B + , 0x01172B + , 0x011839 + , 0x011839 + , 0x01193D + , 0x01193E + , 0x0119E0 + , 0x0119E0 + , 0x011A34 + , 0x011A34 + , 0x011A47 + , 0x011A47 + , 0x011A99 + , 0x011A99 + , 0x011C3F + , 0x011C3F + , 0x011D44 + , 0x011D45 + , 0x011D97 + , 0x011D97 + , 0x011F41 + , 0x011F42 + , 0x01612F + , 0x01612F + ] diff --git a/internal/Text/IDNA2008/Internal/LabelForm.hs b/internal/Text/IDNA2008/Internal/LabelForm.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/LabelForm.hs @@ -0,0 +1,113 @@ +-- | +-- Module : Text.IDNA2008.Internal.LabelForm +-- Description : Per-label classification singleton. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- The 'LabelForm' singleton type names the nine possible +-- classifications the parser can assign to a single label, plus +-- a tenth sentinel 'NoLabel' returned by +-- 'Text.IDNA2008.Internal.LabelInfo.getLabelForm' for indices +-- outside the parsed domain's label count. The parser never +-- produces 'NoLabel' for a real label. +-- +-- The companion type 'Text.IDNA2008.Internal.LabelFormSet' +-- represents unordered /sets/ of label forms. +{-# LANGUAGE PatternSynonyms #-} + +module Text.IDNA2008.Internal.LabelForm + ( LabelForm(.., LDH, RLDH, FAKEA, ALABEL, ULABEL, ATTRLEAF, OCTET + , WILDLABEL, LAXULABEL, NoLabel) + ) where + +import Data.Word (Word8) + +-- | A single per-label classification. Use the pattern +-- singletons to construct or pattern-match. +newtype LabelForm = LabelForm_ Word8 + deriving (Eq, Ord) + +-- | Letter-digit-hyphen, the conventional hostname alphabet. +-- Lowercase ASCII letters, digits, and the hyphen, with no +-- leading or trailing hyphen and no double-hyphen at positions +-- 3 and 4 (which would make it a Reserved-LDH or A-label). +pattern LDH :: LabelForm +pattern LDH = LabelForm_ 0 + +-- | Reserved LDH: an LDH-shaped label with @--@ at positions 3 +-- and 4 whose first two characters are not (case-folded) +-- @\"xn\"@. In practice these are pre-IDN registrations like +-- @\"l---l\"@, @\"cd--storage-shelves\"@. +pattern RLDH :: LabelForm +pattern RLDH = LabelForm_ 1 + +-- | An ACE-prefixed LDH label that does /not/ strictly +-- round-trip: its Punycode body decodes to something invalid +-- as an IDN label, or the re-encoding doesn't match the input. +pattern FAKEA :: LabelForm +pattern FAKEA = LabelForm_ 2 + +-- | An ACE-prefixed LDH label that strictly round-trips: +-- the Punycode body decodes cleanly, the decoded codepoints are +-- valid in an IDN, and re-encoding produces the same input bytes. +pattern ALABEL :: LabelForm +pattern ALABEL = LabelForm_ 3 + +-- | A label whose source contained at least one non-ASCII +-- character. The library's parser accepts non-ASCII input, +-- validates each codepoint, and encodes the label to its +-- ACE-prefixed form for the wire. +pattern ULABEL :: LabelForm +pattern ULABEL = LabelForm_ 4 + +-- | An LDH-shaped label whose first character is @\'_\'@. Used +-- by service-discovery records and by attribute-leaf naming +-- conventions: @\"_25._tcp\"@, @\"_dmarc\"@, +-- @\"_acme-challenge\"@, etc. +pattern ATTRLEAF :: LabelForm +pattern ATTRLEAF = LabelForm_ 5 + +-- | A label whose bytes are not all in the LDH alphabet. +-- Includes labels admitted via @\\DDD@ or @\\C@ escapes +-- regardless of UTF-8 validity, labels with non-LDH ASCII +-- (e.g. @\'$\'@, @\'+\'@), and labels containing non-ASCII +-- codepoints that aren't valid for IDN classification. +pattern OCTET :: LabelForm +pattern OCTET = LabelForm_ 6 + +-- | The single-byte wildcard label @\'*\'@. Distinct from +-- 'OCTET' so callers that want to admit @\"*.example\"@ +-- queries can do so without admitting arbitrary non-LDH bytes. +pattern WILDLABEL :: LabelForm +pattern WILDLABEL = LabelForm_ 7 + +-- | Like 'ULABEL' but the non-ASCII codepoints fail strict +-- U-label validation. +pattern LAXULABEL :: LabelForm +pattern LAXULABEL = LabelForm_ 8 + +-- | Sentinel value returned by +-- 'Text.IDNA2008.Internal.LabelInfo.getLabelForm' when the +-- index is outside the parsed domain's label count. The parser +-- never produces 'NoLabel' for an actual label, and it is not +-- a member of any 'Text.IDNA2008.Internal.LabelFormSet'. +pattern NoLabel :: LabelForm +pattern NoLabel = LabelForm_ 9 + +{-# COMPLETE LDH, RLDH, FAKEA, ALABEL, ULABEL, ATTRLEAF, OCTET, + WILDLABEL, LAXULABEL, NoLabel #-} + +instance Show LabelForm where + show LDH = "LDH" + show RLDH = "RLDH" + show FAKEA = "FAKEA" + show ALABEL = "ALABEL" + show ULABEL = "ULABEL" + show ATTRLEAF = "ATTRLEAF" + show OCTET = "OCTET" + show WILDLABEL = "WILDLABEL" + show LAXULABEL = "LAXULABEL" + show NoLabel = "NoLabel" diff --git a/internal/Text/IDNA2008/Internal/LabelFormSet.hs b/internal/Text/IDNA2008/Internal/LabelFormSet.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/LabelFormSet.hs @@ -0,0 +1,232 @@ +-- | +-- Module : Text.IDNA2008.Internal.LabelFormSet +-- Description : Sets of label classifications. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Sets of 'LabelForm' values used as the parser's admission set. +-- See 'LabelFormSet' for construction idioms and operations. +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} + +module Text.IDNA2008.Internal.LabelFormSet + ( -- * Set type + LabelFormSet + , unLabelFormSet + + -- * Set construction from singletons + , (<+>) + , (<->) + , labelFormToSet + + -- * Set-level operations + , isLabelFormSubset + , memberLabelFormSet + , withoutLabelFormSet + + -- * Pre-built sets + , allLabelForms + , idnLabelForms + , hostnameLabelForms + + -- * Command-line option syntax + , labelFormSetTokens + , labelFormSetPresets + , parseLabelFormSet + , parseLabelFormSetStr + ) where + +import Control.Monad ((>=>)) +import Data.Bits ((.|.), (.&.), complement, shiftL, testBit, unsafeShiftR) +import Data.ByteString (ByteString) +import Data.List (intercalate) +import Data.Word (Word16) + +import Text.IDNA2008.Internal.LabelForm ( LabelForm(..) ) +import Text.IDNA2008.Internal.Tokens + ( TokenEntry(..), Preset, asciiByteString, parseTokens ) + +infixl 6 <+>, <-> + +-- | A set of 'LabelForm' values, used as the @allowed@ argument +-- to the parser. Build sets from 'mempty' or one of the +-- pre-built named values ('allLabelForms', 'idnLabelForms', +-- 'hostnameLabelForms'), then add or remove individual forms +-- with @('<+>')@ and @('<->')@, or combine sets with @('<>')@. +-- 'labelFormToSet' promotes a single 'LabelForm' to a one-element +-- set; 'withoutLabelFormSet' implements set difference. +-- +-- Examples: +-- +-- > mempty <+> LDH <+> ULABEL <+> ALABEL +-- > foldMap' labelFormToSet [LDH, ULABEL, ALABEL] +-- > hostnameLabelForms <-> FAKEA +-- > idnLabelForms <+> WILDLABEL +newtype LabelFormSet = LabelFormSet Word16 + deriving Eq + +-- | Extract the underlying bitmask. Internal use only. +unLabelFormSet :: LabelFormSet -> Word16 +unLabelFormSet (LabelFormSet w) = w +{-# INLINE unLabelFormSet #-} + +instance Semigroup LabelFormSet where + LabelFormSet a <> LabelFormSet b = LabelFormSet (a .|. b) + {-# INLINE (<>) #-} + +instance Monoid LabelFormSet where + mempty = LabelFormSet 0 + {-# INLINE mempty #-} + +instance Show LabelFormSet where + show s = case formNames s of + [] -> "mempty" + ns -> intercalate "," ns + +-- | Decompose a set into its singleton form names, in canonical +-- bit order. Internal-only; used by 'Show'. +formNames :: LabelFormSet -> [String] +formNames (LabelFormSet w) = go names w + where + names = [ "LDH", "RLDH", "FAKEA", "ALABEL" + , "ULABEL", "ATTRLEAF", "OCTET", "WILDLABEL", "LAXULABEL" ] + go _ 0 = [] + go [] _ = [] + go (s:ss) n + | odd n = s : go ss (n `unsafeShiftR` 1) + | otherwise = go ss (n `unsafeShiftR` 1) + +---------------------------------------------------------------------- +-- Set construction from singletons +---------------------------------------------------------------------- + +-- | Add a 'LabelForm' to a set. +(<+>) :: LabelFormSet -> LabelForm -> LabelFormSet +set <+> form = set <> labelFormToSet form +{-# INLINE (<+>) #-} + +-- | Remove a 'LabelForm' from a set. +(<->) :: LabelFormSet -> LabelForm -> LabelFormSet +set <-> form = set `withoutLabelFormSet` labelFormToSet form +{-# INLINE (<->) #-} + +-- | One-element set containing the given 'LabelForm'. +-- 'NoLabel' maps to 'mempty'. +labelFormToSet :: LabelForm -> LabelFormSet +labelFormToSet (LabelForm_ form) + | form < 9 = LabelFormSet (1 `shiftL` fromIntegral form) + | otherwise = mempty +{-# INLINE labelFormToSet #-} + +---------------------------------------------------------------------- +-- Set-level operations +---------------------------------------------------------------------- + +-- | Subset test: @a \`isLabelFormSubset\` b@ is 'True' iff +-- every form in @a@ is also in @b@. 'mempty' is a subset of +-- every set. +isLabelFormSubset :: LabelFormSet -> LabelFormSet -> Bool +LabelFormSet a `isLabelFormSubset` LabelFormSet b = a == a .&. b +{-# INLINE isLabelFormSubset #-} + +-- | Membership test: does the given 'LabelForm' belong to the +-- set? Returns 'False' for 'NoLabel' against any set. +memberLabelFormSet :: LabelForm -> LabelFormSet -> Bool +memberLabelFormSet (LabelForm_ form) (LabelFormSet b) + | form < 9 = testBit b (fromIntegral form) + | otherwise = False +{-# INLINE memberLabelFormSet #-} + +-- | Set difference: @a \`withoutLabelFormSet\` b@ removes every +-- element of @b@ from @a@. Companion to '<>' (union). +withoutLabelFormSet :: LabelFormSet -> LabelFormSet -> LabelFormSet +LabelFormSet a `withoutLabelFormSet` LabelFormSet b = + LabelFormSet (a .&. complement b) +{-# INLINE withoutLabelFormSet #-} + +---------------------------------------------------------------------- +-- Pre-built sets +---------------------------------------------------------------------- + +-- | Every label classification that arises from a clean-path +-- validation outcome: 'LDH', 'RLDH', 'FAKEA', 'ALABEL', +-- 'ULABEL', 'ATTRLEAF', 'OCTET', 'WILDLABEL'. +-- +-- /Excludes/ 'LAXULABEL': that classification is the explicit +-- opt-in for admitting U-labels that fail strict IDN +-- validation, and on the unparse side admitting it suppresses +-- the cross-label Bidi check. To include it, write +-- @allLabelForms '<+>' 'LAXULABEL'@ (or @\"all,+laxulabel\"@ +-- in command-line option syntax). +allLabelForms :: LabelFormSet +allLabelForms = LabelFormSet 0x00FF + +-- | The strict subset for internationalised domain names: +-- ordinary hostname-style labels ('LDH'), valid A-labels, and +-- Unicode labels. +idnLabelForms :: LabelFormSet +idnLabelForms = mempty <+> LDH <+> ALABEL <+> ULABEL + +-- | Permissive hostname-like labels: 'idnLabelForms' plus 'RLDH' and +-- 'FAKEA'. Useful when dealing with names found in the wild where +-- unusual but syntactically valid prior to IDNA2008 LDH labels may appear. +hostnameLabelForms :: LabelFormSet +hostnameLabelForms = idnLabelForms <+> RLDH <+> FAKEA + +---------------------------------------------------------------------- +-- Command-line option syntax +---------------------------------------------------------------------- + +-- | Single-bit token table for the command-line option syntax. +labelFormSetTokens :: [TokenEntry LabelFormSet] +labelFormSetTokens = + [ TokenEntry (labelFormToSet LDH) "ldh" [] + , TokenEntry (labelFormToSet RLDH) "rldh" [] + , TokenEntry (labelFormToSet FAKEA) "fakea" [] + , TokenEntry (labelFormToSet ALABEL) "alabel" [] + , TokenEntry (labelFormToSet ULABEL) "ulabel" [] + , TokenEntry (labelFormToSet ATTRLEAF) "attrleaf" [] + , TokenEntry (labelFormToSet OCTET) "octet" [] + , TokenEntry (labelFormToSet WILDLABEL) "wildlabel" [] + , TokenEntry (labelFormToSet LAXULABEL) "laxulabel" [] + ] + +-- | Preset names that expand to multi-bit form sets. +labelFormSetPresets :: [Preset LabelFormSet] +labelFormSetPresets = + [ ("idn", idnLabelForms) + , ("strict", idnLabelForms) -- alias of @idn@ + , ("host", hostnameLabelForms) + , ("all", allLabelForms) + ] + +-- | Parse a comma-separated CLI value as a 'LabelFormSet'. +-- The first argument is the application-specific default. +-- +-- Each comma-separated token may optionally be prefixed with +-- @\'+\'@ (additive, the default) or @\'-\'@ (subtractive). +-- If the first token has a sign prefix, the running result is +-- seeded with the supplied default; otherwise the running +-- result starts empty and the tokens replace the default +-- cleanly. +-- +-- Tokens are matched case-insensitively, with unambiguous prefix +-- matching (3-character minimum) as a fallback when there's no +-- exact match. The token @default@ or an unambiguous prefix +-- matches the supplied default value. +parseLabelFormSet :: LabelFormSet -- ^ Application-specific default + -> ByteString -- ^ Tokens to parse + -> Either String LabelFormSet +parseLabelFormSet !defS = + parseTokens labelFormSetTokens + (("default", defS) : labelFormSetPresets) + defS + withoutLabelFormSet + +-- | 'String'-based wrapper for 'parseLabelFormSet': validates that +-- the input is pure ASCII. +parseLabelFormSetStr :: LabelFormSet -> String -> Either String LabelFormSet +parseLabelFormSetStr !defS = asciiByteString >=> parseLabelFormSet defS diff --git a/internal/Text/IDNA2008/Internal/LabelInfo.hs b/internal/Text/IDNA2008/Internal/LabelInfo.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/LabelInfo.hs @@ -0,0 +1,152 @@ +-- | +-- Module : Text.IDNA2008.Internal.LabelInfo +-- Description : Per-label classification result. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- A 'LabelInfo' is the parser's per-label classification result, +-- paired with the wire-form @Domain@ in the @parseDomain*@ return +-- tuple. Opaque; inspect via the accessors: +-- +-- * 'getLabelForms' returns the list of forms in label order. +-- * 'getLabelFormCount' returns the number of labels. +-- * 'getLabelForm' indexes the array; out-of-range indices +-- (including any index for the zero-label root domain's +-- 'LabelInfo') return 'NoLabel'. +-- * 'allLabelFormsIn' asks whether every label belongs to a +-- given 'LabelFormSet'. +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE PatternSynonyms #-} + +module Text.IDNA2008.Internal.LabelInfo + ( -- * Opaque per-label result + LabelInfo + , mkLabelInfo + + -- * Accessors + , getLabelForms + , getLabelFormCount + , getLabelForm + + -- * Conformance to a permitted-set + , allLabelFormsIn + ) where + +import qualified Data.ByteString.Short as SBS +import Control.Monad.ST (ST, runST) +import Data.Array.Byte (ByteArray(..)) +import Data.Bits ((.|.), (.&.), unsafeShiftL, unsafeShiftR) +import Data.ByteString.Short (ShortByteString) +import Data.ByteString.Short (ShortByteString(SBS)) +import Data.Primitive.ByteArray + ( MutableByteArray + , newByteArray + , readByteArray + , setByteArray + , unsafeFreezeByteArray + , writeByteArray + ) +import Data.Word (Word8) + +import Text.IDNA2008.Internal.LabelForm ( LabelForm(..) ) +import qualified Text.IDNA2008.Internal.LabelFormSet as LFS +import Text.IDNA2008.Internal.LabelFormSet (LabelFormSet) + +-- | The parser's per-label classification result. Opaque; +-- inspect via the accessors. +data LabelInfo = LabelInfo + {-# UNPACK #-} !Word8 + {-# UNPACK #-} !ShortByteString + +instance Show LabelInfo where + show info = show (getLabelForms info) + +-- | Number of packed bytes needed to hold @n@ four-bit values. +packedBytes :: Int -> Int +packedBytes !n = (n + 1) `unsafeShiftR` 1 +{-# INLINE packedBytes #-} + +-- | Build a 'LabelInfo' from a list of 'LabelForm' values in +-- label order. Internal-only; not re-exported by the public +-- API. A 'LabelInfo' is normally produced by the parser, not +-- constructed by hand; the caller is expected to honour the DNS +-- 127-label cap, since the stored count is a 'Word8'. +mkLabelInfo :: [LabelForm] -> LabelInfo +mkLabelInfo forms = + let !n = length forms + in LabelInfo (fromIntegral n) (packForms n forms) + +-- | Pack a list of 'LabelForm' tags into a 'ShortByteString', +-- one nibble per label. +packForms :: Int -> [LabelForm] -> ShortByteString +packForms !n forms = runST do + let !nb = packedBytes n + mba <- newByteArray nb + setByteArray mba 0 nb (0 :: Word8) + let loop !_ [] = pure () + loop !i (f:fs) = do + writeForm mba i f + loop (i + 1) fs + loop 0 forms + (ByteArray frozen) <- unsafeFreezeByteArray mba + pure (SBS frozen) + +-- | Write a single 'LabelForm' at logical position @i@ into the +-- mutable buffer. Assumes the buffer is zero-initialised at +-- the relevant nibble; the write OR-merges the four bits in. +-- Even indices land in the low nibble of byte @i/2@; odd +-- indices in the high nibble. +writeForm :: forall s. MutableByteArray s -> Int -> LabelForm -> ST s () +writeForm mba i (LabelForm_ form) = do + let !w = form .&. 0x0F + !byteIx = i `unsafeShiftR` 1 + !nibOff = (i .&. 1) `unsafeShiftL` 2 -- 0 (low) or 4 (high) + cur <- readByteArray @Word8 mba byteIx + writeByteArray mba byteIx (cur .|. (w `unsafeShiftL` nibOff)) + +-- | Read the four bits at logical position @i@ from the packed +-- buffer. Assumes @0 <= i < count@. +readForm :: ShortByteString -> Int -> Word8 +readForm sbs i = + let !byteIx = i `unsafeShiftR` 1 + !nibOff = (i .&. 1) `unsafeShiftL` 2 -- 0 (low) or 4 (high) + !byte = SBS.index sbs byteIx + in (byte `unsafeShiftR` nibOff) .&. 0x0F +{-# INLINE readForm #-} + +-- | The list of 'LabelForm' values, in input label order. +-- Returns an empty list for the root domain's 'LabelInfo'. +-- Lazy: callers may consume a prefix without forcing the rest. +getLabelForms :: LabelInfo -> [LabelForm] +getLabelForms (LabelInfo n sbs) = + [ LabelForm_ (readForm sbs i) | i <- [0 .. fromIntegral n - 1] ] + +-- | The number of labels. Zero for the root domain; capped at +-- 127 in practice (the DNS limit on labels per name). +getLabelFormCount :: LabelInfo -> Int +getLabelFormCount (LabelInfo n _) = fromIntegral n +{-# INLINE getLabelFormCount #-} + +-- | Indexed access. Returns 'NoLabel' for any index outside +-- @[0, 'getLabelFormCount' - 1]@, including all indices when +-- the count is zero (the root domain). +getLabelForm :: LabelInfo -> Int -> LabelForm +getLabelForm (LabelInfo n sbs) i + | i < 0 || i >= fromIntegral n = NoLabel + | otherwise = LabelForm_ (readForm sbs i) +{-# INLINE getLabelForm #-} + +-- | Does every label's form match the given 'LabelFormSet'? +-- +-- Vacuously returns 'True' for the root domain's 'LabelInfo'. +-- Applications that want to reject the root domain can check the +-- value of 'getLabelFormCount', or check whether the label list +-- returned by 'Text.IDNA2008.toLabels' is 'null'. +allLabelFormsIn :: LabelInfo -- ^ The per-label forms + -> LabelFormSet -- ^ The target label form set + -> Bool +allLabelFormsIn info allowed = + all (`LFS.memberLabelFormSet` allowed) (getLabelForms info) diff --git a/internal/Text/IDNA2008/Internal/NFC.hs b/internal/Text/IDNA2008/Internal/NFC.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/NFC.hs @@ -0,0 +1,335 @@ +-- | +-- Module : Text.IDNA2008.Internal.NFC +-- Description : Unicode NFC validation for IDNA U-labels +-- (RFC 5891 section 5.3). +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Two layers: +-- +-- * 'nfcQuickCheck' / 'NfcQc' -- the Unicode @NFC_Quick_Check@ +-- property as a trichotomy. Cheap; the typical label has +-- every codepoint @NfcYes@ and the validator can short-circuit. +-- +-- * 'isNFC' -- the precise validator. Runs the QC fast path +-- first; on any non-@Yes@ codepoint it falls through to a +-- full normalize-and-compare using +-- "Text.IDNA2008.Internal.NFC.Tables" (canonical +-- decomposition + reorder + primary composition, including +-- the Hangul algorithmic short-circuit). +{-# LANGUAGE BangPatterns #-} + +module Text.IDNA2008.Internal.NFC + ( -- * Quick_Check trichotomy + NfcQc(..) + , nfcQuickCheck + + -- * Full NFC validation + , isNFC + + -- * In-place NFC normalisation + , normalizeNFC + ) where + +import Control.Monad.ST (ST) +import Control.Monad (when) +import Data.Primitive.PrimArray + ( MutablePrimArray + , copyMutablePrimArray + , newPrimArray + , readPrimArray + , writePrimArray + ) + +import Text.IDNA2008.Internal.Ranges (inRanges) +import Text.IDNA2008.Internal.NFC.Data + ( nfcNoRangeCount + , nfcNoRanges + , nfcMaybeRangeCount + , nfcMaybeRanges + ) +import Text.IDNA2008.Internal.NFC.Tables + ( canonCompose + , canonDecompose + , cccOf + , hangulCompose + , hangulDecompose + ) + +---------------------------------------------------------------------- +-- Quick_Check +---------------------------------------------------------------------- + +-- | Three-valued result of the Unicode @NFC_Quick_Check@ +-- property. See . +data NfcQc + = NfcYes -- ^ Codepoint is unconditionally in NFC. + | NfcNo -- ^ Codepoint is never in NFC; an NFC + -- normaliser would always replace it. + | NfcMaybe -- ^ Codepoint may or may not be in NFC depending + -- on its neighbours; deciding requires running + -- the full normalisation algorithm. + deriving (Eq, Show) + +-- | Look up the @NFC_Quick_Check@ value for the given codepoint. +-- Codepoints absent from both range tables are 'NfcYes'. +nfcQuickCheck :: Int -> NfcQc +nfcQuickCheck !cp + | inRanges nfcNoRanges nfcNoRangeCount cp = NfcNo + | inRanges nfcMaybeRanges nfcMaybeRangeCount cp = NfcMaybe + | otherwise = NfcYes +{-# INLINE nfcQuickCheck #-} + +---------------------------------------------------------------------- +-- Full validator +---------------------------------------------------------------------- + +-- | Decide precisely whether the codepoint sequence +-- @cpBuf[0..cnt)@ is in Normalization Form C. +-- +-- Strategy: a Quick_Check pass first. If every codepoint is +-- 'NfcYes', the input is in NFC. On the first non-@Yes@ +-- codepoint we fall through to a full normalize-and-compare: +-- canonical decomposition (recursive, including Hangul +-- algorithmic decomposition) into a working buffer; in-place +-- canonical reorder by 'cccOf' (insertion sort within +-- combining-mark runs); in-place primary composition +-- (canonical-blocking rule, plus Hangul algorithmic +-- composition); compare the result to the input. +isNFC :: forall s. MutablePrimArray s Int -> Int -> ST s Bool +isNFC !cpBuf !cnt = do + qcYes <- allYes cpBuf cnt 0 + if qcYes + then pure True + else do + let !workCap = cnt * 4 + 32 + workBuf <- newPrimArray workCap + nDecomp <- decomposeInto cpBuf cnt workBuf + reorderRun workBuf nDecomp + nCompose <- composeRun workBuf nDecomp + if nCompose /= cnt + then pure False + else equalRuns cpBuf workBuf cnt + +-- | Normalise the codepoint buffer @cpBuf[0..cnt)@ to NFC, in +-- place, and return the new length. Reuses the same decompose / +-- reorder / compose pipeline as 'isNFC', then copies the composed +-- run back to @cpBuf@. +-- +-- The caller's @cpBuf@ must have room for the result. In practice +-- NFC composition does not increase the codepoint count beyond +-- the input length, but the caller should size the buffer for the +-- worst case if it cannot guarantee that. +-- +-- Fast path: if every input codepoint passes the @NFC_Quick_Check@ +-- @Yes@ test, the buffer is already in NFC and the function +-- returns @cnt@ unchanged with no work done. +normalizeNFC + :: forall s. MutablePrimArray s Int -> Int -> ST s Int +normalizeNFC !cpBuf !cnt = do + qcYes <- allYes cpBuf cnt 0 + if qcYes + then pure cnt + else do + let !workCap = cnt * 4 + 32 + workBuf <- newPrimArray workCap + nDecomp <- decomposeInto cpBuf cnt workBuf + reorderRun workBuf nDecomp + nCompose <- composeRun workBuf nDecomp + when (nCompose > 0) $ + copyMutablePrimArray cpBuf 0 workBuf 0 nCompose + pure nCompose + +-- | True iff every codepoint in @cpBuf[0..cnt)@ passes the +-- @NFC_Quick_Check@ @Yes@ test (sufficient condition for NFC, +-- not necessary). +allYes :: forall s. MutablePrimArray s Int -> Int -> Int -> ST s Bool +allYes !cpBuf !cnt = go + where + go :: Int -> ST s Bool + go !i + | i >= cnt = pure True + | otherwise = do + cp <- readPrimArray cpBuf i + case nfcQuickCheck cp of + NfcYes -> go (i + 1) + _ -> pure False + +-- | Walk the input buffer and write each codepoint's full +-- canonical decomposition into the output buffer. Returns the +-- number of codepoints written. +decomposeInto + :: forall s + . MutablePrimArray s Int -- input + -> Int -- input length + -> MutablePrimArray s Int -- output (must have room) + -> ST s Int +decomposeInto !inBuf !cnt !outBuf = go 0 0 + where + go :: Int -> Int -> ST s Int + go !inIdx !outIdx + | inIdx >= cnt = pure outIdx + | otherwise = do + cp <- readPrimArray inBuf inIdx + outIdx' <- decomposeCp cp outBuf outIdx + go (inIdx + 1) outIdx' + +-- | Recursively expand @cp@'s canonical decomposition into the +-- output buffer at @outIdx@. Returns the next free output index. +decomposeCp + :: forall s. Int -> MutablePrimArray s Int -> Int -> ST s Int +decomposeCp !cp !outBuf !outIdx + | Just (l, v, mt) <- hangulDecompose cp = do + writePrimArray outBuf outIdx l + writePrimArray outBuf (outIdx + 1) v + case mt of + Nothing -> pure (outIdx + 2) + Just t -> do + writePrimArray outBuf (outIdx + 2) t + pure (outIdx + 3) + | Just (a, b) <- canonDecompose cp = do + outIdx' <- decomposeCp a outBuf outIdx + if b == 0 + then pure outIdx' + else decomposeCp b outBuf outIdx' + | otherwise = do + writePrimArray outBuf outIdx cp + pure (outIdx + 1) + +-- | Canonical reorder: walk the buffer, and within every run of +-- consecutive codepoints whose CCC is non-zero, sort by CCC +-- ascending using insertion sort (which is naturally stable). +-- Codepoints with CCC=0 act as boundaries — no reordering +-- crosses them. +reorderRun :: forall s. MutablePrimArray s Int -> Int -> ST s () +reorderRun !buf !n = go 1 + where + -- Insertion-sort step: at index i, bubble the element backwards + -- as long as its CCC is non-zero AND strictly less than the + -- previous element's non-zero CCC. + go :: Int -> ST s () + go !i + | i >= n = pure () + | otherwise = do + cur <- readPrimArray buf i + let !curCCC = fromIntegral (cccOf cur) :: Int + if curCCC == 0 + then go (i + 1) + else do + bubble i curCCC + go (i + 1) + + bubble :: Int -> Int -> ST s () + bubble !i !curCCC + | i == 0 = pure () + | otherwise = do + prev <- readPrimArray buf (i - 1) + let !prevCCC = fromIntegral (cccOf prev) :: Int + if prevCCC /= 0 && prevCCC > curCCC + then do + cur <- readPrimArray buf i + writePrimArray buf (i - 1) cur + writePrimArray buf i prev + bubble (i - 1) curCCC + else pure () + +-- | In-place primary composition over a canonically-ordered +-- buffer. Walks @[0, n)@ and emits the composed form into +-- @[0, returnedLength)@. The walk maintains: +-- +-- * @starterIdx@: index in the output of the most recently +-- emitted starter, or @-1@ if none yet. +-- * @lastCCC@: the CCC of the most recent /non-starter/ that +-- has been emitted /since/ that starter; used to determine +-- whether a subsequent non-starter is blocked from composing. +-- +-- For each input codepoint: +-- +-- * If it is a non-starter (@CCC > 0@) and not blocked +-- (@lastCCC < ccc@), and a primary composition with the +-- output starter exists (canonical or Hangul), the starter +-- is replaced by the composed codepoint and the input is +-- consumed. Else the input is emitted; @lastCCC@ is updated. +-- +-- * If it is a starter (@CCC = 0@), and the previous output +-- starter is unblocked (@lastCCC == 0@), and a primary +-- composition exists (this is the Hangul L+V \/ LV+T case), +-- the starter is replaced and the input is consumed. Else +-- the input becomes the new output starter, and @lastCCC@ +-- resets to @0@. +composeRun :: forall s. MutablePrimArray s Int -> Int -> ST s Int +composeRun !buf !n = go 0 0 (-1) 0 + where + go :: Int -> Int -> Int -> Int -> ST s Int + go !inIdx !outIdx !starterIdx !lastCCC + | inIdx >= n = pure outIdx + | otherwise = do + cp <- readPrimArray buf inIdx + let !cpCCC = fromIntegral (cccOf cp) :: Int + if cpCCC == 0 + then handleStarter cp inIdx outIdx starterIdx lastCCC + else handleNonStarter cp cpCCC inIdx outIdx starterIdx lastCCC + + handleStarter + :: Int -> Int -> Int -> Int -> Int -> ST s Int + handleStarter !cp !inIdx !outIdx !starterIdx !lastCCC + | starterIdx >= 0 && lastCCC == 0 = do + starter <- readPrimArray buf starterIdx + case composePair starter cp of + Just composed -> do + writePrimArray buf starterIdx composed + go (inIdx + 1) outIdx starterIdx 0 + Nothing -> emitStarter cp inIdx outIdx + | otherwise = emitStarter cp inIdx outIdx + + emitStarter :: Int -> Int -> Int -> ST s Int + emitStarter !cp !inIdx !outIdx = do + writePrimArray buf outIdx cp + go (inIdx + 1) (outIdx + 1) outIdx 0 + + handleNonStarter + :: Int -> Int -> Int -> Int -> Int -> Int -> ST s Int + handleNonStarter !cp !cpCCC !inIdx !outIdx !starterIdx !lastCCC + | starterIdx >= 0 && lastCCC < cpCCC = do + starter <- readPrimArray buf starterIdx + case composePair starter cp of + Just composed -> do + writePrimArray buf starterIdx composed + go (inIdx + 1) outIdx starterIdx lastCCC + Nothing -> emitNonStarter cp cpCCC inIdx outIdx starterIdx + | otherwise = emitNonStarter cp cpCCC inIdx outIdx starterIdx + + emitNonStarter + :: Int -> Int -> Int -> Int -> Int -> ST s Int + emitNonStarter !cp !cpCCC !inIdx !outIdx !starterIdx = do + writePrimArray buf outIdx cp + go (inIdx + 1) (outIdx + 1) starterIdx cpCCC + +-- | Try Hangul algorithmic composition first; fall back to the +-- canonical composition table. +composePair :: Int -> Int -> Maybe Int +composePair !a !b = case hangulCompose a b of + Just c -> Just c + Nothing -> canonCompose a b +{-# INLINE composePair #-} + +-- | Codepoint-by-codepoint equality of @a[0..n)@ and @b[0..n)@. +equalRuns + :: forall s + . MutablePrimArray s Int + -> MutablePrimArray s Int + -> Int + -> ST s Bool +equalRuns !a !b !n = go 0 + where + go :: Int -> ST s Bool + go !i + | i >= n = pure True + | otherwise = do + x <- readPrimArray a i + y <- readPrimArray b i + if x == y then go (i + 1) else pure False + diff --git a/internal/Text/IDNA2008/Internal/NFC/Data.hs b/internal/Text/IDNA2008/Internal/NFC/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/NFC/Data.hs @@ -0,0 +1,306 @@ +-- | +-- Module : Text.IDNA2008.Internal.NFC.Data +-- Description : Codepoint range tables for the Unicode +-- @NFC_Quick_Check@ property values @No@ and +-- @Maybe@. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaNFC.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/DerivedNormalizationProps.txt +-- Aligned with: Unicode 17.0.0 +-- NFC_QC=No ranges: 73 +-- NFC_QC=Maybe ranges: 49 +-- +-- Codepoints absent from both tables are implicitly @Yes@ +-- (already in NFC). +module Text.IDNA2008.Internal.NFC.Data + ( nfcNoRangeCount + , nfcNoRanges + , nfcMaybeRangeCount + , nfcMaybeRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- NFC_Quick_Check = No +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'nfcNoRanges'. +nfcNoRangeCount :: Int +nfcNoRangeCount = 73 + +-- | Codepoint ranges with @NFC_Quick_Check = No@, +-- alternating start\/end @Word32@ pairs. +nfcNoRanges :: ByteArray +nfcNoRanges = PBA.byteArrayFromList + [ (0x000340 :: Word32) + , 0x000341 + , 0x000343 + , 0x000344 + , 0x000374 + , 0x000374 + , 0x00037E + , 0x00037E + , 0x000387 + , 0x000387 + , 0x000958 + , 0x00095F + , 0x0009DC + , 0x0009DD + , 0x0009DF + , 0x0009DF + , 0x000A33 + , 0x000A33 + , 0x000A36 + , 0x000A36 + , 0x000A59 + , 0x000A5B + , 0x000A5E + , 0x000A5E + , 0x000B5C + , 0x000B5D + , 0x000F43 + , 0x000F43 + , 0x000F4D + , 0x000F4D + , 0x000F52 + , 0x000F52 + , 0x000F57 + , 0x000F57 + , 0x000F5C + , 0x000F5C + , 0x000F69 + , 0x000F69 + , 0x000F73 + , 0x000F73 + , 0x000F75 + , 0x000F76 + , 0x000F78 + , 0x000F78 + , 0x000F81 + , 0x000F81 + , 0x000F93 + , 0x000F93 + , 0x000F9D + , 0x000F9D + , 0x000FA2 + , 0x000FA2 + , 0x000FA7 + , 0x000FA7 + , 0x000FAC + , 0x000FAC + , 0x000FB9 + , 0x000FB9 + , 0x001F71 + , 0x001F71 + , 0x001F73 + , 0x001F73 + , 0x001F75 + , 0x001F75 + , 0x001F77 + , 0x001F77 + , 0x001F79 + , 0x001F79 + , 0x001F7B + , 0x001F7B + , 0x001F7D + , 0x001F7D + , 0x001FBB + , 0x001FBB + , 0x001FBE + , 0x001FBE + , 0x001FC9 + , 0x001FC9 + , 0x001FCB + , 0x001FCB + , 0x001FD3 + , 0x001FD3 + , 0x001FDB + , 0x001FDB + , 0x001FE3 + , 0x001FE3 + , 0x001FEB + , 0x001FEB + , 0x001FEE + , 0x001FEF + , 0x001FF9 + , 0x001FF9 + , 0x001FFB + , 0x001FFB + , 0x001FFD + , 0x001FFD + , 0x002000 + , 0x002001 + , 0x002126 + , 0x002126 + , 0x00212A + , 0x00212B + , 0x002329 + , 0x00232A + , 0x002ADC + , 0x002ADC + , 0x00F900 + , 0x00FA0D + , 0x00FA10 + , 0x00FA10 + , 0x00FA12 + , 0x00FA12 + , 0x00FA15 + , 0x00FA1E + , 0x00FA20 + , 0x00FA20 + , 0x00FA22 + , 0x00FA22 + , 0x00FA25 + , 0x00FA26 + , 0x00FA2A + , 0x00FA6D + , 0x00FA70 + , 0x00FAD9 + , 0x00FB1D + , 0x00FB1D + , 0x00FB1F + , 0x00FB1F + , 0x00FB2A + , 0x00FB36 + , 0x00FB38 + , 0x00FB3C + , 0x00FB3E + , 0x00FB3E + , 0x00FB40 + , 0x00FB41 + , 0x00FB43 + , 0x00FB44 + , 0x00FB46 + , 0x00FB4E + , 0x01D15E + , 0x01D164 + , 0x01D1BB + , 0x01D1C0 + , 0x02F800 + , 0x02FA1D + ] + + +---------------------------------------------------------------------- +-- NFC_Quick_Check = Maybe +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'nfcMaybeRanges'. +nfcMaybeRangeCount :: Int +nfcMaybeRangeCount = 49 + +-- | Codepoint ranges with @NFC_Quick_Check = Maybe@, +-- alternating start\/end @Word32@ pairs. +nfcMaybeRanges :: ByteArray +nfcMaybeRanges = PBA.byteArrayFromList + [ (0x000300 :: Word32) + , 0x000304 + , 0x000306 + , 0x00030C + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x000313 + , 0x000314 + , 0x00031B + , 0x00031B + , 0x000323 + , 0x000328 + , 0x00032D + , 0x00032E + , 0x000330 + , 0x000331 + , 0x000338 + , 0x000338 + , 0x000342 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000653 + , 0x000655 + , 0x00093C + , 0x00093C + , 0x0009BE + , 0x0009BE + , 0x0009D7 + , 0x0009D7 + , 0x000B3E + , 0x000B3E + , 0x000B56 + , 0x000B57 + , 0x000BBE + , 0x000BBE + , 0x000BD7 + , 0x000BD7 + , 0x000C56 + , 0x000C56 + , 0x000CC2 + , 0x000CC2 + , 0x000CD5 + , 0x000CD6 + , 0x000D3E + , 0x000D3E + , 0x000D57 + , 0x000D57 + , 0x000DCA + , 0x000DCA + , 0x000DCF + , 0x000DCF + , 0x000DDF + , 0x000DDF + , 0x00102E + , 0x00102E + , 0x001161 + , 0x001175 + , 0x0011A8 + , 0x0011C2 + , 0x001B35 + , 0x001B35 + , 0x003099 + , 0x00309A + , 0x0110BA + , 0x0110BA + , 0x011127 + , 0x011127 + , 0x01133E + , 0x01133E + , 0x011357 + , 0x011357 + , 0x0113B8 + , 0x0113B8 + , 0x0113BB + , 0x0113BB + , 0x0113C2 + , 0x0113C2 + , 0x0113C5 + , 0x0113C5 + , 0x0113C7 + , 0x0113C9 + , 0x0114B0 + , 0x0114B0 + , 0x0114BA + , 0x0114BA + , 0x0114BD + , 0x0114BD + , 0x0115AF + , 0x0115AF + , 0x011930 + , 0x011930 + , 0x01611E + , 0x016129 + , 0x016D67 + , 0x016D68 + ] diff --git a/internal/Text/IDNA2008/Internal/NFC/Tables.hs b/internal/Text/IDNA2008/Internal/NFC/Tables.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/NFC/Tables.hs @@ -0,0 +1,253 @@ +-- | +-- Module : Text.IDNA2008.Internal.NFC.Tables +-- Description : Typed wrappers over the Unicode normalization +-- tables consumed by the full NFC validator. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Three lookups, plus the Hangul algorithmic decomposition and +-- composition. Each lookup is a binary search over a sorted +-- range or key array in +-- "Text.IDNA2008.Internal.NFC.Tables.Data". Codepoints not in +-- the relevant table are reported as "no decomposition", "no +-- composition", or @CCC = 0@. +-- +-- Hangul syllables are intentionally absent from the tables; they +-- are decomposed and recomposed by closed-form arithmetic +-- ('hangulDecompose', 'hangulCompose') because the table form +-- would be over 11000 entries. +{-# LANGUAGE BangPatterns #-} + +module Text.IDNA2008.Internal.NFC.Tables + ( -- * Decomposition + canonDecompose + , hangulDecompose + + -- * Composition + , canonCompose + , hangulCompose + + -- * Canonical_Combining_Class + , cccOf + + -- * Hangul constants (re-exported for the algorithm) + , hangulSBase + , hangulSCount + ) where + +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word8, Word32) + +import Text.IDNA2008.Internal.NFC.Tables.Data + +---------------------------------------------------------------------- +-- Hangul algorithmic constants and helpers +---------------------------------------------------------------------- + +hangulSBase, hangulSCount :: Int +hangulSBase = 0xAC00 +hangulSCount = 11172 + +hangulLBase, hangulLCount :: Int +hangulLBase = 0x1100 +hangulLCount = 19 + +hangulVBase, hangulVCount :: Int +hangulVBase = 0x1161 +hangulVCount = 21 + +hangulTBase, hangulTCount :: Int +hangulTBase = 0x11A7 +hangulTCount = 28 + +-- | Cluster size: @vCount * tCount@. +hangulNCount :: Int +hangulNCount = hangulVCount * hangulTCount + +-- | Hangul algorithmic canonical decomposition. A syllable in +-- @[U+AC00, U+D7A3]@ decomposes either into @(L, V)@ if its +-- TIndex is zero, or into @(L, V, T)@ otherwise. For codepoints +-- outside the syllable range, 'Nothing'. +hangulDecompose :: Int -> Maybe (Int, Int, Maybe Int) +hangulDecompose !cp + | cp < hangulSBase || sIndex >= hangulSCount = Nothing + | tIndex == 0 = Just (l, v, Nothing) + | otherwise = Just (l, v, Just t) + where + !sIndex = cp - hangulSBase + !lIndex = sIndex `quot` hangulNCount + !vIndex = (sIndex `rem` hangulNCount) `quot` hangulTCount + !tIndex = sIndex `rem` hangulTCount + !l = hangulLBase + lIndex + !v = hangulVBase + vIndex + !t = hangulTBase + tIndex +{-# INLINE hangulDecompose #-} + +-- | Hangul algorithmic primary composition. Two cases: an L +-- jamo plus a V jamo composes into an LV syllable; an LV +-- syllable (one whose TIndex is zero) plus a T jamo composes +-- into an LVT syllable. Anything else returns 'Nothing'. +hangulCompose :: Int -> Int -> Maybe Int +hangulCompose !first !second + -- L + V -> LV + | lIdx <- first - hangulLBase + , vIdx <- second - hangulVBase + , lIdx >= 0, lIdx < hangulLCount + , vIdx >= 0, vIdx < hangulVCount + = Just (hangulSBase + (lIdx * hangulVCount + vIdx) * hangulTCount) + -- LV + T -> LVT (only if @first@ is an LV-shaped syllable + -- and @second@ is a T jamo strictly after the T-zero filler). + | sIdx <- first - hangulSBase + , sIdx >= 0, sIdx < hangulSCount + , sIdx `rem` hangulTCount == 0 + , tIdx <- second - hangulTBase + , tIdx > 0, tIdx < hangulTCount + = Just (first + tIdx) + | otherwise = Nothing +{-# INLINE hangulCompose #-} + +---------------------------------------------------------------------- +-- Canonical decomposition (table) +---------------------------------------------------------------------- + +-- | Look up the canonical decomposition of @cp@ in the +-- 'decompKeys' table. Returns @Just (a, b)@ for a two-codepoint +-- decomposition, or @Just (a, 0)@ for a singleton decomposition +-- (where @0@ is a sentinel meaning \"no second codepoint\"). +-- Hangul syllables are absent from the table; use +-- 'hangulDecompose' instead. +canonDecompose :: Int -> Maybe (Int, Int) +canonDecompose !cp + | decompCount == 0 = Nothing + | cp < keyAt 0 || cp > keyAt (decompCount - 1) = Nothing + | otherwise = case bsearchKey 0 (decompCount - 1) of + Just i -> Just (firstAt i, secondAt i) + Nothing -> Nothing + where + keyAt :: Int -> Int + keyAt !i = fromIntegral (indexByteArray decompKeys i :: Word32) + {-# INLINE keyAt #-} + + firstAt :: Int -> Int + firstAt !i = fromIntegral (indexByteArray decompFirst i :: Word32) + {-# INLINE firstAt #-} + + secondAt :: Int -> Int + secondAt !i = fromIntegral (indexByteArray decompSecond i :: Word32) + {-# INLINE secondAt #-} + + -- Standard binary search for an exact match. + bsearchKey :: Int -> Int -> Maybe Int + bsearchKey !lo !hi + | lo > hi = Nothing + | otherwise = + let !mid = (lo + hi) `quot` 2 + !k = keyAt mid + in if | k == cp -> Just mid + | k < cp -> bsearchKey (mid + 1) hi + | otherwise -> bsearchKey lo (mid - 1) +{-# INLINE canonDecompose #-} + +---------------------------------------------------------------------- +-- Primary composition (table) +---------------------------------------------------------------------- + +-- | Look up the primary composition of @(first, second)@ in the +-- composition table. Returns @Just result@ if such a composition +-- exists in Unicode (and was not excluded), else 'Nothing'. +-- Hangul handled separately via 'hangulCompose'. +canonCompose :: Int -> Int -> Maybe Int +canonCompose !first !second + | compCount == 0 = Nothing + | otherwise = case bsearchFirst 0 (compCount - 1) of + Nothing -> Nothing + Just (lo, hi) -> linearMatchSecond lo hi + where + firstAt :: Int -> Int + firstAt !i = fromIntegral (indexByteArray compFirst i :: Word32) + {-# INLINE firstAt #-} + + secondAt :: Int -> Int + secondAt !i = fromIntegral (indexByteArray compSecond i :: Word32) + {-# INLINE secondAt #-} + + resultAt :: Int -> Int + resultAt !i = fromIntegral (indexByteArray compResult i :: Word32) + {-# INLINE resultAt #-} + + -- Find the (inclusive) range of indices where firstAt i == first. + -- Return 'Nothing' if no such range exists; otherwise (lo, hi). + bsearchFirst :: Int -> Int -> Maybe (Int, Int) + bsearchFirst !lo !hi + | lo > hi = Nothing + | otherwise = + let !mid = (lo + hi) `quot` 2 + !k = firstAt mid + in if | k == first -> + -- Expand around mid to cover the full equal-first run. + let !rl = expandLeft mid + !rh = expandRight mid + in Just (rl, rh) + | k < first -> bsearchFirst (mid + 1) hi + | otherwise -> bsearchFirst lo (mid - 1) + + -- Walk left from `i` while firstAt is still equal to `first`. + expandLeft :: Int -> Int + expandLeft !i + | i > 0, firstAt (i - 1) == first = expandLeft (i - 1) + | otherwise = i + + expandRight :: Int -> Int + expandRight !i + | i + 1 < compCount, firstAt (i + 1) == first = expandRight (i + 1) + | otherwise = i + + -- Within the equal-first range [lo, hi], scan for a matching + -- second. The range is small in practice (a starter composes + -- with at most a handful of combining marks). + linearMatchSecond :: Int -> Int -> Maybe Int + linearMatchSecond !lo !hi + | lo > hi = Nothing + | secondAt lo == second = Just (resultAt lo) + | otherwise = linearMatchSecond (lo + 1) hi +{-# INLINE canonCompose #-} + +---------------------------------------------------------------------- +-- Canonical_Combining_Class +---------------------------------------------------------------------- + +-- | Look up the Canonical_Combining_Class of @cp@. Codepoints +-- absent from the CCC table are starters (CCC = 0). +cccOf :: Int -> Word8 +cccOf !cp + | cccCount == 0 || cp < startAt 0 = 0 + | otherwise = + let !idx = bsearchStarts 0 (cccCount - 1) + in if cp <= endAt idx + then valueAt idx + else 0 + where + startAt :: Int -> Int + startAt !i = fromIntegral (indexByteArray cccStarts i :: Word32) + {-# INLINE startAt #-} + + endAt :: Int -> Int + endAt !i = fromIntegral (indexByteArray cccEnds i :: Word32) + {-# INLINE endAt #-} + + valueAt :: Int -> Word8 + valueAt !i = indexByteArray cccValues i + {-# INLINE valueAt #-} + + bsearchStarts :: Int -> Int -> Int + bsearchStarts !lo !hi + | lo == hi = lo + | otherwise = + let !mid = (lo + hi + 1) `quot` 2 + in if startAt mid <= cp + then bsearchStarts mid hi + else bsearchStarts lo (mid - 1) +{-# INLINE cccOf #-} diff --git a/internal/Text/IDNA2008/Internal/NFC/Tables/Data.hs b/internal/Text/IDNA2008/Internal/NFC/Tables/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/NFC/Tables/Data.hs @@ -0,0 +1,10441 @@ +-- | +-- Module : Text.IDNA2008.Internal.NFC.Tables.Data +-- Description : Canonical decomposition, primary composition, and +-- Canonical_Combining_Class tables used by the full +-- NFC normalize-and-compare validator. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaNFCTables.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/ +-- Aligned with: Unicode 17.0.0 +-- Decomposition entries: 2081 +-- Composition entries: 961 +-- CCC ranges: 403 +-- +-- Hangul syllables are excluded from all three tables; the +-- consumer module handles them via closed-form arithmetic. +module Text.IDNA2008.Internal.NFC.Tables.Data + ( -- * Canonical decomposition + decompCount + , decompKeys + , decompFirst + , decompSecond + + -- * Primary composition + , compCount + , compFirst + , compSecond + , compResult + + -- * Canonical_Combining_Class + , cccCount + , cccStarts + , cccEnds + , cccValues + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- Canonical decomposition +---------------------------------------------------------------------- + +decompCount :: Int +decompCount = 2081 + +decompKeys :: ByteArray +decompKeys = PBA.byteArrayFromList + [ (0x0000C0 :: Word32) + , 0x0000C1 + , 0x0000C2 + , 0x0000C3 + , 0x0000C4 + , 0x0000C5 + , 0x0000C7 + , 0x0000C8 + , 0x0000C9 + , 0x0000CA + , 0x0000CB + , 0x0000CC + , 0x0000CD + , 0x0000CE + , 0x0000CF + , 0x0000D1 + , 0x0000D2 + , 0x0000D3 + , 0x0000D4 + , 0x0000D5 + , 0x0000D6 + , 0x0000D9 + , 0x0000DA + , 0x0000DB + , 0x0000DC + , 0x0000DD + , 0x0000E0 + , 0x0000E1 + , 0x0000E2 + , 0x0000E3 + , 0x0000E4 + , 0x0000E5 + , 0x0000E7 + , 0x0000E8 + , 0x0000E9 + , 0x0000EA + , 0x0000EB + , 0x0000EC + , 0x0000ED + , 0x0000EE + , 0x0000EF + , 0x0000F1 + , 0x0000F2 + , 0x0000F3 + , 0x0000F4 + , 0x0000F5 + , 0x0000F6 + , 0x0000F9 + , 0x0000FA + , 0x0000FB + , 0x0000FC + , 0x0000FD + , 0x0000FF + , 0x000100 + , 0x000101 + , 0x000102 + , 0x000103 + , 0x000104 + , 0x000105 + , 0x000106 + , 0x000107 + , 0x000108 + , 0x000109 + , 0x00010A + , 0x00010B + , 0x00010C + , 0x00010D + , 0x00010E + , 0x00010F + , 0x000112 + , 0x000113 + , 0x000114 + , 0x000115 + , 0x000116 + , 0x000117 + , 0x000118 + , 0x000119 + , 0x00011A + , 0x00011B + , 0x00011C + , 0x00011D + , 0x00011E + , 0x00011F + , 0x000120 + , 0x000121 + , 0x000122 + , 0x000123 + , 0x000124 + , 0x000125 + , 0x000128 + , 0x000129 + , 0x00012A + , 0x00012B + , 0x00012C + , 0x00012D + , 0x00012E + , 0x00012F + , 0x000130 + , 0x000134 + , 0x000135 + , 0x000136 + , 0x000137 + , 0x000139 + , 0x00013A + , 0x00013B + , 0x00013C + , 0x00013D + , 0x00013E + , 0x000143 + , 0x000144 + , 0x000145 + , 0x000146 + , 0x000147 + , 0x000148 + , 0x00014C + , 0x00014D + , 0x00014E + , 0x00014F + , 0x000150 + , 0x000151 + , 0x000154 + , 0x000155 + , 0x000156 + , 0x000157 + , 0x000158 + , 0x000159 + , 0x00015A + , 0x00015B + , 0x00015C + , 0x00015D + , 0x00015E + , 0x00015F + , 0x000160 + , 0x000161 + , 0x000162 + , 0x000163 + , 0x000164 + , 0x000165 + , 0x000168 + , 0x000169 + , 0x00016A + , 0x00016B + , 0x00016C + , 0x00016D + , 0x00016E + , 0x00016F + , 0x000170 + , 0x000171 + , 0x000172 + , 0x000173 + , 0x000174 + , 0x000175 + , 0x000176 + , 0x000177 + , 0x000178 + , 0x000179 + , 0x00017A + , 0x00017B + , 0x00017C + , 0x00017D + , 0x00017E + , 0x0001A0 + , 0x0001A1 + , 0x0001AF + , 0x0001B0 + , 0x0001CD + , 0x0001CE + , 0x0001CF + , 0x0001D0 + , 0x0001D1 + , 0x0001D2 + , 0x0001D3 + , 0x0001D4 + , 0x0001D5 + , 0x0001D6 + , 0x0001D7 + , 0x0001D8 + , 0x0001D9 + , 0x0001DA + , 0x0001DB + , 0x0001DC + , 0x0001DE + , 0x0001DF + , 0x0001E0 + , 0x0001E1 + , 0x0001E2 + , 0x0001E3 + , 0x0001E6 + , 0x0001E7 + , 0x0001E8 + , 0x0001E9 + , 0x0001EA + , 0x0001EB + , 0x0001EC + , 0x0001ED + , 0x0001EE + , 0x0001EF + , 0x0001F0 + , 0x0001F4 + , 0x0001F5 + , 0x0001F8 + , 0x0001F9 + , 0x0001FA + , 0x0001FB + , 0x0001FC + , 0x0001FD + , 0x0001FE + , 0x0001FF + , 0x000200 + , 0x000201 + , 0x000202 + , 0x000203 + , 0x000204 + , 0x000205 + , 0x000206 + , 0x000207 + , 0x000208 + , 0x000209 + , 0x00020A + , 0x00020B + , 0x00020C + , 0x00020D + , 0x00020E + , 0x00020F + , 0x000210 + , 0x000211 + , 0x000212 + , 0x000213 + , 0x000214 + , 0x000215 + , 0x000216 + , 0x000217 + , 0x000218 + , 0x000219 + , 0x00021A + , 0x00021B + , 0x00021E + , 0x00021F + , 0x000226 + , 0x000227 + , 0x000228 + , 0x000229 + , 0x00022A + , 0x00022B + , 0x00022C + , 0x00022D + , 0x00022E + , 0x00022F + , 0x000230 + , 0x000231 + , 0x000232 + , 0x000233 + , 0x000340 + , 0x000341 + , 0x000343 + , 0x000344 + , 0x000374 + , 0x00037E + , 0x000385 + , 0x000386 + , 0x000387 + , 0x000388 + , 0x000389 + , 0x00038A + , 0x00038C + , 0x00038E + , 0x00038F + , 0x000390 + , 0x0003AA + , 0x0003AB + , 0x0003AC + , 0x0003AD + , 0x0003AE + , 0x0003AF + , 0x0003B0 + , 0x0003CA + , 0x0003CB + , 0x0003CC + , 0x0003CD + , 0x0003CE + , 0x0003D3 + , 0x0003D4 + , 0x000400 + , 0x000401 + , 0x000403 + , 0x000407 + , 0x00040C + , 0x00040D + , 0x00040E + , 0x000419 + , 0x000439 + , 0x000450 + , 0x000451 + , 0x000453 + , 0x000457 + , 0x00045C + , 0x00045D + , 0x00045E + , 0x000476 + , 0x000477 + , 0x0004C1 + , 0x0004C2 + , 0x0004D0 + , 0x0004D1 + , 0x0004D2 + , 0x0004D3 + , 0x0004D6 + , 0x0004D7 + , 0x0004DA + , 0x0004DB + , 0x0004DC + , 0x0004DD + , 0x0004DE + , 0x0004DF + , 0x0004E2 + , 0x0004E3 + , 0x0004E4 + , 0x0004E5 + , 0x0004E6 + , 0x0004E7 + , 0x0004EA + , 0x0004EB + , 0x0004EC + , 0x0004ED + , 0x0004EE + , 0x0004EF + , 0x0004F0 + , 0x0004F1 + , 0x0004F2 + , 0x0004F3 + , 0x0004F4 + , 0x0004F5 + , 0x0004F8 + , 0x0004F9 + , 0x000622 + , 0x000623 + , 0x000624 + , 0x000625 + , 0x000626 + , 0x0006C0 + , 0x0006C2 + , 0x0006D3 + , 0x000929 + , 0x000931 + , 0x000934 + , 0x000958 + , 0x000959 + , 0x00095A + , 0x00095B + , 0x00095C + , 0x00095D + , 0x00095E + , 0x00095F + , 0x0009CB + , 0x0009CC + , 0x0009DC + , 0x0009DD + , 0x0009DF + , 0x000A33 + , 0x000A36 + , 0x000A59 + , 0x000A5A + , 0x000A5B + , 0x000A5E + , 0x000B48 + , 0x000B4B + , 0x000B4C + , 0x000B5C + , 0x000B5D + , 0x000B94 + , 0x000BCA + , 0x000BCB + , 0x000BCC + , 0x000C48 + , 0x000CC0 + , 0x000CC7 + , 0x000CC8 + , 0x000CCA + , 0x000CCB + , 0x000D4A + , 0x000D4B + , 0x000D4C + , 0x000DDA + , 0x000DDC + , 0x000DDD + , 0x000DDE + , 0x000F43 + , 0x000F4D + , 0x000F52 + , 0x000F57 + , 0x000F5C + , 0x000F69 + , 0x000F73 + , 0x000F75 + , 0x000F76 + , 0x000F78 + , 0x000F81 + , 0x000F93 + , 0x000F9D + , 0x000FA2 + , 0x000FA7 + , 0x000FAC + , 0x000FB9 + , 0x001026 + , 0x001B06 + , 0x001B08 + , 0x001B0A + , 0x001B0C + , 0x001B0E + , 0x001B12 + , 0x001B3B + , 0x001B3D + , 0x001B40 + , 0x001B41 + , 0x001B43 + , 0x001E00 + , 0x001E01 + , 0x001E02 + , 0x001E03 + , 0x001E04 + , 0x001E05 + , 0x001E06 + , 0x001E07 + , 0x001E08 + , 0x001E09 + , 0x001E0A + , 0x001E0B + , 0x001E0C + , 0x001E0D + , 0x001E0E + , 0x001E0F + , 0x001E10 + , 0x001E11 + , 0x001E12 + , 0x001E13 + , 0x001E14 + , 0x001E15 + , 0x001E16 + , 0x001E17 + , 0x001E18 + , 0x001E19 + , 0x001E1A + , 0x001E1B + , 0x001E1C + , 0x001E1D + , 0x001E1E + , 0x001E1F + , 0x001E20 + , 0x001E21 + , 0x001E22 + , 0x001E23 + , 0x001E24 + , 0x001E25 + , 0x001E26 + , 0x001E27 + , 0x001E28 + , 0x001E29 + , 0x001E2A + , 0x001E2B + , 0x001E2C + , 0x001E2D + , 0x001E2E + , 0x001E2F + , 0x001E30 + , 0x001E31 + , 0x001E32 + , 0x001E33 + , 0x001E34 + , 0x001E35 + , 0x001E36 + , 0x001E37 + , 0x001E38 + , 0x001E39 + , 0x001E3A + , 0x001E3B + , 0x001E3C + , 0x001E3D + , 0x001E3E + , 0x001E3F + , 0x001E40 + , 0x001E41 + , 0x001E42 + , 0x001E43 + , 0x001E44 + , 0x001E45 + , 0x001E46 + , 0x001E47 + , 0x001E48 + , 0x001E49 + , 0x001E4A + , 0x001E4B + , 0x001E4C + , 0x001E4D + , 0x001E4E + , 0x001E4F + , 0x001E50 + , 0x001E51 + , 0x001E52 + , 0x001E53 + , 0x001E54 + , 0x001E55 + , 0x001E56 + , 0x001E57 + , 0x001E58 + , 0x001E59 + , 0x001E5A + , 0x001E5B + , 0x001E5C + , 0x001E5D + , 0x001E5E + , 0x001E5F + , 0x001E60 + , 0x001E61 + , 0x001E62 + , 0x001E63 + , 0x001E64 + , 0x001E65 + , 0x001E66 + , 0x001E67 + , 0x001E68 + , 0x001E69 + , 0x001E6A + , 0x001E6B + , 0x001E6C + , 0x001E6D + , 0x001E6E + , 0x001E6F + , 0x001E70 + , 0x001E71 + , 0x001E72 + , 0x001E73 + , 0x001E74 + , 0x001E75 + , 0x001E76 + , 0x001E77 + , 0x001E78 + , 0x001E79 + , 0x001E7A + , 0x001E7B + , 0x001E7C + , 0x001E7D + , 0x001E7E + , 0x001E7F + , 0x001E80 + , 0x001E81 + , 0x001E82 + , 0x001E83 + , 0x001E84 + , 0x001E85 + , 0x001E86 + , 0x001E87 + , 0x001E88 + , 0x001E89 + , 0x001E8A + , 0x001E8B + , 0x001E8C + , 0x001E8D + , 0x001E8E + , 0x001E8F + , 0x001E90 + , 0x001E91 + , 0x001E92 + , 0x001E93 + , 0x001E94 + , 0x001E95 + , 0x001E96 + , 0x001E97 + , 0x001E98 + , 0x001E99 + , 0x001E9B + , 0x001EA0 + , 0x001EA1 + , 0x001EA2 + , 0x001EA3 + , 0x001EA4 + , 0x001EA5 + , 0x001EA6 + , 0x001EA7 + , 0x001EA8 + , 0x001EA9 + , 0x001EAA + , 0x001EAB + , 0x001EAC + , 0x001EAD + , 0x001EAE + , 0x001EAF + , 0x001EB0 + , 0x001EB1 + , 0x001EB2 + , 0x001EB3 + , 0x001EB4 + , 0x001EB5 + , 0x001EB6 + , 0x001EB7 + , 0x001EB8 + , 0x001EB9 + , 0x001EBA + , 0x001EBB + , 0x001EBC + , 0x001EBD + , 0x001EBE + , 0x001EBF + , 0x001EC0 + , 0x001EC1 + , 0x001EC2 + , 0x001EC3 + , 0x001EC4 + , 0x001EC5 + , 0x001EC6 + , 0x001EC7 + , 0x001EC8 + , 0x001EC9 + , 0x001ECA + , 0x001ECB + , 0x001ECC + , 0x001ECD + , 0x001ECE + , 0x001ECF + , 0x001ED0 + , 0x001ED1 + , 0x001ED2 + , 0x001ED3 + , 0x001ED4 + , 0x001ED5 + , 0x001ED6 + , 0x001ED7 + , 0x001ED8 + , 0x001ED9 + , 0x001EDA + , 0x001EDB + , 0x001EDC + , 0x001EDD + , 0x001EDE + , 0x001EDF + , 0x001EE0 + , 0x001EE1 + , 0x001EE2 + , 0x001EE3 + , 0x001EE4 + , 0x001EE5 + , 0x001EE6 + , 0x001EE7 + , 0x001EE8 + , 0x001EE9 + , 0x001EEA + , 0x001EEB + , 0x001EEC + , 0x001EED + , 0x001EEE + , 0x001EEF + , 0x001EF0 + , 0x001EF1 + , 0x001EF2 + , 0x001EF3 + , 0x001EF4 + , 0x001EF5 + , 0x001EF6 + , 0x001EF7 + , 0x001EF8 + , 0x001EF9 + , 0x001F00 + , 0x001F01 + , 0x001F02 + , 0x001F03 + , 0x001F04 + , 0x001F05 + , 0x001F06 + , 0x001F07 + , 0x001F08 + , 0x001F09 + , 0x001F0A + , 0x001F0B + , 0x001F0C + , 0x001F0D + , 0x001F0E + , 0x001F0F + , 0x001F10 + , 0x001F11 + , 0x001F12 + , 0x001F13 + , 0x001F14 + , 0x001F15 + , 0x001F18 + , 0x001F19 + , 0x001F1A + , 0x001F1B + , 0x001F1C + , 0x001F1D + , 0x001F20 + , 0x001F21 + , 0x001F22 + , 0x001F23 + , 0x001F24 + , 0x001F25 + , 0x001F26 + , 0x001F27 + , 0x001F28 + , 0x001F29 + , 0x001F2A + , 0x001F2B + , 0x001F2C + , 0x001F2D + , 0x001F2E + , 0x001F2F + , 0x001F30 + , 0x001F31 + , 0x001F32 + , 0x001F33 + , 0x001F34 + , 0x001F35 + , 0x001F36 + , 0x001F37 + , 0x001F38 + , 0x001F39 + , 0x001F3A + , 0x001F3B + , 0x001F3C + , 0x001F3D + , 0x001F3E + , 0x001F3F + , 0x001F40 + , 0x001F41 + , 0x001F42 + , 0x001F43 + , 0x001F44 + , 0x001F45 + , 0x001F48 + , 0x001F49 + , 0x001F4A + , 0x001F4B + , 0x001F4C + , 0x001F4D + , 0x001F50 + , 0x001F51 + , 0x001F52 + , 0x001F53 + , 0x001F54 + , 0x001F55 + , 0x001F56 + , 0x001F57 + , 0x001F59 + , 0x001F5B + , 0x001F5D + , 0x001F5F + , 0x001F60 + , 0x001F61 + , 0x001F62 + , 0x001F63 + , 0x001F64 + , 0x001F65 + , 0x001F66 + , 0x001F67 + , 0x001F68 + , 0x001F69 + , 0x001F6A + , 0x001F6B + , 0x001F6C + , 0x001F6D + , 0x001F6E + , 0x001F6F + , 0x001F70 + , 0x001F71 + , 0x001F72 + , 0x001F73 + , 0x001F74 + , 0x001F75 + , 0x001F76 + , 0x001F77 + , 0x001F78 + , 0x001F79 + , 0x001F7A + , 0x001F7B + , 0x001F7C + , 0x001F7D + , 0x001F80 + , 0x001F81 + , 0x001F82 + , 0x001F83 + , 0x001F84 + , 0x001F85 + , 0x001F86 + , 0x001F87 + , 0x001F88 + , 0x001F89 + , 0x001F8A + , 0x001F8B + , 0x001F8C + , 0x001F8D + , 0x001F8E + , 0x001F8F + , 0x001F90 + , 0x001F91 + , 0x001F92 + , 0x001F93 + , 0x001F94 + , 0x001F95 + , 0x001F96 + , 0x001F97 + , 0x001F98 + , 0x001F99 + , 0x001F9A + , 0x001F9B + , 0x001F9C + , 0x001F9D + , 0x001F9E + , 0x001F9F + , 0x001FA0 + , 0x001FA1 + , 0x001FA2 + , 0x001FA3 + , 0x001FA4 + , 0x001FA5 + , 0x001FA6 + , 0x001FA7 + , 0x001FA8 + , 0x001FA9 + , 0x001FAA + , 0x001FAB + , 0x001FAC + , 0x001FAD + , 0x001FAE + , 0x001FAF + , 0x001FB0 + , 0x001FB1 + , 0x001FB2 + , 0x001FB3 + , 0x001FB4 + , 0x001FB6 + , 0x001FB7 + , 0x001FB8 + , 0x001FB9 + , 0x001FBA + , 0x001FBB + , 0x001FBC + , 0x001FBE + , 0x001FC1 + , 0x001FC2 + , 0x001FC3 + , 0x001FC4 + , 0x001FC6 + , 0x001FC7 + , 0x001FC8 + , 0x001FC9 + , 0x001FCA + , 0x001FCB + , 0x001FCC + , 0x001FCD + , 0x001FCE + , 0x001FCF + , 0x001FD0 + , 0x001FD1 + , 0x001FD2 + , 0x001FD3 + , 0x001FD6 + , 0x001FD7 + , 0x001FD8 + , 0x001FD9 + , 0x001FDA + , 0x001FDB + , 0x001FDD + , 0x001FDE + , 0x001FDF + , 0x001FE0 + , 0x001FE1 + , 0x001FE2 + , 0x001FE3 + , 0x001FE4 + , 0x001FE5 + , 0x001FE6 + , 0x001FE7 + , 0x001FE8 + , 0x001FE9 + , 0x001FEA + , 0x001FEB + , 0x001FEC + , 0x001FED + , 0x001FEE + , 0x001FEF + , 0x001FF2 + , 0x001FF3 + , 0x001FF4 + , 0x001FF6 + , 0x001FF7 + , 0x001FF8 + , 0x001FF9 + , 0x001FFA + , 0x001FFB + , 0x001FFC + , 0x001FFD + , 0x002000 + , 0x002001 + , 0x002126 + , 0x00212A + , 0x00212B + , 0x00219A + , 0x00219B + , 0x0021AE + , 0x0021CD + , 0x0021CE + , 0x0021CF + , 0x002204 + , 0x002209 + , 0x00220C + , 0x002224 + , 0x002226 + , 0x002241 + , 0x002244 + , 0x002247 + , 0x002249 + , 0x002260 + , 0x002262 + , 0x00226D + , 0x00226E + , 0x00226F + , 0x002270 + , 0x002271 + , 0x002274 + , 0x002275 + , 0x002278 + , 0x002279 + , 0x002280 + , 0x002281 + , 0x002284 + , 0x002285 + , 0x002288 + , 0x002289 + , 0x0022AC + , 0x0022AD + , 0x0022AE + , 0x0022AF + , 0x0022E0 + , 0x0022E1 + , 0x0022E2 + , 0x0022E3 + , 0x0022EA + , 0x0022EB + , 0x0022EC + , 0x0022ED + , 0x002329 + , 0x00232A + , 0x002ADC + , 0x00304C + , 0x00304E + , 0x003050 + , 0x003052 + , 0x003054 + , 0x003056 + , 0x003058 + , 0x00305A + , 0x00305C + , 0x00305E + , 0x003060 + , 0x003062 + , 0x003065 + , 0x003067 + , 0x003069 + , 0x003070 + , 0x003071 + , 0x003073 + , 0x003074 + , 0x003076 + , 0x003077 + , 0x003079 + , 0x00307A + , 0x00307C + , 0x00307D + , 0x003094 + , 0x00309E + , 0x0030AC + , 0x0030AE + , 0x0030B0 + , 0x0030B2 + , 0x0030B4 + , 0x0030B6 + , 0x0030B8 + , 0x0030BA + , 0x0030BC + , 0x0030BE + , 0x0030C0 + , 0x0030C2 + , 0x0030C5 + , 0x0030C7 + , 0x0030C9 + , 0x0030D0 + , 0x0030D1 + , 0x0030D3 + , 0x0030D4 + , 0x0030D6 + , 0x0030D7 + , 0x0030D9 + , 0x0030DA + , 0x0030DC + , 0x0030DD + , 0x0030F4 + , 0x0030F7 + , 0x0030F8 + , 0x0030F9 + , 0x0030FA + , 0x0030FE + , 0x00F900 + , 0x00F901 + , 0x00F902 + , 0x00F903 + , 0x00F904 + , 0x00F905 + , 0x00F906 + , 0x00F907 + , 0x00F908 + , 0x00F909 + , 0x00F90A + , 0x00F90B + , 0x00F90C + , 0x00F90D + , 0x00F90E + , 0x00F90F + , 0x00F910 + , 0x00F911 + , 0x00F912 + , 0x00F913 + , 0x00F914 + , 0x00F915 + , 0x00F916 + , 0x00F917 + , 0x00F918 + , 0x00F919 + , 0x00F91A + , 0x00F91B + , 0x00F91C + , 0x00F91D + , 0x00F91E + , 0x00F91F + , 0x00F920 + , 0x00F921 + , 0x00F922 + , 0x00F923 + , 0x00F924 + , 0x00F925 + , 0x00F926 + , 0x00F927 + , 0x00F928 + , 0x00F929 + , 0x00F92A + , 0x00F92B + , 0x00F92C + , 0x00F92D + , 0x00F92E + , 0x00F92F + , 0x00F930 + , 0x00F931 + , 0x00F932 + , 0x00F933 + , 0x00F934 + , 0x00F935 + , 0x00F936 + , 0x00F937 + , 0x00F938 + , 0x00F939 + , 0x00F93A + , 0x00F93B + , 0x00F93C + , 0x00F93D + , 0x00F93E + , 0x00F93F + , 0x00F940 + , 0x00F941 + , 0x00F942 + , 0x00F943 + , 0x00F944 + , 0x00F945 + , 0x00F946 + , 0x00F947 + , 0x00F948 + , 0x00F949 + , 0x00F94A + , 0x00F94B + , 0x00F94C + , 0x00F94D + , 0x00F94E + , 0x00F94F + , 0x00F950 + , 0x00F951 + , 0x00F952 + , 0x00F953 + , 0x00F954 + , 0x00F955 + , 0x00F956 + , 0x00F957 + , 0x00F958 + , 0x00F959 + , 0x00F95A + , 0x00F95B + , 0x00F95C + , 0x00F95D + , 0x00F95E + , 0x00F95F + , 0x00F960 + , 0x00F961 + , 0x00F962 + , 0x00F963 + , 0x00F964 + , 0x00F965 + , 0x00F966 + , 0x00F967 + , 0x00F968 + , 0x00F969 + , 0x00F96A + , 0x00F96B + , 0x00F96C + , 0x00F96D + , 0x00F96E + , 0x00F96F + , 0x00F970 + , 0x00F971 + , 0x00F972 + , 0x00F973 + , 0x00F974 + , 0x00F975 + , 0x00F976 + , 0x00F977 + , 0x00F978 + , 0x00F979 + , 0x00F97A + , 0x00F97B + , 0x00F97C + , 0x00F97D + , 0x00F97E + , 0x00F97F + , 0x00F980 + , 0x00F981 + , 0x00F982 + , 0x00F983 + , 0x00F984 + , 0x00F985 + , 0x00F986 + , 0x00F987 + , 0x00F988 + , 0x00F989 + , 0x00F98A + , 0x00F98B + , 0x00F98C + , 0x00F98D + , 0x00F98E + , 0x00F98F + , 0x00F990 + , 0x00F991 + , 0x00F992 + , 0x00F993 + , 0x00F994 + , 0x00F995 + , 0x00F996 + , 0x00F997 + , 0x00F998 + , 0x00F999 + , 0x00F99A + , 0x00F99B + , 0x00F99C + , 0x00F99D + , 0x00F99E + , 0x00F99F + , 0x00F9A0 + , 0x00F9A1 + , 0x00F9A2 + , 0x00F9A3 + , 0x00F9A4 + , 0x00F9A5 + , 0x00F9A6 + , 0x00F9A7 + , 0x00F9A8 + , 0x00F9A9 + , 0x00F9AA + , 0x00F9AB + , 0x00F9AC + , 0x00F9AD + , 0x00F9AE + , 0x00F9AF + , 0x00F9B0 + , 0x00F9B1 + , 0x00F9B2 + , 0x00F9B3 + , 0x00F9B4 + , 0x00F9B5 + , 0x00F9B6 + , 0x00F9B7 + , 0x00F9B8 + , 0x00F9B9 + , 0x00F9BA + , 0x00F9BB + , 0x00F9BC + , 0x00F9BD + , 0x00F9BE + , 0x00F9BF + , 0x00F9C0 + , 0x00F9C1 + , 0x00F9C2 + , 0x00F9C3 + , 0x00F9C4 + , 0x00F9C5 + , 0x00F9C6 + , 0x00F9C7 + , 0x00F9C8 + , 0x00F9C9 + , 0x00F9CA + , 0x00F9CB + , 0x00F9CC + , 0x00F9CD + , 0x00F9CE + , 0x00F9CF + , 0x00F9D0 + , 0x00F9D1 + , 0x00F9D2 + , 0x00F9D3 + , 0x00F9D4 + , 0x00F9D5 + , 0x00F9D6 + , 0x00F9D7 + , 0x00F9D8 + , 0x00F9D9 + , 0x00F9DA + , 0x00F9DB + , 0x00F9DC + , 0x00F9DD + , 0x00F9DE + , 0x00F9DF + , 0x00F9E0 + , 0x00F9E1 + , 0x00F9E2 + , 0x00F9E3 + , 0x00F9E4 + , 0x00F9E5 + , 0x00F9E6 + , 0x00F9E7 + , 0x00F9E8 + , 0x00F9E9 + , 0x00F9EA + , 0x00F9EB + , 0x00F9EC + , 0x00F9ED + , 0x00F9EE + , 0x00F9EF + , 0x00F9F0 + , 0x00F9F1 + , 0x00F9F2 + , 0x00F9F3 + , 0x00F9F4 + , 0x00F9F5 + , 0x00F9F6 + , 0x00F9F7 + , 0x00F9F8 + , 0x00F9F9 + , 0x00F9FA + , 0x00F9FB + , 0x00F9FC + , 0x00F9FD + , 0x00F9FE + , 0x00F9FF + , 0x00FA00 + , 0x00FA01 + , 0x00FA02 + , 0x00FA03 + , 0x00FA04 + , 0x00FA05 + , 0x00FA06 + , 0x00FA07 + , 0x00FA08 + , 0x00FA09 + , 0x00FA0A + , 0x00FA0B + , 0x00FA0C + , 0x00FA0D + , 0x00FA10 + , 0x00FA12 + , 0x00FA15 + , 0x00FA16 + , 0x00FA17 + , 0x00FA18 + , 0x00FA19 + , 0x00FA1A + , 0x00FA1B + , 0x00FA1C + , 0x00FA1D + , 0x00FA1E + , 0x00FA20 + , 0x00FA22 + , 0x00FA25 + , 0x00FA26 + , 0x00FA2A + , 0x00FA2B + , 0x00FA2C + , 0x00FA2D + , 0x00FA2E + , 0x00FA2F + , 0x00FA30 + , 0x00FA31 + , 0x00FA32 + , 0x00FA33 + , 0x00FA34 + , 0x00FA35 + , 0x00FA36 + , 0x00FA37 + , 0x00FA38 + , 0x00FA39 + , 0x00FA3A + , 0x00FA3B + , 0x00FA3C + , 0x00FA3D + , 0x00FA3E + , 0x00FA3F + , 0x00FA40 + , 0x00FA41 + , 0x00FA42 + , 0x00FA43 + , 0x00FA44 + , 0x00FA45 + , 0x00FA46 + , 0x00FA47 + , 0x00FA48 + , 0x00FA49 + , 0x00FA4A + , 0x00FA4B + , 0x00FA4C + , 0x00FA4D + , 0x00FA4E + , 0x00FA4F + , 0x00FA50 + , 0x00FA51 + , 0x00FA52 + , 0x00FA53 + , 0x00FA54 + , 0x00FA55 + , 0x00FA56 + , 0x00FA57 + , 0x00FA58 + , 0x00FA59 + , 0x00FA5A + , 0x00FA5B + , 0x00FA5C + , 0x00FA5D + , 0x00FA5E + , 0x00FA5F + , 0x00FA60 + , 0x00FA61 + , 0x00FA62 + , 0x00FA63 + , 0x00FA64 + , 0x00FA65 + , 0x00FA66 + , 0x00FA67 + , 0x00FA68 + , 0x00FA69 + , 0x00FA6A + , 0x00FA6B + , 0x00FA6C + , 0x00FA6D + , 0x00FA70 + , 0x00FA71 + , 0x00FA72 + , 0x00FA73 + , 0x00FA74 + , 0x00FA75 + , 0x00FA76 + , 0x00FA77 + , 0x00FA78 + , 0x00FA79 + , 0x00FA7A + , 0x00FA7B + , 0x00FA7C + , 0x00FA7D + , 0x00FA7E + , 0x00FA7F + , 0x00FA80 + , 0x00FA81 + , 0x00FA82 + , 0x00FA83 + , 0x00FA84 + , 0x00FA85 + , 0x00FA86 + , 0x00FA87 + , 0x00FA88 + , 0x00FA89 + , 0x00FA8A + , 0x00FA8B + , 0x00FA8C + , 0x00FA8D + , 0x00FA8E + , 0x00FA8F + , 0x00FA90 + , 0x00FA91 + , 0x00FA92 + , 0x00FA93 + , 0x00FA94 + , 0x00FA95 + , 0x00FA96 + , 0x00FA97 + , 0x00FA98 + , 0x00FA99 + , 0x00FA9A + , 0x00FA9B + , 0x00FA9C + , 0x00FA9D + , 0x00FA9E + , 0x00FA9F + , 0x00FAA0 + , 0x00FAA1 + , 0x00FAA2 + , 0x00FAA3 + , 0x00FAA4 + , 0x00FAA5 + , 0x00FAA6 + , 0x00FAA7 + , 0x00FAA8 + , 0x00FAA9 + , 0x00FAAA + , 0x00FAAB + , 0x00FAAC + , 0x00FAAD + , 0x00FAAE + , 0x00FAAF + , 0x00FAB0 + , 0x00FAB1 + , 0x00FAB2 + , 0x00FAB3 + , 0x00FAB4 + , 0x00FAB5 + , 0x00FAB6 + , 0x00FAB7 + , 0x00FAB8 + , 0x00FAB9 + , 0x00FABA + , 0x00FABB + , 0x00FABC + , 0x00FABD + , 0x00FABE + , 0x00FABF + , 0x00FAC0 + , 0x00FAC1 + , 0x00FAC2 + , 0x00FAC3 + , 0x00FAC4 + , 0x00FAC5 + , 0x00FAC6 + , 0x00FAC7 + , 0x00FAC8 + , 0x00FAC9 + , 0x00FACA + , 0x00FACB + , 0x00FACC + , 0x00FACD + , 0x00FACE + , 0x00FACF + , 0x00FAD0 + , 0x00FAD1 + , 0x00FAD2 + , 0x00FAD3 + , 0x00FAD4 + , 0x00FAD5 + , 0x00FAD6 + , 0x00FAD7 + , 0x00FAD8 + , 0x00FAD9 + , 0x00FB1D + , 0x00FB1F + , 0x00FB2A + , 0x00FB2B + , 0x00FB2C + , 0x00FB2D + , 0x00FB2E + , 0x00FB2F + , 0x00FB30 + , 0x00FB31 + , 0x00FB32 + , 0x00FB33 + , 0x00FB34 + , 0x00FB35 + , 0x00FB36 + , 0x00FB38 + , 0x00FB39 + , 0x00FB3A + , 0x00FB3B + , 0x00FB3C + , 0x00FB3E + , 0x00FB40 + , 0x00FB41 + , 0x00FB43 + , 0x00FB44 + , 0x00FB46 + , 0x00FB47 + , 0x00FB48 + , 0x00FB49 + , 0x00FB4A + , 0x00FB4B + , 0x00FB4C + , 0x00FB4D + , 0x00FB4E + , 0x0105C9 + , 0x0105E4 + , 0x01109A + , 0x01109C + , 0x0110AB + , 0x01112E + , 0x01112F + , 0x01134B + , 0x01134C + , 0x011383 + , 0x011385 + , 0x01138E + , 0x011391 + , 0x0113C5 + , 0x0113C7 + , 0x0113C8 + , 0x0114BB + , 0x0114BC + , 0x0114BE + , 0x0115BA + , 0x0115BB + , 0x011938 + , 0x016121 + , 0x016122 + , 0x016123 + , 0x016124 + , 0x016125 + , 0x016126 + , 0x016127 + , 0x016128 + , 0x016D68 + , 0x016D69 + , 0x016D6A + , 0x01D15E + , 0x01D15F + , 0x01D160 + , 0x01D161 + , 0x01D162 + , 0x01D163 + , 0x01D164 + , 0x01D1BB + , 0x01D1BC + , 0x01D1BD + , 0x01D1BE + , 0x01D1BF + , 0x01D1C0 + , 0x02F800 + , 0x02F801 + , 0x02F802 + , 0x02F803 + , 0x02F804 + , 0x02F805 + , 0x02F806 + , 0x02F807 + , 0x02F808 + , 0x02F809 + , 0x02F80A + , 0x02F80B + , 0x02F80C + , 0x02F80D + , 0x02F80E + , 0x02F80F + , 0x02F810 + , 0x02F811 + , 0x02F812 + , 0x02F813 + , 0x02F814 + , 0x02F815 + , 0x02F816 + , 0x02F817 + , 0x02F818 + , 0x02F819 + , 0x02F81A + , 0x02F81B + , 0x02F81C + , 0x02F81D + , 0x02F81E + , 0x02F81F + , 0x02F820 + , 0x02F821 + , 0x02F822 + , 0x02F823 + , 0x02F824 + , 0x02F825 + , 0x02F826 + , 0x02F827 + , 0x02F828 + , 0x02F829 + , 0x02F82A + , 0x02F82B + , 0x02F82C + , 0x02F82D + , 0x02F82E + , 0x02F82F + , 0x02F830 + , 0x02F831 + , 0x02F832 + , 0x02F833 + , 0x02F834 + , 0x02F835 + , 0x02F836 + , 0x02F837 + , 0x02F838 + , 0x02F839 + , 0x02F83A + , 0x02F83B + , 0x02F83C + , 0x02F83D + , 0x02F83E + , 0x02F83F + , 0x02F840 + , 0x02F841 + , 0x02F842 + , 0x02F843 + , 0x02F844 + , 0x02F845 + , 0x02F846 + , 0x02F847 + , 0x02F848 + , 0x02F849 + , 0x02F84A + , 0x02F84B + , 0x02F84C + , 0x02F84D + , 0x02F84E + , 0x02F84F + , 0x02F850 + , 0x02F851 + , 0x02F852 + , 0x02F853 + , 0x02F854 + , 0x02F855 + , 0x02F856 + , 0x02F857 + , 0x02F858 + , 0x02F859 + , 0x02F85A + , 0x02F85B + , 0x02F85C + , 0x02F85D + , 0x02F85E + , 0x02F85F + , 0x02F860 + , 0x02F861 + , 0x02F862 + , 0x02F863 + , 0x02F864 + , 0x02F865 + , 0x02F866 + , 0x02F867 + , 0x02F868 + , 0x02F869 + , 0x02F86A + , 0x02F86B + , 0x02F86C + , 0x02F86D + , 0x02F86E + , 0x02F86F + , 0x02F870 + , 0x02F871 + , 0x02F872 + , 0x02F873 + , 0x02F874 + , 0x02F875 + , 0x02F876 + , 0x02F877 + , 0x02F878 + , 0x02F879 + , 0x02F87A + , 0x02F87B + , 0x02F87C + , 0x02F87D + , 0x02F87E + , 0x02F87F + , 0x02F880 + , 0x02F881 + , 0x02F882 + , 0x02F883 + , 0x02F884 + , 0x02F885 + , 0x02F886 + , 0x02F887 + , 0x02F888 + , 0x02F889 + , 0x02F88A + , 0x02F88B + , 0x02F88C + , 0x02F88D + , 0x02F88E + , 0x02F88F + , 0x02F890 + , 0x02F891 + , 0x02F892 + , 0x02F893 + , 0x02F894 + , 0x02F895 + , 0x02F896 + , 0x02F897 + , 0x02F898 + , 0x02F899 + , 0x02F89A + , 0x02F89B + , 0x02F89C + , 0x02F89D + , 0x02F89E + , 0x02F89F + , 0x02F8A0 + , 0x02F8A1 + , 0x02F8A2 + , 0x02F8A3 + , 0x02F8A4 + , 0x02F8A5 + , 0x02F8A6 + , 0x02F8A7 + , 0x02F8A8 + , 0x02F8A9 + , 0x02F8AA + , 0x02F8AB + , 0x02F8AC + , 0x02F8AD + , 0x02F8AE + , 0x02F8AF + , 0x02F8B0 + , 0x02F8B1 + , 0x02F8B2 + , 0x02F8B3 + , 0x02F8B4 + , 0x02F8B5 + , 0x02F8B6 + , 0x02F8B7 + , 0x02F8B8 + , 0x02F8B9 + , 0x02F8BA + , 0x02F8BB + , 0x02F8BC + , 0x02F8BD + , 0x02F8BE + , 0x02F8BF + , 0x02F8C0 + , 0x02F8C1 + , 0x02F8C2 + , 0x02F8C3 + , 0x02F8C4 + , 0x02F8C5 + , 0x02F8C6 + , 0x02F8C7 + , 0x02F8C8 + , 0x02F8C9 + , 0x02F8CA + , 0x02F8CB + , 0x02F8CC + , 0x02F8CD + , 0x02F8CE + , 0x02F8CF + , 0x02F8D0 + , 0x02F8D1 + , 0x02F8D2 + , 0x02F8D3 + , 0x02F8D4 + , 0x02F8D5 + , 0x02F8D6 + , 0x02F8D7 + , 0x02F8D8 + , 0x02F8D9 + , 0x02F8DA + , 0x02F8DB + , 0x02F8DC + , 0x02F8DD + , 0x02F8DE + , 0x02F8DF + , 0x02F8E0 + , 0x02F8E1 + , 0x02F8E2 + , 0x02F8E3 + , 0x02F8E4 + , 0x02F8E5 + , 0x02F8E6 + , 0x02F8E7 + , 0x02F8E8 + , 0x02F8E9 + , 0x02F8EA + , 0x02F8EB + , 0x02F8EC + , 0x02F8ED + , 0x02F8EE + , 0x02F8EF + , 0x02F8F0 + , 0x02F8F1 + , 0x02F8F2 + , 0x02F8F3 + , 0x02F8F4 + , 0x02F8F5 + , 0x02F8F6 + , 0x02F8F7 + , 0x02F8F8 + , 0x02F8F9 + , 0x02F8FA + , 0x02F8FB + , 0x02F8FC + , 0x02F8FD + , 0x02F8FE + , 0x02F8FF + , 0x02F900 + , 0x02F901 + , 0x02F902 + , 0x02F903 + , 0x02F904 + , 0x02F905 + , 0x02F906 + , 0x02F907 + , 0x02F908 + , 0x02F909 + , 0x02F90A + , 0x02F90B + , 0x02F90C + , 0x02F90D + , 0x02F90E + , 0x02F90F + , 0x02F910 + , 0x02F911 + , 0x02F912 + , 0x02F913 + , 0x02F914 + , 0x02F915 + , 0x02F916 + , 0x02F917 + , 0x02F918 + , 0x02F919 + , 0x02F91A + , 0x02F91B + , 0x02F91C + , 0x02F91D + , 0x02F91E + , 0x02F91F + , 0x02F920 + , 0x02F921 + , 0x02F922 + , 0x02F923 + , 0x02F924 + , 0x02F925 + , 0x02F926 + , 0x02F927 + , 0x02F928 + , 0x02F929 + , 0x02F92A + , 0x02F92B + , 0x02F92C + , 0x02F92D + , 0x02F92E + , 0x02F92F + , 0x02F930 + , 0x02F931 + , 0x02F932 + , 0x02F933 + , 0x02F934 + , 0x02F935 + , 0x02F936 + , 0x02F937 + , 0x02F938 + , 0x02F939 + , 0x02F93A + , 0x02F93B + , 0x02F93C + , 0x02F93D + , 0x02F93E + , 0x02F93F + , 0x02F940 + , 0x02F941 + , 0x02F942 + , 0x02F943 + , 0x02F944 + , 0x02F945 + , 0x02F946 + , 0x02F947 + , 0x02F948 + , 0x02F949 + , 0x02F94A + , 0x02F94B + , 0x02F94C + , 0x02F94D + , 0x02F94E + , 0x02F94F + , 0x02F950 + , 0x02F951 + , 0x02F952 + , 0x02F953 + , 0x02F954 + , 0x02F955 + , 0x02F956 + , 0x02F957 + , 0x02F958 + , 0x02F959 + , 0x02F95A + , 0x02F95B + , 0x02F95C + , 0x02F95D + , 0x02F95E + , 0x02F95F + , 0x02F960 + , 0x02F961 + , 0x02F962 + , 0x02F963 + , 0x02F964 + , 0x02F965 + , 0x02F966 + , 0x02F967 + , 0x02F968 + , 0x02F969 + , 0x02F96A + , 0x02F96B + , 0x02F96C + , 0x02F96D + , 0x02F96E + , 0x02F96F + , 0x02F970 + , 0x02F971 + , 0x02F972 + , 0x02F973 + , 0x02F974 + , 0x02F975 + , 0x02F976 + , 0x02F977 + , 0x02F978 + , 0x02F979 + , 0x02F97A + , 0x02F97B + , 0x02F97C + , 0x02F97D + , 0x02F97E + , 0x02F97F + , 0x02F980 + , 0x02F981 + , 0x02F982 + , 0x02F983 + , 0x02F984 + , 0x02F985 + , 0x02F986 + , 0x02F987 + , 0x02F988 + , 0x02F989 + , 0x02F98A + , 0x02F98B + , 0x02F98C + , 0x02F98D + , 0x02F98E + , 0x02F98F + , 0x02F990 + , 0x02F991 + , 0x02F992 + , 0x02F993 + , 0x02F994 + , 0x02F995 + , 0x02F996 + , 0x02F997 + , 0x02F998 + , 0x02F999 + , 0x02F99A + , 0x02F99B + , 0x02F99C + , 0x02F99D + , 0x02F99E + , 0x02F99F + , 0x02F9A0 + , 0x02F9A1 + , 0x02F9A2 + , 0x02F9A3 + , 0x02F9A4 + , 0x02F9A5 + , 0x02F9A6 + , 0x02F9A7 + , 0x02F9A8 + , 0x02F9A9 + , 0x02F9AA + , 0x02F9AB + , 0x02F9AC + , 0x02F9AD + , 0x02F9AE + , 0x02F9AF + , 0x02F9B0 + , 0x02F9B1 + , 0x02F9B2 + , 0x02F9B3 + , 0x02F9B4 + , 0x02F9B5 + , 0x02F9B6 + , 0x02F9B7 + , 0x02F9B8 + , 0x02F9B9 + , 0x02F9BA + , 0x02F9BB + , 0x02F9BC + , 0x02F9BD + , 0x02F9BE + , 0x02F9BF + , 0x02F9C0 + , 0x02F9C1 + , 0x02F9C2 + , 0x02F9C3 + , 0x02F9C4 + , 0x02F9C5 + , 0x02F9C6 + , 0x02F9C7 + , 0x02F9C8 + , 0x02F9C9 + , 0x02F9CA + , 0x02F9CB + , 0x02F9CC + , 0x02F9CD + , 0x02F9CE + , 0x02F9CF + , 0x02F9D0 + , 0x02F9D1 + , 0x02F9D2 + , 0x02F9D3 + , 0x02F9D4 + , 0x02F9D5 + , 0x02F9D6 + , 0x02F9D7 + , 0x02F9D8 + , 0x02F9D9 + , 0x02F9DA + , 0x02F9DB + , 0x02F9DC + , 0x02F9DD + , 0x02F9DE + , 0x02F9DF + , 0x02F9E0 + , 0x02F9E1 + , 0x02F9E2 + , 0x02F9E3 + , 0x02F9E4 + , 0x02F9E5 + , 0x02F9E6 + , 0x02F9E7 + , 0x02F9E8 + , 0x02F9E9 + , 0x02F9EA + , 0x02F9EB + , 0x02F9EC + , 0x02F9ED + , 0x02F9EE + , 0x02F9EF + , 0x02F9F0 + , 0x02F9F1 + , 0x02F9F2 + , 0x02F9F3 + , 0x02F9F4 + , 0x02F9F5 + , 0x02F9F6 + , 0x02F9F7 + , 0x02F9F8 + , 0x02F9F9 + , 0x02F9FA + , 0x02F9FB + , 0x02F9FC + , 0x02F9FD + , 0x02F9FE + , 0x02F9FF + , 0x02FA00 + , 0x02FA01 + , 0x02FA02 + , 0x02FA03 + , 0x02FA04 + , 0x02FA05 + , 0x02FA06 + , 0x02FA07 + , 0x02FA08 + , 0x02FA09 + , 0x02FA0A + , 0x02FA0B + , 0x02FA0C + , 0x02FA0D + , 0x02FA0E + , 0x02FA0F + , 0x02FA10 + , 0x02FA11 + , 0x02FA12 + , 0x02FA13 + , 0x02FA14 + , 0x02FA15 + , 0x02FA16 + , 0x02FA17 + , 0x02FA18 + , 0x02FA19 + , 0x02FA1A + , 0x02FA1B + , 0x02FA1C + , 0x02FA1D + ] + +decompFirst :: ByteArray +decompFirst = PBA.byteArrayFromList + [ (0x000041 :: Word32) + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000043 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x00004E + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000059 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000063 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x00006E + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000079 + , 0x000079 + , 0x000041 + , 0x000061 + , 0x000041 + , 0x000061 + , 0x000041 + , 0x000061 + , 0x000043 + , 0x000063 + , 0x000043 + , 0x000063 + , 0x000043 + , 0x000063 + , 0x000043 + , 0x000063 + , 0x000044 + , 0x000064 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000047 + , 0x000067 + , 0x000047 + , 0x000067 + , 0x000047 + , 0x000067 + , 0x000047 + , 0x000067 + , 0x000048 + , 0x000068 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x00004A + , 0x00006A + , 0x00004B + , 0x00006B + , 0x00004C + , 0x00006C + , 0x00004C + , 0x00006C + , 0x00004C + , 0x00006C + , 0x00004E + , 0x00006E + , 0x00004E + , 0x00006E + , 0x00004E + , 0x00006E + , 0x00004F + , 0x00006F + , 0x00004F + , 0x00006F + , 0x00004F + , 0x00006F + , 0x000052 + , 0x000072 + , 0x000052 + , 0x000072 + , 0x000052 + , 0x000072 + , 0x000053 + , 0x000073 + , 0x000053 + , 0x000073 + , 0x000053 + , 0x000073 + , 0x000053 + , 0x000073 + , 0x000054 + , 0x000074 + , 0x000054 + , 0x000074 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000057 + , 0x000077 + , 0x000059 + , 0x000079 + , 0x000059 + , 0x00005A + , 0x00007A + , 0x00005A + , 0x00007A + , 0x00005A + , 0x00007A + , 0x00004F + , 0x00006F + , 0x000055 + , 0x000075 + , 0x000041 + , 0x000061 + , 0x000049 + , 0x000069 + , 0x00004F + , 0x00006F + , 0x000055 + , 0x000075 + , 0x0000DC + , 0x0000FC + , 0x0000DC + , 0x0000FC + , 0x0000DC + , 0x0000FC + , 0x0000DC + , 0x0000FC + , 0x0000C4 + , 0x0000E4 + , 0x000226 + , 0x000227 + , 0x0000C6 + , 0x0000E6 + , 0x000047 + , 0x000067 + , 0x00004B + , 0x00006B + , 0x00004F + , 0x00006F + , 0x0001EA + , 0x0001EB + , 0x0001B7 + , 0x000292 + , 0x00006A + , 0x000047 + , 0x000067 + , 0x00004E + , 0x00006E + , 0x0000C5 + , 0x0000E5 + , 0x0000C6 + , 0x0000E6 + , 0x0000D8 + , 0x0000F8 + , 0x000041 + , 0x000061 + , 0x000041 + , 0x000061 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x000069 + , 0x00004F + , 0x00006F + , 0x00004F + , 0x00006F + , 0x000052 + , 0x000072 + , 0x000052 + , 0x000072 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000053 + , 0x000073 + , 0x000054 + , 0x000074 + , 0x000048 + , 0x000068 + , 0x000041 + , 0x000061 + , 0x000045 + , 0x000065 + , 0x0000D6 + , 0x0000F6 + , 0x0000D5 + , 0x0000F5 + , 0x00004F + , 0x00006F + , 0x00022E + , 0x00022F + , 0x000059 + , 0x000079 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000308 + , 0x0002B9 + , 0x00003B + , 0x0000A8 + , 0x000391 + , 0x0000B7 + , 0x000395 + , 0x000397 + , 0x000399 + , 0x00039F + , 0x0003A5 + , 0x0003A9 + , 0x0003CA + , 0x000399 + , 0x0003A5 + , 0x0003B1 + , 0x0003B5 + , 0x0003B7 + , 0x0003B9 + , 0x0003CB + , 0x0003B9 + , 0x0003C5 + , 0x0003BF + , 0x0003C5 + , 0x0003C9 + , 0x0003D2 + , 0x0003D2 + , 0x000415 + , 0x000415 + , 0x000413 + , 0x000406 + , 0x00041A + , 0x000418 + , 0x000423 + , 0x000418 + , 0x000438 + , 0x000435 + , 0x000435 + , 0x000433 + , 0x000456 + , 0x00043A + , 0x000438 + , 0x000443 + , 0x000474 + , 0x000475 + , 0x000416 + , 0x000436 + , 0x000410 + , 0x000430 + , 0x000410 + , 0x000430 + , 0x000415 + , 0x000435 + , 0x0004D8 + , 0x0004D9 + , 0x000416 + , 0x000436 + , 0x000417 + , 0x000437 + , 0x000418 + , 0x000438 + , 0x000418 + , 0x000438 + , 0x00041E + , 0x00043E + , 0x0004E8 + , 0x0004E9 + , 0x00042D + , 0x00044D + , 0x000423 + , 0x000443 + , 0x000423 + , 0x000443 + , 0x000423 + , 0x000443 + , 0x000427 + , 0x000447 + , 0x00042B + , 0x00044B + , 0x000627 + , 0x000627 + , 0x000648 + , 0x000627 + , 0x00064A + , 0x0006D5 + , 0x0006C1 + , 0x0006D2 + , 0x000928 + , 0x000930 + , 0x000933 + , 0x000915 + , 0x000916 + , 0x000917 + , 0x00091C + , 0x000921 + , 0x000922 + , 0x00092B + , 0x00092F + , 0x0009C7 + , 0x0009C7 + , 0x0009A1 + , 0x0009A2 + , 0x0009AF + , 0x000A32 + , 0x000A38 + , 0x000A16 + , 0x000A17 + , 0x000A1C + , 0x000A2B + , 0x000B47 + , 0x000B47 + , 0x000B47 + , 0x000B21 + , 0x000B22 + , 0x000B92 + , 0x000BC6 + , 0x000BC7 + , 0x000BC6 + , 0x000C46 + , 0x000CBF + , 0x000CC6 + , 0x000CC6 + , 0x000CC6 + , 0x000CCA + , 0x000D46 + , 0x000D47 + , 0x000D46 + , 0x000DD9 + , 0x000DD9 + , 0x000DDC + , 0x000DD9 + , 0x000F42 + , 0x000F4C + , 0x000F51 + , 0x000F56 + , 0x000F5B + , 0x000F40 + , 0x000F71 + , 0x000F71 + , 0x000FB2 + , 0x000FB3 + , 0x000F71 + , 0x000F92 + , 0x000F9C + , 0x000FA1 + , 0x000FA6 + , 0x000FAB + , 0x000F90 + , 0x001025 + , 0x001B05 + , 0x001B07 + , 0x001B09 + , 0x001B0B + , 0x001B0D + , 0x001B11 + , 0x001B3A + , 0x001B3C + , 0x001B3E + , 0x001B3F + , 0x001B42 + , 0x000041 + , 0x000061 + , 0x000042 + , 0x000062 + , 0x000042 + , 0x000062 + , 0x000042 + , 0x000062 + , 0x0000C7 + , 0x0000E7 + , 0x000044 + , 0x000064 + , 0x000044 + , 0x000064 + , 0x000044 + , 0x000064 + , 0x000044 + , 0x000064 + , 0x000044 + , 0x000064 + , 0x000112 + , 0x000113 + , 0x000112 + , 0x000113 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000228 + , 0x000229 + , 0x000046 + , 0x000066 + , 0x000047 + , 0x000067 + , 0x000048 + , 0x000068 + , 0x000048 + , 0x000068 + , 0x000048 + , 0x000068 + , 0x000048 + , 0x000068 + , 0x000048 + , 0x000068 + , 0x000049 + , 0x000069 + , 0x0000CF + , 0x0000EF + , 0x00004B + , 0x00006B + , 0x00004B + , 0x00006B + , 0x00004B + , 0x00006B + , 0x00004C + , 0x00006C + , 0x001E36 + , 0x001E37 + , 0x00004C + , 0x00006C + , 0x00004C + , 0x00006C + , 0x00004D + , 0x00006D + , 0x00004D + , 0x00006D + , 0x00004D + , 0x00006D + , 0x00004E + , 0x00006E + , 0x00004E + , 0x00006E + , 0x00004E + , 0x00006E + , 0x00004E + , 0x00006E + , 0x0000D5 + , 0x0000F5 + , 0x0000D5 + , 0x0000F5 + , 0x00014C + , 0x00014D + , 0x00014C + , 0x00014D + , 0x000050 + , 0x000070 + , 0x000050 + , 0x000070 + , 0x000052 + , 0x000072 + , 0x000052 + , 0x000072 + , 0x001E5A + , 0x001E5B + , 0x000052 + , 0x000072 + , 0x000053 + , 0x000073 + , 0x000053 + , 0x000073 + , 0x00015A + , 0x00015B + , 0x000160 + , 0x000161 + , 0x001E62 + , 0x001E63 + , 0x000054 + , 0x000074 + , 0x000054 + , 0x000074 + , 0x000054 + , 0x000074 + , 0x000054 + , 0x000074 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x000168 + , 0x000169 + , 0x00016A + , 0x00016B + , 0x000056 + , 0x000076 + , 0x000056 + , 0x000076 + , 0x000057 + , 0x000077 + , 0x000057 + , 0x000077 + , 0x000057 + , 0x000077 + , 0x000057 + , 0x000077 + , 0x000057 + , 0x000077 + , 0x000058 + , 0x000078 + , 0x000058 + , 0x000078 + , 0x000059 + , 0x000079 + , 0x00005A + , 0x00007A + , 0x00005A + , 0x00007A + , 0x00005A + , 0x00007A + , 0x000068 + , 0x000074 + , 0x000077 + , 0x000079 + , 0x00017F + , 0x000041 + , 0x000061 + , 0x000041 + , 0x000061 + , 0x0000C2 + , 0x0000E2 + , 0x0000C2 + , 0x0000E2 + , 0x0000C2 + , 0x0000E2 + , 0x0000C2 + , 0x0000E2 + , 0x001EA0 + , 0x001EA1 + , 0x000102 + , 0x000103 + , 0x000102 + , 0x000103 + , 0x000102 + , 0x000103 + , 0x000102 + , 0x000103 + , 0x001EA0 + , 0x001EA1 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x000045 + , 0x000065 + , 0x0000CA + , 0x0000EA + , 0x0000CA + , 0x0000EA + , 0x0000CA + , 0x0000EA + , 0x0000CA + , 0x0000EA + , 0x001EB8 + , 0x001EB9 + , 0x000049 + , 0x000069 + , 0x000049 + , 0x000069 + , 0x00004F + , 0x00006F + , 0x00004F + , 0x00006F + , 0x0000D4 + , 0x0000F4 + , 0x0000D4 + , 0x0000F4 + , 0x0000D4 + , 0x0000F4 + , 0x0000D4 + , 0x0000F4 + , 0x001ECC + , 0x001ECD + , 0x0001A0 + , 0x0001A1 + , 0x0001A0 + , 0x0001A1 + , 0x0001A0 + , 0x0001A1 + , 0x0001A0 + , 0x0001A1 + , 0x0001A0 + , 0x0001A1 + , 0x000055 + , 0x000075 + , 0x000055 + , 0x000075 + , 0x0001AF + , 0x0001B0 + , 0x0001AF + , 0x0001B0 + , 0x0001AF + , 0x0001B0 + , 0x0001AF + , 0x0001B0 + , 0x0001AF + , 0x0001B0 + , 0x000059 + , 0x000079 + , 0x000059 + , 0x000079 + , 0x000059 + , 0x000079 + , 0x000059 + , 0x000079 + , 0x0003B1 + , 0x0003B1 + , 0x001F00 + , 0x001F01 + , 0x001F00 + , 0x001F01 + , 0x001F00 + , 0x001F01 + , 0x000391 + , 0x000391 + , 0x001F08 + , 0x001F09 + , 0x001F08 + , 0x001F09 + , 0x001F08 + , 0x001F09 + , 0x0003B5 + , 0x0003B5 + , 0x001F10 + , 0x001F11 + , 0x001F10 + , 0x001F11 + , 0x000395 + , 0x000395 + , 0x001F18 + , 0x001F19 + , 0x001F18 + , 0x001F19 + , 0x0003B7 + , 0x0003B7 + , 0x001F20 + , 0x001F21 + , 0x001F20 + , 0x001F21 + , 0x001F20 + , 0x001F21 + , 0x000397 + , 0x000397 + , 0x001F28 + , 0x001F29 + , 0x001F28 + , 0x001F29 + , 0x001F28 + , 0x001F29 + , 0x0003B9 + , 0x0003B9 + , 0x001F30 + , 0x001F31 + , 0x001F30 + , 0x001F31 + , 0x001F30 + , 0x001F31 + , 0x000399 + , 0x000399 + , 0x001F38 + , 0x001F39 + , 0x001F38 + , 0x001F39 + , 0x001F38 + , 0x001F39 + , 0x0003BF + , 0x0003BF + , 0x001F40 + , 0x001F41 + , 0x001F40 + , 0x001F41 + , 0x00039F + , 0x00039F + , 0x001F48 + , 0x001F49 + , 0x001F48 + , 0x001F49 + , 0x0003C5 + , 0x0003C5 + , 0x001F50 + , 0x001F51 + , 0x001F50 + , 0x001F51 + , 0x001F50 + , 0x001F51 + , 0x0003A5 + , 0x001F59 + , 0x001F59 + , 0x001F59 + , 0x0003C9 + , 0x0003C9 + , 0x001F60 + , 0x001F61 + , 0x001F60 + , 0x001F61 + , 0x001F60 + , 0x001F61 + , 0x0003A9 + , 0x0003A9 + , 0x001F68 + , 0x001F69 + , 0x001F68 + , 0x001F69 + , 0x001F68 + , 0x001F69 + , 0x0003B1 + , 0x0003AC + , 0x0003B5 + , 0x0003AD + , 0x0003B7 + , 0x0003AE + , 0x0003B9 + , 0x0003AF + , 0x0003BF + , 0x0003CC + , 0x0003C5 + , 0x0003CD + , 0x0003C9 + , 0x0003CE + , 0x001F00 + , 0x001F01 + , 0x001F02 + , 0x001F03 + , 0x001F04 + , 0x001F05 + , 0x001F06 + , 0x001F07 + , 0x001F08 + , 0x001F09 + , 0x001F0A + , 0x001F0B + , 0x001F0C + , 0x001F0D + , 0x001F0E + , 0x001F0F + , 0x001F20 + , 0x001F21 + , 0x001F22 + , 0x001F23 + , 0x001F24 + , 0x001F25 + , 0x001F26 + , 0x001F27 + , 0x001F28 + , 0x001F29 + , 0x001F2A + , 0x001F2B + , 0x001F2C + , 0x001F2D + , 0x001F2E + , 0x001F2F + , 0x001F60 + , 0x001F61 + , 0x001F62 + , 0x001F63 + , 0x001F64 + , 0x001F65 + , 0x001F66 + , 0x001F67 + , 0x001F68 + , 0x001F69 + , 0x001F6A + , 0x001F6B + , 0x001F6C + , 0x001F6D + , 0x001F6E + , 0x001F6F + , 0x0003B1 + , 0x0003B1 + , 0x001F70 + , 0x0003B1 + , 0x0003AC + , 0x0003B1 + , 0x001FB6 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000386 + , 0x000391 + , 0x0003B9 + , 0x0000A8 + , 0x001F74 + , 0x0003B7 + , 0x0003AE + , 0x0003B7 + , 0x001FC6 + , 0x000395 + , 0x000388 + , 0x000397 + , 0x000389 + , 0x000397 + , 0x001FBF + , 0x001FBF + , 0x001FBF + , 0x0003B9 + , 0x0003B9 + , 0x0003CA + , 0x000390 + , 0x0003B9 + , 0x0003CA + , 0x000399 + , 0x000399 + , 0x000399 + , 0x00038A + , 0x001FFE + , 0x001FFE + , 0x001FFE + , 0x0003C5 + , 0x0003C5 + , 0x0003CB + , 0x0003B0 + , 0x0003C1 + , 0x0003C1 + , 0x0003C5 + , 0x0003CB + , 0x0003A5 + , 0x0003A5 + , 0x0003A5 + , 0x00038E + , 0x0003A1 + , 0x0000A8 + , 0x000385 + , 0x000060 + , 0x001F7C + , 0x0003C9 + , 0x0003CE + , 0x0003C9 + , 0x001FF6 + , 0x00039F + , 0x00038C + , 0x0003A9 + , 0x00038F + , 0x0003A9 + , 0x0000B4 + , 0x002002 + , 0x002003 + , 0x0003A9 + , 0x00004B + , 0x0000C5 + , 0x002190 + , 0x002192 + , 0x002194 + , 0x0021D0 + , 0x0021D4 + , 0x0021D2 + , 0x002203 + , 0x002208 + , 0x00220B + , 0x002223 + , 0x002225 + , 0x00223C + , 0x002243 + , 0x002245 + , 0x002248 + , 0x00003D + , 0x002261 + , 0x00224D + , 0x00003C + , 0x00003E + , 0x002264 + , 0x002265 + , 0x002272 + , 0x002273 + , 0x002276 + , 0x002277 + , 0x00227A + , 0x00227B + , 0x002282 + , 0x002283 + , 0x002286 + , 0x002287 + , 0x0022A2 + , 0x0022A8 + , 0x0022A9 + , 0x0022AB + , 0x00227C + , 0x00227D + , 0x002291 + , 0x002292 + , 0x0022B2 + , 0x0022B3 + , 0x0022B4 + , 0x0022B5 + , 0x003008 + , 0x003009 + , 0x002ADD + , 0x00304B + , 0x00304D + , 0x00304F + , 0x003051 + , 0x003053 + , 0x003055 + , 0x003057 + , 0x003059 + , 0x00305B + , 0x00305D + , 0x00305F + , 0x003061 + , 0x003064 + , 0x003066 + , 0x003068 + , 0x00306F + , 0x00306F + , 0x003072 + , 0x003072 + , 0x003075 + , 0x003075 + , 0x003078 + , 0x003078 + , 0x00307B + , 0x00307B + , 0x003046 + , 0x00309D + , 0x0030AB + , 0x0030AD + , 0x0030AF + , 0x0030B1 + , 0x0030B3 + , 0x0030B5 + , 0x0030B7 + , 0x0030B9 + , 0x0030BB + , 0x0030BD + , 0x0030BF + , 0x0030C1 + , 0x0030C4 + , 0x0030C6 + , 0x0030C8 + , 0x0030CF + , 0x0030CF + , 0x0030D2 + , 0x0030D2 + , 0x0030D5 + , 0x0030D5 + , 0x0030D8 + , 0x0030D8 + , 0x0030DB + , 0x0030DB + , 0x0030A6 + , 0x0030EF + , 0x0030F0 + , 0x0030F1 + , 0x0030F2 + , 0x0030FD + , 0x008C48 + , 0x0066F4 + , 0x008ECA + , 0x008CC8 + , 0x006ED1 + , 0x004E32 + , 0x0053E5 + , 0x009F9C + , 0x009F9C + , 0x005951 + , 0x0091D1 + , 0x005587 + , 0x005948 + , 0x0061F6 + , 0x007669 + , 0x007F85 + , 0x00863F + , 0x0087BA + , 0x0088F8 + , 0x00908F + , 0x006A02 + , 0x006D1B + , 0x0070D9 + , 0x0073DE + , 0x00843D + , 0x00916A + , 0x0099F1 + , 0x004E82 + , 0x005375 + , 0x006B04 + , 0x00721B + , 0x00862D + , 0x009E1E + , 0x005D50 + , 0x006FEB + , 0x0085CD + , 0x008964 + , 0x0062C9 + , 0x0081D8 + , 0x00881F + , 0x005ECA + , 0x006717 + , 0x006D6A + , 0x0072FC + , 0x0090CE + , 0x004F86 + , 0x0051B7 + , 0x0052DE + , 0x0064C4 + , 0x006AD3 + , 0x007210 + , 0x0076E7 + , 0x008001 + , 0x008606 + , 0x00865C + , 0x008DEF + , 0x009732 + , 0x009B6F + , 0x009DFA + , 0x00788C + , 0x00797F + , 0x007DA0 + , 0x0083C9 + , 0x009304 + , 0x009E7F + , 0x008AD6 + , 0x0058DF + , 0x005F04 + , 0x007C60 + , 0x00807E + , 0x007262 + , 0x0078CA + , 0x008CC2 + , 0x0096F7 + , 0x0058D8 + , 0x005C62 + , 0x006A13 + , 0x006DDA + , 0x006F0F + , 0x007D2F + , 0x007E37 + , 0x00964B + , 0x0052D2 + , 0x00808B + , 0x0051DC + , 0x0051CC + , 0x007A1C + , 0x007DBE + , 0x0083F1 + , 0x009675 + , 0x008B80 + , 0x0062CF + , 0x006A02 + , 0x008AFE + , 0x004E39 + , 0x005BE7 + , 0x006012 + , 0x007387 + , 0x007570 + , 0x005317 + , 0x0078FB + , 0x004FBF + , 0x005FA9 + , 0x004E0D + , 0x006CCC + , 0x006578 + , 0x007D22 + , 0x0053C3 + , 0x00585E + , 0x007701 + , 0x008449 + , 0x008AAA + , 0x006BBA + , 0x008FB0 + , 0x006C88 + , 0x0062FE + , 0x0082E5 + , 0x0063A0 + , 0x007565 + , 0x004EAE + , 0x005169 + , 0x0051C9 + , 0x006881 + , 0x007CE7 + , 0x00826F + , 0x008AD2 + , 0x0091CF + , 0x0052F5 + , 0x005442 + , 0x005973 + , 0x005EEC + , 0x0065C5 + , 0x006FFE + , 0x00792A + , 0x0095AD + , 0x009A6A + , 0x009E97 + , 0x009ECE + , 0x00529B + , 0x0066C6 + , 0x006B77 + , 0x008F62 + , 0x005E74 + , 0x006190 + , 0x006200 + , 0x00649A + , 0x006F23 + , 0x007149 + , 0x007489 + , 0x0079CA + , 0x007DF4 + , 0x00806F + , 0x008F26 + , 0x0084EE + , 0x009023 + , 0x00934A + , 0x005217 + , 0x0052A3 + , 0x0054BD + , 0x0070C8 + , 0x0088C2 + , 0x008AAA + , 0x005EC9 + , 0x005FF5 + , 0x00637B + , 0x006BAE + , 0x007C3E + , 0x007375 + , 0x004EE4 + , 0x0056F9 + , 0x005BE7 + , 0x005DBA + , 0x00601C + , 0x0073B2 + , 0x007469 + , 0x007F9A + , 0x008046 + , 0x009234 + , 0x0096F6 + , 0x009748 + , 0x009818 + , 0x004F8B + , 0x0079AE + , 0x0091B4 + , 0x0096B8 + , 0x0060E1 + , 0x004E86 + , 0x0050DA + , 0x005BEE + , 0x005C3F + , 0x006599 + , 0x006A02 + , 0x0071CE + , 0x007642 + , 0x0084FC + , 0x00907C + , 0x009F8D + , 0x006688 + , 0x00962E + , 0x005289 + , 0x00677B + , 0x0067F3 + , 0x006D41 + , 0x006E9C + , 0x007409 + , 0x007559 + , 0x00786B + , 0x007D10 + , 0x00985E + , 0x00516D + , 0x00622E + , 0x009678 + , 0x00502B + , 0x005D19 + , 0x006DEA + , 0x008F2A + , 0x005F8B + , 0x006144 + , 0x006817 + , 0x007387 + , 0x009686 + , 0x005229 + , 0x00540F + , 0x005C65 + , 0x006613 + , 0x00674E + , 0x0068A8 + , 0x006CE5 + , 0x007406 + , 0x0075E2 + , 0x007F79 + , 0x0088CF + , 0x0088E1 + , 0x0091CC + , 0x0096E2 + , 0x00533F + , 0x006EBA + , 0x00541D + , 0x0071D0 + , 0x007498 + , 0x0085FA + , 0x0096A3 + , 0x009C57 + , 0x009E9F + , 0x006797 + , 0x006DCB + , 0x0081E8 + , 0x007ACB + , 0x007B20 + , 0x007C92 + , 0x0072C0 + , 0x007099 + , 0x008B58 + , 0x004EC0 + , 0x008336 + , 0x00523A + , 0x005207 + , 0x005EA6 + , 0x0062D3 + , 0x007CD6 + , 0x005B85 + , 0x006D1E + , 0x0066B4 + , 0x008F3B + , 0x00884C + , 0x00964D + , 0x00898B + , 0x005ED3 + , 0x005140 + , 0x0055C0 + , 0x00585A + , 0x006674 + , 0x0051DE + , 0x00732A + , 0x0076CA + , 0x00793C + , 0x00795E + , 0x007965 + , 0x00798F + , 0x009756 + , 0x007CBE + , 0x007FBD + , 0x008612 + , 0x008AF8 + , 0x009038 + , 0x0090FD + , 0x0098EF + , 0x0098FC + , 0x009928 + , 0x009DB4 + , 0x0090DE + , 0x0096B7 + , 0x004FAE + , 0x0050E7 + , 0x00514D + , 0x0052C9 + , 0x0052E4 + , 0x005351 + , 0x00559D + , 0x005606 + , 0x005668 + , 0x005840 + , 0x0058A8 + , 0x005C64 + , 0x005C6E + , 0x006094 + , 0x006168 + , 0x00618E + , 0x0061F2 + , 0x00654F + , 0x0065E2 + , 0x006691 + , 0x006885 + , 0x006D77 + , 0x006E1A + , 0x006F22 + , 0x00716E + , 0x00722B + , 0x007422 + , 0x007891 + , 0x00793E + , 0x007949 + , 0x007948 + , 0x007950 + , 0x007956 + , 0x00795D + , 0x00798D + , 0x00798E + , 0x007A40 + , 0x007A81 + , 0x007BC0 + , 0x007DF4 + , 0x007E09 + , 0x007E41 + , 0x007F72 + , 0x008005 + , 0x0081ED + , 0x008279 + , 0x008279 + , 0x008457 + , 0x008910 + , 0x008996 + , 0x008B01 + , 0x008B39 + , 0x008CD3 + , 0x008D08 + , 0x008FB6 + , 0x009038 + , 0x0096E3 + , 0x0097FF + , 0x00983B + , 0x006075 + , 0x0242EE + , 0x008218 + , 0x004E26 + , 0x0051B5 + , 0x005168 + , 0x004F80 + , 0x005145 + , 0x005180 + , 0x0052C7 + , 0x0052FA + , 0x00559D + , 0x005555 + , 0x005599 + , 0x0055E2 + , 0x00585A + , 0x0058B3 + , 0x005944 + , 0x005954 + , 0x005A62 + , 0x005B28 + , 0x005ED2 + , 0x005ED9 + , 0x005F69 + , 0x005FAD + , 0x0060D8 + , 0x00614E + , 0x006108 + , 0x00618E + , 0x006160 + , 0x0061F2 + , 0x006234 + , 0x0063C4 + , 0x00641C + , 0x006452 + , 0x006556 + , 0x006674 + , 0x006717 + , 0x00671B + , 0x006756 + , 0x006B79 + , 0x006BBA + , 0x006D41 + , 0x006EDB + , 0x006ECB + , 0x006F22 + , 0x00701E + , 0x00716E + , 0x0077A7 + , 0x007235 + , 0x0072AF + , 0x00732A + , 0x007471 + , 0x007506 + , 0x00753B + , 0x00761D + , 0x00761F + , 0x0076CA + , 0x0076DB + , 0x0076F4 + , 0x00774A + , 0x007740 + , 0x0078CC + , 0x007AB1 + , 0x007BC0 + , 0x007C7B + , 0x007D5B + , 0x007DF4 + , 0x007F3E + , 0x008005 + , 0x008352 + , 0x0083EF + , 0x008779 + , 0x008941 + , 0x008986 + , 0x008996 + , 0x008ABF + , 0x008AF8 + , 0x008ACB + , 0x008B01 + , 0x008AFE + , 0x008AED + , 0x008B39 + , 0x008B8A + , 0x008D08 + , 0x008F38 + , 0x009072 + , 0x009199 + , 0x009276 + , 0x00967C + , 0x0096E3 + , 0x009756 + , 0x0097DB + , 0x0097FF + , 0x00980B + , 0x00983B + , 0x009B12 + , 0x009F9C + , 0x02284A + , 0x022844 + , 0x0233D5 + , 0x003B9D + , 0x004018 + , 0x004039 + , 0x025249 + , 0x025CD0 + , 0x027ED3 + , 0x009F43 + , 0x009F8E + , 0x0005D9 + , 0x0005F2 + , 0x0005E9 + , 0x0005E9 + , 0x00FB49 + , 0x00FB49 + , 0x0005D0 + , 0x0005D0 + , 0x0005D0 + , 0x0005D1 + , 0x0005D2 + , 0x0005D3 + , 0x0005D4 + , 0x0005D5 + , 0x0005D6 + , 0x0005D8 + , 0x0005D9 + , 0x0005DA + , 0x0005DB + , 0x0005DC + , 0x0005DE + , 0x0005E0 + , 0x0005E1 + , 0x0005E3 + , 0x0005E4 + , 0x0005E6 + , 0x0005E7 + , 0x0005E8 + , 0x0005E9 + , 0x0005EA + , 0x0005D5 + , 0x0005D1 + , 0x0005DB + , 0x0005E4 + , 0x0105D2 + , 0x0105DA + , 0x011099 + , 0x01109B + , 0x0110A5 + , 0x011131 + , 0x011132 + , 0x011347 + , 0x011347 + , 0x011382 + , 0x011384 + , 0x01138B + , 0x011390 + , 0x0113C2 + , 0x0113C2 + , 0x0113C2 + , 0x0114B9 + , 0x0114B9 + , 0x0114B9 + , 0x0115B8 + , 0x0115B9 + , 0x011935 + , 0x01611E + , 0x01611E + , 0x01611E + , 0x016129 + , 0x01611E + , 0x016121 + , 0x016122 + , 0x016121 + , 0x016D67 + , 0x016D63 + , 0x016D69 + , 0x01D157 + , 0x01D158 + , 0x01D15F + , 0x01D15F + , 0x01D15F + , 0x01D15F + , 0x01D15F + , 0x01D1B9 + , 0x01D1BA + , 0x01D1BB + , 0x01D1BC + , 0x01D1BB + , 0x01D1BC + , 0x004E3D + , 0x004E38 + , 0x004E41 + , 0x020122 + , 0x004F60 + , 0x004FAE + , 0x004FBB + , 0x005002 + , 0x00507A + , 0x005099 + , 0x0050E7 + , 0x0050CF + , 0x00349E + , 0x02063A + , 0x00514D + , 0x005154 + , 0x005164 + , 0x005177 + , 0x02051C + , 0x0034B9 + , 0x005167 + , 0x00518D + , 0x02054B + , 0x005197 + , 0x0051A4 + , 0x004ECC + , 0x0051AC + , 0x0051B5 + , 0x0291DF + , 0x0051F5 + , 0x005203 + , 0x0034DF + , 0x00523B + , 0x005246 + , 0x005272 + , 0x005277 + , 0x003515 + , 0x0052C7 + , 0x0052C9 + , 0x0052E4 + , 0x0052FA + , 0x005305 + , 0x005306 + , 0x005317 + , 0x005349 + , 0x005351 + , 0x00535A + , 0x005373 + , 0x00537D + , 0x00537F + , 0x00537F + , 0x00537F + , 0x020A2C + , 0x007070 + , 0x0053CA + , 0x0053DF + , 0x020B63 + , 0x0053EB + , 0x0053F1 + , 0x005406 + , 0x00549E + , 0x005438 + , 0x005448 + , 0x005468 + , 0x0054A2 + , 0x0054F6 + , 0x005510 + , 0x005553 + , 0x005563 + , 0x005584 + , 0x005584 + , 0x005599 + , 0x0055AB + , 0x0055B3 + , 0x0055C2 + , 0x005716 + , 0x005606 + , 0x005717 + , 0x005651 + , 0x005674 + , 0x005207 + , 0x0058EE + , 0x0057CE + , 0x0057F4 + , 0x00580D + , 0x00578B + , 0x005832 + , 0x005831 + , 0x0058AC + , 0x0214E4 + , 0x0058F2 + , 0x0058F7 + , 0x005906 + , 0x00591A + , 0x005922 + , 0x005962 + , 0x0216A8 + , 0x0216EA + , 0x0059EC + , 0x005A1B + , 0x005A27 + , 0x0059D8 + , 0x005A66 + , 0x0036EE + , 0x0036FC + , 0x005B08 + , 0x005B3E + , 0x005B3E + , 0x0219C8 + , 0x005BC3 + , 0x005BD8 + , 0x005BE7 + , 0x005BF3 + , 0x021B18 + , 0x005BFF + , 0x005C06 + , 0x005F53 + , 0x005C22 + , 0x003781 + , 0x005C60 + , 0x005C6E + , 0x005CC0 + , 0x005C8D + , 0x021DE4 + , 0x005D43 + , 0x021DE6 + , 0x005D6E + , 0x005D6B + , 0x005D7C + , 0x005DE1 + , 0x005DE2 + , 0x00382F + , 0x005DFD + , 0x005E28 + , 0x005E3D + , 0x005E69 + , 0x003862 + , 0x022183 + , 0x00387C + , 0x005EB0 + , 0x005EB3 + , 0x005EB6 + , 0x005ECA + , 0x02A392 + , 0x005EFE + , 0x022331 + , 0x022331 + , 0x008201 + , 0x005F22 + , 0x005F22 + , 0x0038C7 + , 0x0232B8 + , 0x0261DA + , 0x005F62 + , 0x005F6B + , 0x0038E3 + , 0x005F9A + , 0x005FCD + , 0x005FD7 + , 0x005FF9 + , 0x006081 + , 0x00393A + , 0x00391C + , 0x006094 + , 0x0226D4 + , 0x0060C7 + , 0x006148 + , 0x00614C + , 0x00614E + , 0x00614C + , 0x00617A + , 0x00618E + , 0x0061B2 + , 0x0061A4 + , 0x0061AF + , 0x0061DE + , 0x0061F2 + , 0x0061F6 + , 0x006210 + , 0x00621B + , 0x00625D + , 0x0062B1 + , 0x0062D4 + , 0x006350 + , 0x022B0C + , 0x00633D + , 0x0062FC + , 0x006368 + , 0x006383 + , 0x0063E4 + , 0x022BF1 + , 0x006422 + , 0x0063C5 + , 0x0063A9 + , 0x003A2E + , 0x006469 + , 0x00647E + , 0x00649D + , 0x006477 + , 0x003A6C + , 0x00654F + , 0x00656C + , 0x02300A + , 0x0065E3 + , 0x0066F8 + , 0x006649 + , 0x003B19 + , 0x006691 + , 0x003B08 + , 0x003AE4 + , 0x005192 + , 0x005195 + , 0x006700 + , 0x00669C + , 0x0080AD + , 0x0043D9 + , 0x006717 + , 0x00671B + , 0x006721 + , 0x00675E + , 0x006753 + , 0x0233C3 + , 0x003B49 + , 0x0067FA + , 0x006785 + , 0x006852 + , 0x006885 + , 0x02346D + , 0x00688E + , 0x00681F + , 0x006914 + , 0x003B9D + , 0x006942 + , 0x0069A3 + , 0x0069EA + , 0x006AA8 + , 0x0236A3 + , 0x006ADB + , 0x003C18 + , 0x006B21 + , 0x0238A7 + , 0x006B54 + , 0x003C4E + , 0x006B72 + , 0x006B9F + , 0x006BBA + , 0x006BBB + , 0x023A8D + , 0x021D0B + , 0x023AFA + , 0x006C4E + , 0x023CBC + , 0x006CBF + , 0x006CCD + , 0x006C67 + , 0x006D16 + , 0x006D3E + , 0x006D77 + , 0x006D41 + , 0x006D69 + , 0x006D78 + , 0x006D85 + , 0x023D1E + , 0x006D34 + , 0x006E2F + , 0x006E6E + , 0x003D33 + , 0x006ECB + , 0x006EC7 + , 0x023ED1 + , 0x006DF9 + , 0x006F6E + , 0x023F5E + , 0x023F8E + , 0x006FC6 + , 0x007039 + , 0x00701E + , 0x00701B + , 0x003D96 + , 0x00704A + , 0x00707D + , 0x007077 + , 0x0070AD + , 0x020525 + , 0x007145 + , 0x024263 + , 0x00719C + , 0x0243AB + , 0x007228 + , 0x007235 + , 0x007250 + , 0x024608 + , 0x007280 + , 0x007295 + , 0x024735 + , 0x024814 + , 0x00737A + , 0x00738B + , 0x003EAC + , 0x0073A5 + , 0x003EB8 + , 0x003EB8 + , 0x007447 + , 0x00745C + , 0x007471 + , 0x007485 + , 0x0074CA + , 0x003F1B + , 0x007524 + , 0x024C36 + , 0x00753E + , 0x024C92 + , 0x007570 + , 0x02219F + , 0x007610 + , 0x024FA1 + , 0x024FB8 + , 0x025044 + , 0x003FFC + , 0x004008 + , 0x0076F4 + , 0x0250F3 + , 0x0250F2 + , 0x025119 + , 0x025133 + , 0x00771E + , 0x00771F + , 0x00771F + , 0x00774A + , 0x004039 + , 0x00778B + , 0x004046 + , 0x004096 + , 0x02541D + , 0x00784E + , 0x00788C + , 0x0078CC + , 0x0040E3 + , 0x025626 + , 0x007956 + , 0x02569A + , 0x0256C5 + , 0x00798F + , 0x0079EB + , 0x00412F + , 0x007A40 + , 0x007A4A + , 0x007A4F + , 0x02597C + , 0x025AA7 + , 0x025AA7 + , 0x007AEE + , 0x004202 + , 0x025BAB + , 0x007BC6 + , 0x007BC9 + , 0x004227 + , 0x025C80 + , 0x007CD2 + , 0x0042A0 + , 0x007CE8 + , 0x007CE3 + , 0x007D00 + , 0x025F86 + , 0x007D63 + , 0x004301 + , 0x007DC7 + , 0x007E02 + , 0x007E45 + , 0x004334 + , 0x026228 + , 0x026247 + , 0x004359 + , 0x0262D9 + , 0x007F7A + , 0x02633E + , 0x007F95 + , 0x007FFA + , 0x008005 + , 0x0264DA + , 0x026523 + , 0x008060 + , 0x0265A8 + , 0x008070 + , 0x02335F + , 0x0043D5 + , 0x0080B2 + , 0x008103 + , 0x00440B + , 0x00813E + , 0x005AB5 + , 0x0267A7 + , 0x0267B5 + , 0x023393 + , 0x02339C + , 0x008201 + , 0x008204 + , 0x008F9E + , 0x00446B + , 0x008291 + , 0x00828B + , 0x00829D + , 0x0052B3 + , 0x0082B1 + , 0x0082B3 + , 0x0082BD + , 0x0082E6 + , 0x026B3C + , 0x0082E5 + , 0x00831D + , 0x008363 + , 0x0083AD + , 0x008323 + , 0x0083BD + , 0x0083E7 + , 0x008457 + , 0x008353 + , 0x0083CA + , 0x0083CC + , 0x0083DC + , 0x026C36 + , 0x026D6B + , 0x026CD5 + , 0x00452B + , 0x0084F1 + , 0x0084F3 + , 0x008516 + , 0x0273CA + , 0x008564 + , 0x026F2C + , 0x00455D + , 0x004561 + , 0x026FB1 + , 0x0270D2 + , 0x00456B + , 0x008650 + , 0x00865C + , 0x008667 + , 0x008669 + , 0x0086A9 + , 0x008688 + , 0x00870E + , 0x0086E2 + , 0x008779 + , 0x008728 + , 0x00876B + , 0x008786 + , 0x0045D7 + , 0x0087E1 + , 0x008801 + , 0x0045F9 + , 0x008860 + , 0x008863 + , 0x027667 + , 0x0088D7 + , 0x0088DE + , 0x004635 + , 0x0088FA + , 0x0034BB + , 0x0278AE + , 0x027966 + , 0x0046BE + , 0x0046C7 + , 0x008AA0 + , 0x008AED + , 0x008B8A + , 0x008C55 + , 0x027CA8 + , 0x008CAB + , 0x008CC1 + , 0x008D1B + , 0x008D77 + , 0x027F2F + , 0x020804 + , 0x008DCB + , 0x008DBC + , 0x008DF0 + , 0x0208DE + , 0x008ED4 + , 0x008F38 + , 0x0285D2 + , 0x0285ED + , 0x009094 + , 0x0090F1 + , 0x009111 + , 0x02872E + , 0x00911B + , 0x009238 + , 0x0092D7 + , 0x0092D8 + , 0x00927C + , 0x0093F9 + , 0x009415 + , 0x028BFA + , 0x00958B + , 0x004995 + , 0x0095B7 + , 0x028D77 + , 0x0049E6 + , 0x0096C3 + , 0x005DB2 + , 0x009723 + , 0x029145 + , 0x02921A + , 0x004A6E + , 0x004A76 + , 0x0097E0 + , 0x02940A + , 0x004AB2 + , 0x029496 + , 0x00980B + , 0x00980B + , 0x009829 + , 0x0295B6 + , 0x0098E2 + , 0x004B33 + , 0x009929 + , 0x0099A7 + , 0x0099C2 + , 0x0099FE + , 0x004BCE + , 0x029B30 + , 0x009B12 + , 0x009C40 + , 0x009CFD + , 0x004CCE + , 0x004CED + , 0x009D67 + , 0x02A0CE + , 0x004CF8 + , 0x02A105 + , 0x02A20E + , 0x02A291 + , 0x009EBB + , 0x004D56 + , 0x009EF9 + , 0x009EFE + , 0x009F05 + , 0x009F0F + , 0x009F16 + , 0x009F3B + , 0x02A600 + ] + +decompSecond :: ByteArray +decompSecond = PBA.byteArrayFromList + [ (0x000300 :: Word32) + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000308 + , 0x00030A + , 0x000327 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000303 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000308 + , 0x00030A + , 0x000327 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000303 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000308 + , 0x000301 + , 0x000308 + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x000328 + , 0x000328 + , 0x000301 + , 0x000301 + , 0x000302 + , 0x000302 + , 0x000307 + , 0x000307 + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x000307 + , 0x000307 + , 0x000328 + , 0x000328 + , 0x00030C + , 0x00030C + , 0x000302 + , 0x000302 + , 0x000306 + , 0x000306 + , 0x000307 + , 0x000307 + , 0x000327 + , 0x000327 + , 0x000302 + , 0x000302 + , 0x000303 + , 0x000303 + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x000328 + , 0x000328 + , 0x000307 + , 0x000302 + , 0x000302 + , 0x000327 + , 0x000327 + , 0x000301 + , 0x000301 + , 0x000327 + , 0x000327 + , 0x00030C + , 0x00030C + , 0x000301 + , 0x000301 + , 0x000327 + , 0x000327 + , 0x00030C + , 0x00030C + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x00030B + , 0x00030B + , 0x000301 + , 0x000301 + , 0x000327 + , 0x000327 + , 0x00030C + , 0x00030C + , 0x000301 + , 0x000301 + , 0x000302 + , 0x000302 + , 0x000327 + , 0x000327 + , 0x00030C + , 0x00030C + , 0x000327 + , 0x000327 + , 0x00030C + , 0x00030C + , 0x000303 + , 0x000303 + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x00030A + , 0x00030A + , 0x00030B + , 0x00030B + , 0x000328 + , 0x000328 + , 0x000302 + , 0x000302 + , 0x000302 + , 0x000302 + , 0x000308 + , 0x000301 + , 0x000301 + , 0x000307 + , 0x000307 + , 0x00030C + , 0x00030C + , 0x00031B + , 0x00031B + , 0x00031B + , 0x00031B + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x000304 + , 0x000304 + , 0x000301 + , 0x000301 + , 0x00030C + , 0x00030C + , 0x000300 + , 0x000300 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x00030C + , 0x00030C + , 0x00030C + , 0x00030C + , 0x000328 + , 0x000328 + , 0x000304 + , 0x000304 + , 0x00030C + , 0x00030C + , 0x00030C + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x00030F + , 0x00030F + , 0x000311 + , 0x000311 + , 0x000326 + , 0x000326 + , 0x000326 + , 0x000326 + , 0x00030C + , 0x00030C + , 0x000307 + , 0x000307 + , 0x000327 + , 0x000327 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000307 + , 0x000307 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000301 + , 0x000000 + , 0x000000 + , 0x000301 + , 0x000301 + , 0x000000 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000300 + , 0x000308 + , 0x000301 + , 0x000308 + , 0x000301 + , 0x000300 + , 0x000306 + , 0x000306 + , 0x000306 + , 0x000300 + , 0x000308 + , 0x000301 + , 0x000308 + , 0x000301 + , 0x000300 + , 0x000306 + , 0x00030F + , 0x00030F + , 0x000306 + , 0x000306 + , 0x000306 + , 0x000306 + , 0x000308 + , 0x000308 + , 0x000306 + , 0x000306 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000304 + , 0x000304 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000304 + , 0x000304 + , 0x000308 + , 0x000308 + , 0x00030B + , 0x00030B + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000653 + , 0x000654 + , 0x000654 + , 0x000655 + , 0x000654 + , 0x000654 + , 0x000654 + , 0x000654 + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x00093C + , 0x0009BE + , 0x0009D7 + , 0x0009BC + , 0x0009BC + , 0x0009BC + , 0x000A3C + , 0x000A3C + , 0x000A3C + , 0x000A3C + , 0x000A3C + , 0x000A3C + , 0x000B56 + , 0x000B3E + , 0x000B57 + , 0x000B3C + , 0x000B3C + , 0x000BD7 + , 0x000BBE + , 0x000BBE + , 0x000BD7 + , 0x000C56 + , 0x000CD5 + , 0x000CD5 + , 0x000CD6 + , 0x000CC2 + , 0x000CD5 + , 0x000D3E + , 0x000D3E + , 0x000D57 + , 0x000DCA + , 0x000DCF + , 0x000DCA + , 0x000DDF + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB5 + , 0x000F72 + , 0x000F74 + , 0x000F80 + , 0x000F80 + , 0x000F80 + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB7 + , 0x000FB5 + , 0x00102E + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x000325 + , 0x000325 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x000301 + , 0x000301 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x000327 + , 0x000327 + , 0x00032D + , 0x00032D + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x00032D + , 0x00032D + , 0x000330 + , 0x000330 + , 0x000306 + , 0x000306 + , 0x000307 + , 0x000307 + , 0x000304 + , 0x000304 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000308 + , 0x000308 + , 0x000327 + , 0x000327 + , 0x00032E + , 0x00032E + , 0x000330 + , 0x000330 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x000323 + , 0x000323 + , 0x000304 + , 0x000304 + , 0x000331 + , 0x000331 + , 0x00032D + , 0x00032D + , 0x000301 + , 0x000301 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x00032D + , 0x00032D + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000301 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000304 + , 0x000304 + , 0x000331 + , 0x000331 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x00032D + , 0x00032D + , 0x000324 + , 0x000324 + , 0x000330 + , 0x000330 + , 0x00032D + , 0x00032D + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000303 + , 0x000303 + , 0x000323 + , 0x000323 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000307 + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000307 + , 0x000307 + , 0x000308 + , 0x000308 + , 0x000307 + , 0x000307 + , 0x000302 + , 0x000302 + , 0x000323 + , 0x000323 + , 0x000331 + , 0x000331 + , 0x000331 + , 0x000308 + , 0x00030A + , 0x00030A + , 0x000307 + , 0x000323 + , 0x000323 + , 0x000309 + , 0x000309 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000302 + , 0x000302 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000306 + , 0x000306 + , 0x000323 + , 0x000323 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000302 + , 0x000302 + , 0x000309 + , 0x000309 + , 0x000323 + , 0x000323 + , 0x000323 + , 0x000323 + , 0x000309 + , 0x000309 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000302 + , 0x000302 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000323 + , 0x000323 + , 0x000323 + , 0x000323 + , 0x000309 + , 0x000309 + , 0x000301 + , 0x000301 + , 0x000300 + , 0x000300 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000323 + , 0x000323 + , 0x000300 + , 0x000300 + , 0x000323 + , 0x000323 + , 0x000309 + , 0x000309 + , 0x000303 + , 0x000303 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000300 + , 0x000301 + , 0x000301 + , 0x000342 + , 0x000342 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000306 + , 0x000304 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000342 + , 0x000345 + , 0x000306 + , 0x000304 + , 0x000300 + , 0x000000 + , 0x000345 + , 0x000000 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000306 + , 0x000304 + , 0x000300 + , 0x000000 + , 0x000342 + , 0x000342 + , 0x000306 + , 0x000304 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000306 + , 0x000304 + , 0x000300 + , 0x000000 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000342 + , 0x000306 + , 0x000304 + , 0x000300 + , 0x000000 + , 0x000314 + , 0x000300 + , 0x000000 + , 0x000000 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000000 + , 0x000300 + , 0x000000 + , 0x000345 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000000 + , 0x000000 + , 0x000338 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x0005B4 + , 0x0005B7 + , 0x0005C1 + , 0x0005C2 + , 0x0005C1 + , 0x0005C2 + , 0x0005B7 + , 0x0005B8 + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005BC + , 0x0005B9 + , 0x0005BF + , 0x0005BF + , 0x0005BF + , 0x000307 + , 0x000307 + , 0x0110BA + , 0x0110BA + , 0x0110BA + , 0x011127 + , 0x011127 + , 0x01133E + , 0x011357 + , 0x0113C9 + , 0x0113BB + , 0x0113C2 + , 0x0113C9 + , 0x0113C2 + , 0x0113B8 + , 0x0113C9 + , 0x0114BA + , 0x0114B0 + , 0x0114BD + , 0x0115AF + , 0x0115AF + , 0x011930 + , 0x01611E + , 0x016129 + , 0x01611F + , 0x01611F + , 0x016120 + , 0x01611F + , 0x01611F + , 0x016120 + , 0x016D67 + , 0x016D67 + , 0x016D67 + , 0x01D165 + , 0x01D165 + , 0x01D16E + , 0x01D16F + , 0x01D170 + , 0x01D171 + , 0x01D172 + , 0x01D165 + , 0x01D165 + , 0x01D16E + , 0x01D16E + , 0x01D16F + , 0x01D16F + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + , 0x000000 + ] + + +---------------------------------------------------------------------- +-- Primary composition +---------------------------------------------------------------------- + +compCount :: Int +compCount = 961 + +compFirst :: ByteArray +compFirst = PBA.byteArrayFromList + [ (0x00003C :: Word32) + , 0x00003D + , 0x00003E + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000041 + , 0x000042 + , 0x000042 + , 0x000042 + , 0x000043 + , 0x000043 + , 0x000043 + , 0x000043 + , 0x000043 + , 0x000044 + , 0x000044 + , 0x000044 + , 0x000044 + , 0x000044 + , 0x000044 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000045 + , 0x000046 + , 0x000047 + , 0x000047 + , 0x000047 + , 0x000047 + , 0x000047 + , 0x000047 + , 0x000047 + , 0x000048 + , 0x000048 + , 0x000048 + , 0x000048 + , 0x000048 + , 0x000048 + , 0x000048 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x000049 + , 0x00004A + , 0x00004B + , 0x00004B + , 0x00004B + , 0x00004B + , 0x00004B + , 0x00004C + , 0x00004C + , 0x00004C + , 0x00004C + , 0x00004C + , 0x00004C + , 0x00004D + , 0x00004D + , 0x00004D + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004E + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x00004F + , 0x000050 + , 0x000050 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000052 + , 0x000053 + , 0x000053 + , 0x000053 + , 0x000053 + , 0x000053 + , 0x000053 + , 0x000053 + , 0x000054 + , 0x000054 + , 0x000054 + , 0x000054 + , 0x000054 + , 0x000054 + , 0x000054 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000055 + , 0x000056 + , 0x000056 + , 0x000057 + , 0x000057 + , 0x000057 + , 0x000057 + , 0x000057 + , 0x000057 + , 0x000058 + , 0x000058 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x000059 + , 0x00005A + , 0x00005A + , 0x00005A + , 0x00005A + , 0x00005A + , 0x00005A + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000061 + , 0x000062 + , 0x000062 + , 0x000062 + , 0x000063 + , 0x000063 + , 0x000063 + , 0x000063 + , 0x000063 + , 0x000064 + , 0x000064 + , 0x000064 + , 0x000064 + , 0x000064 + , 0x000064 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000065 + , 0x000066 + , 0x000067 + , 0x000067 + , 0x000067 + , 0x000067 + , 0x000067 + , 0x000067 + , 0x000067 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000068 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x000069 + , 0x00006A + , 0x00006A + , 0x00006B + , 0x00006B + , 0x00006B + , 0x00006B + , 0x00006B + , 0x00006C + , 0x00006C + , 0x00006C + , 0x00006C + , 0x00006C + , 0x00006C + , 0x00006D + , 0x00006D + , 0x00006D + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006E + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x00006F + , 0x000070 + , 0x000070 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000072 + , 0x000073 + , 0x000073 + , 0x000073 + , 0x000073 + , 0x000073 + , 0x000073 + , 0x000073 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000074 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000075 + , 0x000076 + , 0x000076 + , 0x000077 + , 0x000077 + , 0x000077 + , 0x000077 + , 0x000077 + , 0x000077 + , 0x000077 + , 0x000078 + , 0x000078 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x000079 + , 0x00007A + , 0x00007A + , 0x00007A + , 0x00007A + , 0x00007A + , 0x00007A + , 0x0000A8 + , 0x0000A8 + , 0x0000A8 + , 0x0000C2 + , 0x0000C2 + , 0x0000C2 + , 0x0000C2 + , 0x0000C4 + , 0x0000C5 + , 0x0000C6 + , 0x0000C6 + , 0x0000C7 + , 0x0000CA + , 0x0000CA + , 0x0000CA + , 0x0000CA + , 0x0000CF + , 0x0000D4 + , 0x0000D4 + , 0x0000D4 + , 0x0000D4 + , 0x0000D5 + , 0x0000D5 + , 0x0000D5 + , 0x0000D6 + , 0x0000D8 + , 0x0000DC + , 0x0000DC + , 0x0000DC + , 0x0000DC + , 0x0000E2 + , 0x0000E2 + , 0x0000E2 + , 0x0000E2 + , 0x0000E4 + , 0x0000E5 + , 0x0000E6 + , 0x0000E6 + , 0x0000E7 + , 0x0000EA + , 0x0000EA + , 0x0000EA + , 0x0000EA + , 0x0000EF + , 0x0000F4 + , 0x0000F4 + , 0x0000F4 + , 0x0000F4 + , 0x0000F5 + , 0x0000F5 + , 0x0000F5 + , 0x0000F6 + , 0x0000F8 + , 0x0000FC + , 0x0000FC + , 0x0000FC + , 0x0000FC + , 0x000102 + , 0x000102 + , 0x000102 + , 0x000102 + , 0x000103 + , 0x000103 + , 0x000103 + , 0x000103 + , 0x000112 + , 0x000112 + , 0x000113 + , 0x000113 + , 0x00014C + , 0x00014C + , 0x00014D + , 0x00014D + , 0x00015A + , 0x00015B + , 0x000160 + , 0x000161 + , 0x000168 + , 0x000169 + , 0x00016A + , 0x00016B + , 0x00017F + , 0x0001A0 + , 0x0001A0 + , 0x0001A0 + , 0x0001A0 + , 0x0001A0 + , 0x0001A1 + , 0x0001A1 + , 0x0001A1 + , 0x0001A1 + , 0x0001A1 + , 0x0001AF + , 0x0001AF + , 0x0001AF + , 0x0001AF + , 0x0001AF + , 0x0001B0 + , 0x0001B0 + , 0x0001B0 + , 0x0001B0 + , 0x0001B0 + , 0x0001B7 + , 0x0001EA + , 0x0001EB + , 0x000226 + , 0x000227 + , 0x000228 + , 0x000229 + , 0x00022E + , 0x00022F + , 0x000292 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000391 + , 0x000395 + , 0x000395 + , 0x000395 + , 0x000395 + , 0x000397 + , 0x000397 + , 0x000397 + , 0x000397 + , 0x000397 + , 0x000399 + , 0x000399 + , 0x000399 + , 0x000399 + , 0x000399 + , 0x000399 + , 0x000399 + , 0x00039F + , 0x00039F + , 0x00039F + , 0x00039F + , 0x0003A1 + , 0x0003A5 + , 0x0003A5 + , 0x0003A5 + , 0x0003A5 + , 0x0003A5 + , 0x0003A5 + , 0x0003A9 + , 0x0003A9 + , 0x0003A9 + , 0x0003A9 + , 0x0003A9 + , 0x0003AC + , 0x0003AE + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B1 + , 0x0003B5 + , 0x0003B5 + , 0x0003B5 + , 0x0003B5 + , 0x0003B7 + , 0x0003B7 + , 0x0003B7 + , 0x0003B7 + , 0x0003B7 + , 0x0003B7 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003B9 + , 0x0003BF + , 0x0003BF + , 0x0003BF + , 0x0003BF + , 0x0003C1 + , 0x0003C1 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C5 + , 0x0003C9 + , 0x0003C9 + , 0x0003C9 + , 0x0003C9 + , 0x0003C9 + , 0x0003C9 + , 0x0003CA + , 0x0003CA + , 0x0003CA + , 0x0003CB + , 0x0003CB + , 0x0003CB + , 0x0003CE + , 0x0003D2 + , 0x0003D2 + , 0x000406 + , 0x000410 + , 0x000410 + , 0x000413 + , 0x000415 + , 0x000415 + , 0x000415 + , 0x000416 + , 0x000416 + , 0x000417 + , 0x000418 + , 0x000418 + , 0x000418 + , 0x000418 + , 0x00041A + , 0x00041E + , 0x000423 + , 0x000423 + , 0x000423 + , 0x000423 + , 0x000427 + , 0x00042B + , 0x00042D + , 0x000430 + , 0x000430 + , 0x000433 + , 0x000435 + , 0x000435 + , 0x000435 + , 0x000436 + , 0x000436 + , 0x000437 + , 0x000438 + , 0x000438 + , 0x000438 + , 0x000438 + , 0x00043A + , 0x00043E + , 0x000443 + , 0x000443 + , 0x000443 + , 0x000443 + , 0x000447 + , 0x00044B + , 0x00044D + , 0x000456 + , 0x000474 + , 0x000475 + , 0x0004D8 + , 0x0004D9 + , 0x0004E8 + , 0x0004E9 + , 0x000627 + , 0x000627 + , 0x000627 + , 0x000648 + , 0x00064A + , 0x0006C1 + , 0x0006D2 + , 0x0006D5 + , 0x000928 + , 0x000930 + , 0x000933 + , 0x0009C7 + , 0x0009C7 + , 0x000B47 + , 0x000B47 + , 0x000B47 + , 0x000B92 + , 0x000BC6 + , 0x000BC6 + , 0x000BC7 + , 0x000C46 + , 0x000CBF + , 0x000CC6 + , 0x000CC6 + , 0x000CC6 + , 0x000CCA + , 0x000D46 + , 0x000D46 + , 0x000D47 + , 0x000DD9 + , 0x000DD9 + , 0x000DD9 + , 0x000DDC + , 0x001025 + , 0x001B05 + , 0x001B07 + , 0x001B09 + , 0x001B0B + , 0x001B0D + , 0x001B11 + , 0x001B3A + , 0x001B3C + , 0x001B3E + , 0x001B3F + , 0x001B42 + , 0x001E36 + , 0x001E37 + , 0x001E5A + , 0x001E5B + , 0x001E62 + , 0x001E63 + , 0x001EA0 + , 0x001EA0 + , 0x001EA1 + , 0x001EA1 + , 0x001EB8 + , 0x001EB9 + , 0x001ECC + , 0x001ECD + , 0x001F00 + , 0x001F00 + , 0x001F00 + , 0x001F00 + , 0x001F01 + , 0x001F01 + , 0x001F01 + , 0x001F01 + , 0x001F02 + , 0x001F03 + , 0x001F04 + , 0x001F05 + , 0x001F06 + , 0x001F07 + , 0x001F08 + , 0x001F08 + , 0x001F08 + , 0x001F08 + , 0x001F09 + , 0x001F09 + , 0x001F09 + , 0x001F09 + , 0x001F0A + , 0x001F0B + , 0x001F0C + , 0x001F0D + , 0x001F0E + , 0x001F0F + , 0x001F10 + , 0x001F10 + , 0x001F11 + , 0x001F11 + , 0x001F18 + , 0x001F18 + , 0x001F19 + , 0x001F19 + , 0x001F20 + , 0x001F20 + , 0x001F20 + , 0x001F20 + , 0x001F21 + , 0x001F21 + , 0x001F21 + , 0x001F21 + , 0x001F22 + , 0x001F23 + , 0x001F24 + , 0x001F25 + , 0x001F26 + , 0x001F27 + , 0x001F28 + , 0x001F28 + , 0x001F28 + , 0x001F28 + , 0x001F29 + , 0x001F29 + , 0x001F29 + , 0x001F29 + , 0x001F2A + , 0x001F2B + , 0x001F2C + , 0x001F2D + , 0x001F2E + , 0x001F2F + , 0x001F30 + , 0x001F30 + , 0x001F30 + , 0x001F31 + , 0x001F31 + , 0x001F31 + , 0x001F38 + , 0x001F38 + , 0x001F38 + , 0x001F39 + , 0x001F39 + , 0x001F39 + , 0x001F40 + , 0x001F40 + , 0x001F41 + , 0x001F41 + , 0x001F48 + , 0x001F48 + , 0x001F49 + , 0x001F49 + , 0x001F50 + , 0x001F50 + , 0x001F50 + , 0x001F51 + , 0x001F51 + , 0x001F51 + , 0x001F59 + , 0x001F59 + , 0x001F59 + , 0x001F60 + , 0x001F60 + , 0x001F60 + , 0x001F60 + , 0x001F61 + , 0x001F61 + , 0x001F61 + , 0x001F61 + , 0x001F62 + , 0x001F63 + , 0x001F64 + , 0x001F65 + , 0x001F66 + , 0x001F67 + , 0x001F68 + , 0x001F68 + , 0x001F68 + , 0x001F68 + , 0x001F69 + , 0x001F69 + , 0x001F69 + , 0x001F69 + , 0x001F6A + , 0x001F6B + , 0x001F6C + , 0x001F6D + , 0x001F6E + , 0x001F6F + , 0x001F70 + , 0x001F74 + , 0x001F7C + , 0x001FB6 + , 0x001FBF + , 0x001FBF + , 0x001FBF + , 0x001FC6 + , 0x001FF6 + , 0x001FFE + , 0x001FFE + , 0x001FFE + , 0x002190 + , 0x002192 + , 0x002194 + , 0x0021D0 + , 0x0021D2 + , 0x0021D4 + , 0x002203 + , 0x002208 + , 0x00220B + , 0x002223 + , 0x002225 + , 0x00223C + , 0x002243 + , 0x002245 + , 0x002248 + , 0x00224D + , 0x002261 + , 0x002264 + , 0x002265 + , 0x002272 + , 0x002273 + , 0x002276 + , 0x002277 + , 0x00227A + , 0x00227B + , 0x00227C + , 0x00227D + , 0x002282 + , 0x002283 + , 0x002286 + , 0x002287 + , 0x002291 + , 0x002292 + , 0x0022A2 + , 0x0022A8 + , 0x0022A9 + , 0x0022AB + , 0x0022B2 + , 0x0022B3 + , 0x0022B4 + , 0x0022B5 + , 0x003046 + , 0x00304B + , 0x00304D + , 0x00304F + , 0x003051 + , 0x003053 + , 0x003055 + , 0x003057 + , 0x003059 + , 0x00305B + , 0x00305D + , 0x00305F + , 0x003061 + , 0x003064 + , 0x003066 + , 0x003068 + , 0x00306F + , 0x00306F + , 0x003072 + , 0x003072 + , 0x003075 + , 0x003075 + , 0x003078 + , 0x003078 + , 0x00307B + , 0x00307B + , 0x00309D + , 0x0030A6 + , 0x0030AB + , 0x0030AD + , 0x0030AF + , 0x0030B1 + , 0x0030B3 + , 0x0030B5 + , 0x0030B7 + , 0x0030B9 + , 0x0030BB + , 0x0030BD + , 0x0030BF + , 0x0030C1 + , 0x0030C4 + , 0x0030C6 + , 0x0030C8 + , 0x0030CF + , 0x0030CF + , 0x0030D2 + , 0x0030D2 + , 0x0030D5 + , 0x0030D5 + , 0x0030D8 + , 0x0030D8 + , 0x0030DB + , 0x0030DB + , 0x0030EF + , 0x0030F0 + , 0x0030F1 + , 0x0030F2 + , 0x0030FD + , 0x0105D2 + , 0x0105DA + , 0x011099 + , 0x01109B + , 0x0110A5 + , 0x011131 + , 0x011132 + , 0x011347 + , 0x011347 + , 0x011382 + , 0x011384 + , 0x01138B + , 0x011390 + , 0x0113C2 + , 0x0113C2 + , 0x0113C2 + , 0x0114B9 + , 0x0114B9 + , 0x0114B9 + , 0x0115B8 + , 0x0115B9 + , 0x011935 + , 0x01611E + , 0x01611E + , 0x01611E + , 0x01611E + , 0x016121 + , 0x016121 + , 0x016122 + , 0x016129 + , 0x016D63 + , 0x016D67 + , 0x016D69 + ] + +compSecond :: ByteArray +compSecond = PBA.byteArrayFromList + [ (0x000338 :: Word32) + , 0x000338 + , 0x000338 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030A + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000325 + , 0x000328 + , 0x000307 + , 0x000323 + , 0x000331 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000327 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000327 + , 0x000328 + , 0x00032D + , 0x000330 + , 0x000307 + , 0x000301 + , 0x000302 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x00030C + , 0x000327 + , 0x000302 + , 0x000307 + , 0x000308 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032E + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000328 + , 0x000330 + , 0x000302 + , 0x000301 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x000331 + , 0x000301 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000301 + , 0x000307 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030B + , 0x00030C + , 0x00030F + , 0x000311 + , 0x00031B + , 0x000323 + , 0x000328 + , 0x000301 + , 0x000307 + , 0x000301 + , 0x000307 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000327 + , 0x000331 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000326 + , 0x000327 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000326 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000309 + , 0x00030A + , 0x00030B + , 0x00030C + , 0x00030F + , 0x000311 + , 0x00031B + , 0x000323 + , 0x000324 + , 0x000328 + , 0x00032D + , 0x000330 + , 0x000303 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x000308 + , 0x000323 + , 0x000307 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x000323 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030A + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000325 + , 0x000328 + , 0x000307 + , 0x000323 + , 0x000331 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000327 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000327 + , 0x000328 + , 0x00032D + , 0x000330 + , 0x000307 + , 0x000301 + , 0x000302 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x00030C + , 0x000327 + , 0x000302 + , 0x000307 + , 0x000308 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032E + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000309 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000328 + , 0x000330 + , 0x000302 + , 0x00030C + , 0x000301 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x000331 + , 0x000301 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000301 + , 0x000307 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030B + , 0x00030C + , 0x00030F + , 0x000311 + , 0x00031B + , 0x000323 + , 0x000328 + , 0x000301 + , 0x000307 + , 0x000301 + , 0x000307 + , 0x00030C + , 0x00030F + , 0x000311 + , 0x000323 + , 0x000327 + , 0x000331 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000326 + , 0x000327 + , 0x000307 + , 0x000308 + , 0x00030C + , 0x000323 + , 0x000326 + , 0x000327 + , 0x00032D + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000309 + , 0x00030A + , 0x00030B + , 0x00030C + , 0x00030F + , 0x000311 + , 0x00031B + , 0x000323 + , 0x000324 + , 0x000328 + , 0x00032D + , 0x000330 + , 0x000303 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x000308 + , 0x00030A + , 0x000323 + , 0x000307 + , 0x000308 + , 0x000300 + , 0x000301 + , 0x000302 + , 0x000303 + , 0x000304 + , 0x000307 + , 0x000308 + , 0x000309 + , 0x00030A + , 0x000323 + , 0x000301 + , 0x000302 + , 0x000307 + , 0x00030C + , 0x000323 + , 0x000331 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000304 + , 0x000301 + , 0x000301 + , 0x000304 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000301 + , 0x000304 + , 0x000308 + , 0x000304 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x00030C + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000304 + , 0x000301 + , 0x000301 + , 0x000304 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000301 + , 0x000304 + , 0x000308 + , 0x000304 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x00030C + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000307 + , 0x000301 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000307 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000323 + , 0x000300 + , 0x000301 + , 0x000303 + , 0x000309 + , 0x000323 + , 0x00030C + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000306 + , 0x000306 + , 0x000304 + , 0x000304 + , 0x00030C + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000313 + , 0x000314 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000313 + , 0x000314 + , 0x000300 + , 0x000301 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000313 + , 0x000314 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000301 + , 0x000308 + , 0x000308 + , 0x000306 + , 0x000308 + , 0x000301 + , 0x000300 + , 0x000306 + , 0x000308 + , 0x000306 + , 0x000308 + , 0x000308 + , 0x000300 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000301 + , 0x000308 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x00030B + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000306 + , 0x000308 + , 0x000301 + , 0x000300 + , 0x000306 + , 0x000308 + , 0x000306 + , 0x000308 + , 0x000308 + , 0x000300 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x000301 + , 0x000308 + , 0x000304 + , 0x000306 + , 0x000308 + , 0x00030B + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x00030F + , 0x00030F + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000308 + , 0x000653 + , 0x000654 + , 0x000655 + , 0x000654 + , 0x000654 + , 0x000654 + , 0x000654 + , 0x000654 + , 0x00093C + , 0x00093C + , 0x00093C + , 0x0009BE + , 0x0009D7 + , 0x000B3E + , 0x000B56 + , 0x000B57 + , 0x000BD7 + , 0x000BBE + , 0x000BD7 + , 0x000BBE + , 0x000C56 + , 0x000CD5 + , 0x000CC2 + , 0x000CD5 + , 0x000CD6 + , 0x000CD5 + , 0x000D3E + , 0x000D57 + , 0x000D3E + , 0x000DCA + , 0x000DCF + , 0x000DDF + , 0x000DCA + , 0x00102E + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x001B35 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000304 + , 0x000307 + , 0x000307 + , 0x000302 + , 0x000306 + , 0x000302 + , 0x000306 + , 0x000302 + , 0x000302 + , 0x000302 + , 0x000302 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000345 + , 0x000345 + , 0x000300 + , 0x000301 + , 0x000342 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x000338 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x00309A + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x003099 + , 0x000307 + , 0x000307 + , 0x0110BA + , 0x0110BA + , 0x0110BA + , 0x011127 + , 0x011127 + , 0x01133E + , 0x011357 + , 0x0113C9 + , 0x0113BB + , 0x0113C2 + , 0x0113C9 + , 0x0113B8 + , 0x0113C2 + , 0x0113C9 + , 0x0114B0 + , 0x0114BA + , 0x0114BD + , 0x0115AF + , 0x0115AF + , 0x011930 + , 0x01611E + , 0x01611F + , 0x016120 + , 0x016129 + , 0x01611F + , 0x016120 + , 0x01611F + , 0x01611F + , 0x016D67 + , 0x016D67 + , 0x016D67 + ] + +compResult :: ByteArray +compResult = PBA.byteArrayFromList + [ (0x00226E :: Word32) + , 0x002260 + , 0x00226F + , 0x0000C0 + , 0x0000C1 + , 0x0000C2 + , 0x0000C3 + , 0x000100 + , 0x000102 + , 0x000226 + , 0x0000C4 + , 0x001EA2 + , 0x0000C5 + , 0x0001CD + , 0x000200 + , 0x000202 + , 0x001EA0 + , 0x001E00 + , 0x000104 + , 0x001E02 + , 0x001E04 + , 0x001E06 + , 0x000106 + , 0x000108 + , 0x00010A + , 0x00010C + , 0x0000C7 + , 0x001E0A + , 0x00010E + , 0x001E0C + , 0x001E10 + , 0x001E12 + , 0x001E0E + , 0x0000C8 + , 0x0000C9 + , 0x0000CA + , 0x001EBC + , 0x000112 + , 0x000114 + , 0x000116 + , 0x0000CB + , 0x001EBA + , 0x00011A + , 0x000204 + , 0x000206 + , 0x001EB8 + , 0x000228 + , 0x000118 + , 0x001E18 + , 0x001E1A + , 0x001E1E + , 0x0001F4 + , 0x00011C + , 0x001E20 + , 0x00011E + , 0x000120 + , 0x0001E6 + , 0x000122 + , 0x000124 + , 0x001E22 + , 0x001E26 + , 0x00021E + , 0x001E24 + , 0x001E28 + , 0x001E2A + , 0x0000CC + , 0x0000CD + , 0x0000CE + , 0x000128 + , 0x00012A + , 0x00012C + , 0x000130 + , 0x0000CF + , 0x001EC8 + , 0x0001CF + , 0x000208 + , 0x00020A + , 0x001ECA + , 0x00012E + , 0x001E2C + , 0x000134 + , 0x001E30 + , 0x0001E8 + , 0x001E32 + , 0x000136 + , 0x001E34 + , 0x000139 + , 0x00013D + , 0x001E36 + , 0x00013B + , 0x001E3C + , 0x001E3A + , 0x001E3E + , 0x001E40 + , 0x001E42 + , 0x0001F8 + , 0x000143 + , 0x0000D1 + , 0x001E44 + , 0x000147 + , 0x001E46 + , 0x000145 + , 0x001E4A + , 0x001E48 + , 0x0000D2 + , 0x0000D3 + , 0x0000D4 + , 0x0000D5 + , 0x00014C + , 0x00014E + , 0x00022E + , 0x0000D6 + , 0x001ECE + , 0x000150 + , 0x0001D1 + , 0x00020C + , 0x00020E + , 0x0001A0 + , 0x001ECC + , 0x0001EA + , 0x001E54 + , 0x001E56 + , 0x000154 + , 0x001E58 + , 0x000158 + , 0x000210 + , 0x000212 + , 0x001E5A + , 0x000156 + , 0x001E5E + , 0x00015A + , 0x00015C + , 0x001E60 + , 0x000160 + , 0x001E62 + , 0x000218 + , 0x00015E + , 0x001E6A + , 0x000164 + , 0x001E6C + , 0x00021A + , 0x000162 + , 0x001E70 + , 0x001E6E + , 0x0000D9 + , 0x0000DA + , 0x0000DB + , 0x000168 + , 0x00016A + , 0x00016C + , 0x0000DC + , 0x001EE6 + , 0x00016E + , 0x000170 + , 0x0001D3 + , 0x000214 + , 0x000216 + , 0x0001AF + , 0x001EE4 + , 0x001E72 + , 0x000172 + , 0x001E76 + , 0x001E74 + , 0x001E7C + , 0x001E7E + , 0x001E80 + , 0x001E82 + , 0x000174 + , 0x001E86 + , 0x001E84 + , 0x001E88 + , 0x001E8A + , 0x001E8C + , 0x001EF2 + , 0x0000DD + , 0x000176 + , 0x001EF8 + , 0x000232 + , 0x001E8E + , 0x000178 + , 0x001EF6 + , 0x001EF4 + , 0x000179 + , 0x001E90 + , 0x00017B + , 0x00017D + , 0x001E92 + , 0x001E94 + , 0x0000E0 + , 0x0000E1 + , 0x0000E2 + , 0x0000E3 + , 0x000101 + , 0x000103 + , 0x000227 + , 0x0000E4 + , 0x001EA3 + , 0x0000E5 + , 0x0001CE + , 0x000201 + , 0x000203 + , 0x001EA1 + , 0x001E01 + , 0x000105 + , 0x001E03 + , 0x001E05 + , 0x001E07 + , 0x000107 + , 0x000109 + , 0x00010B + , 0x00010D + , 0x0000E7 + , 0x001E0B + , 0x00010F + , 0x001E0D + , 0x001E11 + , 0x001E13 + , 0x001E0F + , 0x0000E8 + , 0x0000E9 + , 0x0000EA + , 0x001EBD + , 0x000113 + , 0x000115 + , 0x000117 + , 0x0000EB + , 0x001EBB + , 0x00011B + , 0x000205 + , 0x000207 + , 0x001EB9 + , 0x000229 + , 0x000119 + , 0x001E19 + , 0x001E1B + , 0x001E1F + , 0x0001F5 + , 0x00011D + , 0x001E21 + , 0x00011F + , 0x000121 + , 0x0001E7 + , 0x000123 + , 0x000125 + , 0x001E23 + , 0x001E27 + , 0x00021F + , 0x001E25 + , 0x001E29 + , 0x001E2B + , 0x001E96 + , 0x0000EC + , 0x0000ED + , 0x0000EE + , 0x000129 + , 0x00012B + , 0x00012D + , 0x0000EF + , 0x001EC9 + , 0x0001D0 + , 0x000209 + , 0x00020B + , 0x001ECB + , 0x00012F + , 0x001E2D + , 0x000135 + , 0x0001F0 + , 0x001E31 + , 0x0001E9 + , 0x001E33 + , 0x000137 + , 0x001E35 + , 0x00013A + , 0x00013E + , 0x001E37 + , 0x00013C + , 0x001E3D + , 0x001E3B + , 0x001E3F + , 0x001E41 + , 0x001E43 + , 0x0001F9 + , 0x000144 + , 0x0000F1 + , 0x001E45 + , 0x000148 + , 0x001E47 + , 0x000146 + , 0x001E4B + , 0x001E49 + , 0x0000F2 + , 0x0000F3 + , 0x0000F4 + , 0x0000F5 + , 0x00014D + , 0x00014F + , 0x00022F + , 0x0000F6 + , 0x001ECF + , 0x000151 + , 0x0001D2 + , 0x00020D + , 0x00020F + , 0x0001A1 + , 0x001ECD + , 0x0001EB + , 0x001E55 + , 0x001E57 + , 0x000155 + , 0x001E59 + , 0x000159 + , 0x000211 + , 0x000213 + , 0x001E5B + , 0x000157 + , 0x001E5F + , 0x00015B + , 0x00015D + , 0x001E61 + , 0x000161 + , 0x001E63 + , 0x000219 + , 0x00015F + , 0x001E6B + , 0x001E97 + , 0x000165 + , 0x001E6D + , 0x00021B + , 0x000163 + , 0x001E71 + , 0x001E6F + , 0x0000F9 + , 0x0000FA + , 0x0000FB + , 0x000169 + , 0x00016B + , 0x00016D + , 0x0000FC + , 0x001EE7 + , 0x00016F + , 0x000171 + , 0x0001D4 + , 0x000215 + , 0x000217 + , 0x0001B0 + , 0x001EE5 + , 0x001E73 + , 0x000173 + , 0x001E77 + , 0x001E75 + , 0x001E7D + , 0x001E7F + , 0x001E81 + , 0x001E83 + , 0x000175 + , 0x001E87 + , 0x001E85 + , 0x001E98 + , 0x001E89 + , 0x001E8B + , 0x001E8D + , 0x001EF3 + , 0x0000FD + , 0x000177 + , 0x001EF9 + , 0x000233 + , 0x001E8F + , 0x0000FF + , 0x001EF7 + , 0x001E99 + , 0x001EF5 + , 0x00017A + , 0x001E91 + , 0x00017C + , 0x00017E + , 0x001E93 + , 0x001E95 + , 0x001FED + , 0x000385 + , 0x001FC1 + , 0x001EA6 + , 0x001EA4 + , 0x001EAA + , 0x001EA8 + , 0x0001DE + , 0x0001FA + , 0x0001FC + , 0x0001E2 + , 0x001E08 + , 0x001EC0 + , 0x001EBE + , 0x001EC4 + , 0x001EC2 + , 0x001E2E + , 0x001ED2 + , 0x001ED0 + , 0x001ED6 + , 0x001ED4 + , 0x001E4C + , 0x00022C + , 0x001E4E + , 0x00022A + , 0x0001FE + , 0x0001DB + , 0x0001D7 + , 0x0001D5 + , 0x0001D9 + , 0x001EA7 + , 0x001EA5 + , 0x001EAB + , 0x001EA9 + , 0x0001DF + , 0x0001FB + , 0x0001FD + , 0x0001E3 + , 0x001E09 + , 0x001EC1 + , 0x001EBF + , 0x001EC5 + , 0x001EC3 + , 0x001E2F + , 0x001ED3 + , 0x001ED1 + , 0x001ED7 + , 0x001ED5 + , 0x001E4D + , 0x00022D + , 0x001E4F + , 0x00022B + , 0x0001FF + , 0x0001DC + , 0x0001D8 + , 0x0001D6 + , 0x0001DA + , 0x001EB0 + , 0x001EAE + , 0x001EB4 + , 0x001EB2 + , 0x001EB1 + , 0x001EAF + , 0x001EB5 + , 0x001EB3 + , 0x001E14 + , 0x001E16 + , 0x001E15 + , 0x001E17 + , 0x001E50 + , 0x001E52 + , 0x001E51 + , 0x001E53 + , 0x001E64 + , 0x001E65 + , 0x001E66 + , 0x001E67 + , 0x001E78 + , 0x001E79 + , 0x001E7A + , 0x001E7B + , 0x001E9B + , 0x001EDC + , 0x001EDA + , 0x001EE0 + , 0x001EDE + , 0x001EE2 + , 0x001EDD + , 0x001EDB + , 0x001EE1 + , 0x001EDF + , 0x001EE3 + , 0x001EEA + , 0x001EE8 + , 0x001EEE + , 0x001EEC + , 0x001EF0 + , 0x001EEB + , 0x001EE9 + , 0x001EEF + , 0x001EED + , 0x001EF1 + , 0x0001EE + , 0x0001EC + , 0x0001ED + , 0x0001E0 + , 0x0001E1 + , 0x001E1C + , 0x001E1D + , 0x000230 + , 0x000231 + , 0x0001EF + , 0x001FBA + , 0x000386 + , 0x001FB9 + , 0x001FB8 + , 0x001F08 + , 0x001F09 + , 0x001FBC + , 0x001FC8 + , 0x000388 + , 0x001F18 + , 0x001F19 + , 0x001FCA + , 0x000389 + , 0x001F28 + , 0x001F29 + , 0x001FCC + , 0x001FDA + , 0x00038A + , 0x001FD9 + , 0x001FD8 + , 0x0003AA + , 0x001F38 + , 0x001F39 + , 0x001FF8 + , 0x00038C + , 0x001F48 + , 0x001F49 + , 0x001FEC + , 0x001FEA + , 0x00038E + , 0x001FE9 + , 0x001FE8 + , 0x0003AB + , 0x001F59 + , 0x001FFA + , 0x00038F + , 0x001F68 + , 0x001F69 + , 0x001FFC + , 0x001FB4 + , 0x001FC4 + , 0x001F70 + , 0x0003AC + , 0x001FB1 + , 0x001FB0 + , 0x001F00 + , 0x001F01 + , 0x001FB6 + , 0x001FB3 + , 0x001F72 + , 0x0003AD + , 0x001F10 + , 0x001F11 + , 0x001F74 + , 0x0003AE + , 0x001F20 + , 0x001F21 + , 0x001FC6 + , 0x001FC3 + , 0x001F76 + , 0x0003AF + , 0x001FD1 + , 0x001FD0 + , 0x0003CA + , 0x001F30 + , 0x001F31 + , 0x001FD6 + , 0x001F78 + , 0x0003CC + , 0x001F40 + , 0x001F41 + , 0x001FE4 + , 0x001FE5 + , 0x001F7A + , 0x0003CD + , 0x001FE1 + , 0x001FE0 + , 0x0003CB + , 0x001F50 + , 0x001F51 + , 0x001FE6 + , 0x001F7C + , 0x0003CE + , 0x001F60 + , 0x001F61 + , 0x001FF6 + , 0x001FF3 + , 0x001FD2 + , 0x000390 + , 0x001FD7 + , 0x001FE2 + , 0x0003B0 + , 0x001FE7 + , 0x001FF4 + , 0x0003D3 + , 0x0003D4 + , 0x000407 + , 0x0004D0 + , 0x0004D2 + , 0x000403 + , 0x000400 + , 0x0004D6 + , 0x000401 + , 0x0004C1 + , 0x0004DC + , 0x0004DE + , 0x00040D + , 0x0004E2 + , 0x000419 + , 0x0004E4 + , 0x00040C + , 0x0004E6 + , 0x0004EE + , 0x00040E + , 0x0004F0 + , 0x0004F2 + , 0x0004F4 + , 0x0004F8 + , 0x0004EC + , 0x0004D1 + , 0x0004D3 + , 0x000453 + , 0x000450 + , 0x0004D7 + , 0x000451 + , 0x0004C2 + , 0x0004DD + , 0x0004DF + , 0x00045D + , 0x0004E3 + , 0x000439 + , 0x0004E5 + , 0x00045C + , 0x0004E7 + , 0x0004EF + , 0x00045E + , 0x0004F1 + , 0x0004F3 + , 0x0004F5 + , 0x0004F9 + , 0x0004ED + , 0x000457 + , 0x000476 + , 0x000477 + , 0x0004DA + , 0x0004DB + , 0x0004EA + , 0x0004EB + , 0x000622 + , 0x000623 + , 0x000625 + , 0x000624 + , 0x000626 + , 0x0006C2 + , 0x0006D3 + , 0x0006C0 + , 0x000929 + , 0x000931 + , 0x000934 + , 0x0009CB + , 0x0009CC + , 0x000B4B + , 0x000B48 + , 0x000B4C + , 0x000B94 + , 0x000BCA + , 0x000BCC + , 0x000BCB + , 0x000C48 + , 0x000CC0 + , 0x000CCA + , 0x000CC7 + , 0x000CC8 + , 0x000CCB + , 0x000D4A + , 0x000D4C + , 0x000D4B + , 0x000DDA + , 0x000DDC + , 0x000DDE + , 0x000DDD + , 0x001026 + , 0x001B06 + , 0x001B08 + , 0x001B0A + , 0x001B0C + , 0x001B0E + , 0x001B12 + , 0x001B3B + , 0x001B3D + , 0x001B40 + , 0x001B41 + , 0x001B43 + , 0x001E38 + , 0x001E39 + , 0x001E5C + , 0x001E5D + , 0x001E68 + , 0x001E69 + , 0x001EAC + , 0x001EB6 + , 0x001EAD + , 0x001EB7 + , 0x001EC6 + , 0x001EC7 + , 0x001ED8 + , 0x001ED9 + , 0x001F02 + , 0x001F04 + , 0x001F06 + , 0x001F80 + , 0x001F03 + , 0x001F05 + , 0x001F07 + , 0x001F81 + , 0x001F82 + , 0x001F83 + , 0x001F84 + , 0x001F85 + , 0x001F86 + , 0x001F87 + , 0x001F0A + , 0x001F0C + , 0x001F0E + , 0x001F88 + , 0x001F0B + , 0x001F0D + , 0x001F0F + , 0x001F89 + , 0x001F8A + , 0x001F8B + , 0x001F8C + , 0x001F8D + , 0x001F8E + , 0x001F8F + , 0x001F12 + , 0x001F14 + , 0x001F13 + , 0x001F15 + , 0x001F1A + , 0x001F1C + , 0x001F1B + , 0x001F1D + , 0x001F22 + , 0x001F24 + , 0x001F26 + , 0x001F90 + , 0x001F23 + , 0x001F25 + , 0x001F27 + , 0x001F91 + , 0x001F92 + , 0x001F93 + , 0x001F94 + , 0x001F95 + , 0x001F96 + , 0x001F97 + , 0x001F2A + , 0x001F2C + , 0x001F2E + , 0x001F98 + , 0x001F2B + , 0x001F2D + , 0x001F2F + , 0x001F99 + , 0x001F9A + , 0x001F9B + , 0x001F9C + , 0x001F9D + , 0x001F9E + , 0x001F9F + , 0x001F32 + , 0x001F34 + , 0x001F36 + , 0x001F33 + , 0x001F35 + , 0x001F37 + , 0x001F3A + , 0x001F3C + , 0x001F3E + , 0x001F3B + , 0x001F3D + , 0x001F3F + , 0x001F42 + , 0x001F44 + , 0x001F43 + , 0x001F45 + , 0x001F4A + , 0x001F4C + , 0x001F4B + , 0x001F4D + , 0x001F52 + , 0x001F54 + , 0x001F56 + , 0x001F53 + , 0x001F55 + , 0x001F57 + , 0x001F5B + , 0x001F5D + , 0x001F5F + , 0x001F62 + , 0x001F64 + , 0x001F66 + , 0x001FA0 + , 0x001F63 + , 0x001F65 + , 0x001F67 + , 0x001FA1 + , 0x001FA2 + , 0x001FA3 + , 0x001FA4 + , 0x001FA5 + , 0x001FA6 + , 0x001FA7 + , 0x001F6A + , 0x001F6C + , 0x001F6E + , 0x001FA8 + , 0x001F6B + , 0x001F6D + , 0x001F6F + , 0x001FA9 + , 0x001FAA + , 0x001FAB + , 0x001FAC + , 0x001FAD + , 0x001FAE + , 0x001FAF + , 0x001FB2 + , 0x001FC2 + , 0x001FF2 + , 0x001FB7 + , 0x001FCD + , 0x001FCE + , 0x001FCF + , 0x001FC7 + , 0x001FF7 + , 0x001FDD + , 0x001FDE + , 0x001FDF + , 0x00219A + , 0x00219B + , 0x0021AE + , 0x0021CD + , 0x0021CF + , 0x0021CE + , 0x002204 + , 0x002209 + , 0x00220C + , 0x002224 + , 0x002226 + , 0x002241 + , 0x002244 + , 0x002247 + , 0x002249 + , 0x00226D + , 0x002262 + , 0x002270 + , 0x002271 + , 0x002274 + , 0x002275 + , 0x002278 + , 0x002279 + , 0x002280 + , 0x002281 + , 0x0022E0 + , 0x0022E1 + , 0x002284 + , 0x002285 + , 0x002288 + , 0x002289 + , 0x0022E2 + , 0x0022E3 + , 0x0022AC + , 0x0022AD + , 0x0022AE + , 0x0022AF + , 0x0022EA + , 0x0022EB + , 0x0022EC + , 0x0022ED + , 0x003094 + , 0x00304C + , 0x00304E + , 0x003050 + , 0x003052 + , 0x003054 + , 0x003056 + , 0x003058 + , 0x00305A + , 0x00305C + , 0x00305E + , 0x003060 + , 0x003062 + , 0x003065 + , 0x003067 + , 0x003069 + , 0x003070 + , 0x003071 + , 0x003073 + , 0x003074 + , 0x003076 + , 0x003077 + , 0x003079 + , 0x00307A + , 0x00307C + , 0x00307D + , 0x00309E + , 0x0030F4 + , 0x0030AC + , 0x0030AE + , 0x0030B0 + , 0x0030B2 + , 0x0030B4 + , 0x0030B6 + , 0x0030B8 + , 0x0030BA + , 0x0030BC + , 0x0030BE + , 0x0030C0 + , 0x0030C2 + , 0x0030C5 + , 0x0030C7 + , 0x0030C9 + , 0x0030D0 + , 0x0030D1 + , 0x0030D3 + , 0x0030D4 + , 0x0030D6 + , 0x0030D7 + , 0x0030D9 + , 0x0030DA + , 0x0030DC + , 0x0030DD + , 0x0030F7 + , 0x0030F8 + , 0x0030F9 + , 0x0030FA + , 0x0030FE + , 0x0105C9 + , 0x0105E4 + , 0x01109A + , 0x01109C + , 0x0110AB + , 0x01112E + , 0x01112F + , 0x01134B + , 0x01134C + , 0x011383 + , 0x011385 + , 0x01138E + , 0x011391 + , 0x0113C7 + , 0x0113C5 + , 0x0113C8 + , 0x0114BC + , 0x0114BB + , 0x0114BE + , 0x0115BA + , 0x0115BB + , 0x011938 + , 0x016121 + , 0x016123 + , 0x016125 + , 0x016122 + , 0x016126 + , 0x016128 + , 0x016127 + , 0x016124 + , 0x016D69 + , 0x016D68 + , 0x016D6A + ] + + +---------------------------------------------------------------------- +-- Canonical_Combining_Class (CCC) +---------------------------------------------------------------------- + +cccCount :: Int +cccCount = 403 + +cccStarts :: ByteArray +cccStarts = PBA.byteArrayFromList + [ (0x000300 :: Word32) + , 0x000315 + , 0x000316 + , 0x00031A + , 0x00031B + , 0x00031C + , 0x000321 + , 0x000323 + , 0x000327 + , 0x000329 + , 0x000334 + , 0x000339 + , 0x00033D + , 0x000345 + , 0x000346 + , 0x000347 + , 0x00034A + , 0x00034D + , 0x000350 + , 0x000353 + , 0x000357 + , 0x000358 + , 0x000359 + , 0x00035B + , 0x00035C + , 0x00035D + , 0x00035F + , 0x000360 + , 0x000362 + , 0x000363 + , 0x000483 + , 0x000591 + , 0x000592 + , 0x000596 + , 0x000597 + , 0x00059A + , 0x00059B + , 0x00059C + , 0x0005A2 + , 0x0005A8 + , 0x0005AA + , 0x0005AB + , 0x0005AD + , 0x0005AE + , 0x0005AF + , 0x0005B0 + , 0x0005B1 + , 0x0005B2 + , 0x0005B3 + , 0x0005B4 + , 0x0005B5 + , 0x0005B6 + , 0x0005B7 + , 0x0005B8 + , 0x0005B9 + , 0x0005BB + , 0x0005BC + , 0x0005BD + , 0x0005BF + , 0x0005C1 + , 0x0005C2 + , 0x0005C4 + , 0x0005C5 + , 0x0005C7 + , 0x000610 + , 0x000618 + , 0x000619 + , 0x00061A + , 0x00064B + , 0x00064C + , 0x00064D + , 0x00064E + , 0x00064F + , 0x000650 + , 0x000651 + , 0x000652 + , 0x000653 + , 0x000655 + , 0x000657 + , 0x00065C + , 0x00065D + , 0x00065F + , 0x000670 + , 0x0006D6 + , 0x0006DF + , 0x0006E3 + , 0x0006E4 + , 0x0006E7 + , 0x0006EA + , 0x0006EB + , 0x0006ED + , 0x000711 + , 0x000730 + , 0x000731 + , 0x000732 + , 0x000734 + , 0x000735 + , 0x000737 + , 0x00073A + , 0x00073B + , 0x00073D + , 0x00073E + , 0x00073F + , 0x000742 + , 0x000743 + , 0x000744 + , 0x000745 + , 0x000746 + , 0x000747 + , 0x000748 + , 0x000749 + , 0x0007EB + , 0x0007F2 + , 0x0007F3 + , 0x0007FD + , 0x000816 + , 0x00081B + , 0x000825 + , 0x000829 + , 0x000859 + , 0x000897 + , 0x000899 + , 0x00089C + , 0x0008CA + , 0x0008CF + , 0x0008D4 + , 0x0008E3 + , 0x0008E4 + , 0x0008E6 + , 0x0008E7 + , 0x0008E9 + , 0x0008EA + , 0x0008ED + , 0x0008F0 + , 0x0008F1 + , 0x0008F2 + , 0x0008F3 + , 0x0008F6 + , 0x0008F7 + , 0x0008F9 + , 0x0008FB + , 0x00093C + , 0x00094D + , 0x000951 + , 0x000952 + , 0x000953 + , 0x0009BC + , 0x0009CD + , 0x0009FE + , 0x000A3C + , 0x000A4D + , 0x000ABC + , 0x000ACD + , 0x000B3C + , 0x000B4D + , 0x000BCD + , 0x000C3C + , 0x000C4D + , 0x000C55 + , 0x000C56 + , 0x000CBC + , 0x000CCD + , 0x000D3B + , 0x000D4D + , 0x000DCA + , 0x000E38 + , 0x000E3A + , 0x000E48 + , 0x000EB8 + , 0x000EBA + , 0x000EC8 + , 0x000F18 + , 0x000F35 + , 0x000F37 + , 0x000F39 + , 0x000F71 + , 0x000F72 + , 0x000F74 + , 0x000F7A + , 0x000F80 + , 0x000F82 + , 0x000F84 + , 0x000F86 + , 0x000FC6 + , 0x001037 + , 0x001039 + , 0x00108D + , 0x00135D + , 0x001714 + , 0x001734 + , 0x0017D2 + , 0x0017DD + , 0x0018A9 + , 0x001939 + , 0x00193A + , 0x00193B + , 0x001A17 + , 0x001A18 + , 0x001A60 + , 0x001A75 + , 0x001A7F + , 0x001AB0 + , 0x001AB5 + , 0x001ABB + , 0x001ABD + , 0x001ABF + , 0x001AC1 + , 0x001AC3 + , 0x001AC5 + , 0x001ACA + , 0x001ACB + , 0x001ADD + , 0x001AE0 + , 0x001AE6 + , 0x001AE7 + , 0x001AEB + , 0x001B34 + , 0x001B44 + , 0x001B6B + , 0x001B6C + , 0x001B6D + , 0x001BAA + , 0x001BE6 + , 0x001BF2 + , 0x001C37 + , 0x001CD0 + , 0x001CD4 + , 0x001CD5 + , 0x001CDA + , 0x001CDC + , 0x001CE0 + , 0x001CE2 + , 0x001CED + , 0x001CF4 + , 0x001CF8 + , 0x001DC0 + , 0x001DC2 + , 0x001DC3 + , 0x001DCA + , 0x001DCB + , 0x001DCD + , 0x001DCE + , 0x001DCF + , 0x001DD0 + , 0x001DD1 + , 0x001DF6 + , 0x001DF7 + , 0x001DF9 + , 0x001DFA + , 0x001DFB + , 0x001DFC + , 0x001DFD + , 0x001DFE + , 0x001DFF + , 0x0020D0 + , 0x0020D2 + , 0x0020D4 + , 0x0020D8 + , 0x0020DB + , 0x0020E1 + , 0x0020E5 + , 0x0020E7 + , 0x0020E8 + , 0x0020E9 + , 0x0020EA + , 0x0020EC + , 0x0020F0 + , 0x002CEF + , 0x002D7F + , 0x002DE0 + , 0x00302A + , 0x00302B + , 0x00302C + , 0x00302D + , 0x00302E + , 0x003099 + , 0x00A66F + , 0x00A674 + , 0x00A69E + , 0x00A6F0 + , 0x00A806 + , 0x00A82C + , 0x00A8C4 + , 0x00A8E0 + , 0x00A92B + , 0x00A953 + , 0x00A9B3 + , 0x00A9C0 + , 0x00AAB0 + , 0x00AAB2 + , 0x00AAB4 + , 0x00AAB7 + , 0x00AABE + , 0x00AAC1 + , 0x00AAF6 + , 0x00ABED + , 0x00FB1E + , 0x00FE20 + , 0x00FE27 + , 0x00FE2E + , 0x0101FD + , 0x0102E0 + , 0x010376 + , 0x010A0D + , 0x010A0F + , 0x010A38 + , 0x010A39 + , 0x010A3A + , 0x010A3F + , 0x010AE5 + , 0x010AE6 + , 0x010D24 + , 0x010D69 + , 0x010EAB + , 0x010EFA + , 0x010EFD + , 0x010F46 + , 0x010F48 + , 0x010F4B + , 0x010F4C + , 0x010F4D + , 0x010F82 + , 0x010F83 + , 0x010F84 + , 0x010F85 + , 0x011046 + , 0x011070 + , 0x01107F + , 0x0110B9 + , 0x0110BA + , 0x011100 + , 0x011133 + , 0x011173 + , 0x0111C0 + , 0x0111CA + , 0x011235 + , 0x011236 + , 0x0112E9 + , 0x0112EA + , 0x01133B + , 0x01134D + , 0x011366 + , 0x011370 + , 0x0113CE + , 0x011442 + , 0x011446 + , 0x01145E + , 0x0114C2 + , 0x0114C3 + , 0x0115BF + , 0x0115C0 + , 0x01163F + , 0x0116B6 + , 0x0116B7 + , 0x01172B + , 0x011839 + , 0x01183A + , 0x01193D + , 0x011943 + , 0x0119E0 + , 0x011A34 + , 0x011A47 + , 0x011A99 + , 0x011C3F + , 0x011D42 + , 0x011D44 + , 0x011D97 + , 0x011F41 + , 0x01612F + , 0x016AF0 + , 0x016B30 + , 0x016FF0 + , 0x01BC9E + , 0x01D165 + , 0x01D167 + , 0x01D16D + , 0x01D16E + , 0x01D17B + , 0x01D185 + , 0x01D18A + , 0x01D1AA + , 0x01D242 + , 0x01E000 + , 0x01E008 + , 0x01E01B + , 0x01E023 + , 0x01E026 + , 0x01E08F + , 0x01E130 + , 0x01E2AE + , 0x01E2EC + , 0x01E4EC + , 0x01E4EE + , 0x01E4EF + , 0x01E5EE + , 0x01E5EF + , 0x01E6E3 + , 0x01E6E6 + , 0x01E6EE + , 0x01E6F5 + , 0x01E8D0 + , 0x01E944 + , 0x01E94A + ] + +cccEnds :: ByteArray +cccEnds = PBA.byteArrayFromList + [ (0x000314 :: Word32) + , 0x000315 + , 0x000319 + , 0x00031A + , 0x00031B + , 0x000320 + , 0x000322 + , 0x000326 + , 0x000328 + , 0x000333 + , 0x000338 + , 0x00033C + , 0x000344 + , 0x000345 + , 0x000346 + , 0x000349 + , 0x00034C + , 0x00034E + , 0x000352 + , 0x000356 + , 0x000357 + , 0x000358 + , 0x00035A + , 0x00035B + , 0x00035C + , 0x00035E + , 0x00035F + , 0x000361 + , 0x000362 + , 0x00036F + , 0x000487 + , 0x000591 + , 0x000595 + , 0x000596 + , 0x000599 + , 0x00059A + , 0x00059B + , 0x0005A1 + , 0x0005A7 + , 0x0005A9 + , 0x0005AA + , 0x0005AC + , 0x0005AD + , 0x0005AE + , 0x0005AF + , 0x0005B0 + , 0x0005B1 + , 0x0005B2 + , 0x0005B3 + , 0x0005B4 + , 0x0005B5 + , 0x0005B6 + , 0x0005B7 + , 0x0005B8 + , 0x0005BA + , 0x0005BB + , 0x0005BC + , 0x0005BD + , 0x0005BF + , 0x0005C1 + , 0x0005C2 + , 0x0005C4 + , 0x0005C5 + , 0x0005C7 + , 0x000617 + , 0x000618 + , 0x000619 + , 0x00061A + , 0x00064B + , 0x00064C + , 0x00064D + , 0x00064E + , 0x00064F + , 0x000650 + , 0x000651 + , 0x000652 + , 0x000654 + , 0x000656 + , 0x00065B + , 0x00065C + , 0x00065E + , 0x00065F + , 0x000670 + , 0x0006DC + , 0x0006E2 + , 0x0006E3 + , 0x0006E4 + , 0x0006E8 + , 0x0006EA + , 0x0006EC + , 0x0006ED + , 0x000711 + , 0x000730 + , 0x000731 + , 0x000733 + , 0x000734 + , 0x000736 + , 0x000739 + , 0x00073A + , 0x00073C + , 0x00073D + , 0x00073E + , 0x000741 + , 0x000742 + , 0x000743 + , 0x000744 + , 0x000745 + , 0x000746 + , 0x000747 + , 0x000748 + , 0x00074A + , 0x0007F1 + , 0x0007F2 + , 0x0007F3 + , 0x0007FD + , 0x000819 + , 0x000823 + , 0x000827 + , 0x00082D + , 0x00085B + , 0x000898 + , 0x00089B + , 0x00089F + , 0x0008CE + , 0x0008D3 + , 0x0008E1 + , 0x0008E3 + , 0x0008E5 + , 0x0008E6 + , 0x0008E8 + , 0x0008E9 + , 0x0008EC + , 0x0008EF + , 0x0008F0 + , 0x0008F1 + , 0x0008F2 + , 0x0008F5 + , 0x0008F6 + , 0x0008F8 + , 0x0008FA + , 0x0008FF + , 0x00093C + , 0x00094D + , 0x000951 + , 0x000952 + , 0x000954 + , 0x0009BC + , 0x0009CD + , 0x0009FE + , 0x000A3C + , 0x000A4D + , 0x000ABC + , 0x000ACD + , 0x000B3C + , 0x000B4D + , 0x000BCD + , 0x000C3C + , 0x000C4D + , 0x000C55 + , 0x000C56 + , 0x000CBC + , 0x000CCD + , 0x000D3C + , 0x000D4D + , 0x000DCA + , 0x000E39 + , 0x000E3A + , 0x000E4B + , 0x000EB9 + , 0x000EBA + , 0x000ECB + , 0x000F19 + , 0x000F35 + , 0x000F37 + , 0x000F39 + , 0x000F71 + , 0x000F72 + , 0x000F74 + , 0x000F7D + , 0x000F80 + , 0x000F83 + , 0x000F84 + , 0x000F87 + , 0x000FC6 + , 0x001037 + , 0x00103A + , 0x00108D + , 0x00135F + , 0x001715 + , 0x001734 + , 0x0017D2 + , 0x0017DD + , 0x0018A9 + , 0x001939 + , 0x00193A + , 0x00193B + , 0x001A17 + , 0x001A18 + , 0x001A60 + , 0x001A7C + , 0x001A7F + , 0x001AB4 + , 0x001ABA + , 0x001ABC + , 0x001ABD + , 0x001AC0 + , 0x001AC2 + , 0x001AC4 + , 0x001AC9 + , 0x001ACA + , 0x001ADC + , 0x001ADD + , 0x001AE5 + , 0x001AE6 + , 0x001AEA + , 0x001AEB + , 0x001B34 + , 0x001B44 + , 0x001B6B + , 0x001B6C + , 0x001B73 + , 0x001BAB + , 0x001BE6 + , 0x001BF3 + , 0x001C37 + , 0x001CD2 + , 0x001CD4 + , 0x001CD9 + , 0x001CDB + , 0x001CDF + , 0x001CE0 + , 0x001CE8 + , 0x001CED + , 0x001CF4 + , 0x001CF9 + , 0x001DC1 + , 0x001DC2 + , 0x001DC9 + , 0x001DCA + , 0x001DCC + , 0x001DCD + , 0x001DCE + , 0x001DCF + , 0x001DD0 + , 0x001DF5 + , 0x001DF6 + , 0x001DF8 + , 0x001DF9 + , 0x001DFA + , 0x001DFB + , 0x001DFC + , 0x001DFD + , 0x001DFE + , 0x001DFF + , 0x0020D1 + , 0x0020D3 + , 0x0020D7 + , 0x0020DA + , 0x0020DC + , 0x0020E1 + , 0x0020E6 + , 0x0020E7 + , 0x0020E8 + , 0x0020E9 + , 0x0020EB + , 0x0020EF + , 0x0020F0 + , 0x002CF1 + , 0x002D7F + , 0x002DFF + , 0x00302A + , 0x00302B + , 0x00302C + , 0x00302D + , 0x00302F + , 0x00309A + , 0x00A66F + , 0x00A67D + , 0x00A69F + , 0x00A6F1 + , 0x00A806 + , 0x00A82C + , 0x00A8C4 + , 0x00A8F1 + , 0x00A92D + , 0x00A953 + , 0x00A9B3 + , 0x00A9C0 + , 0x00AAB0 + , 0x00AAB3 + , 0x00AAB4 + , 0x00AAB8 + , 0x00AABF + , 0x00AAC1 + , 0x00AAF6 + , 0x00ABED + , 0x00FB1E + , 0x00FE26 + , 0x00FE2D + , 0x00FE2F + , 0x0101FD + , 0x0102E0 + , 0x01037A + , 0x010A0D + , 0x010A0F + , 0x010A38 + , 0x010A39 + , 0x010A3A + , 0x010A3F + , 0x010AE5 + , 0x010AE6 + , 0x010D27 + , 0x010D6D + , 0x010EAC + , 0x010EFB + , 0x010EFF + , 0x010F47 + , 0x010F4A + , 0x010F4B + , 0x010F4C + , 0x010F50 + , 0x010F82 + , 0x010F83 + , 0x010F84 + , 0x010F85 + , 0x011046 + , 0x011070 + , 0x01107F + , 0x0110B9 + , 0x0110BA + , 0x011102 + , 0x011134 + , 0x011173 + , 0x0111C0 + , 0x0111CA + , 0x011235 + , 0x011236 + , 0x0112E9 + , 0x0112EA + , 0x01133C + , 0x01134D + , 0x01136C + , 0x011374 + , 0x0113D0 + , 0x011442 + , 0x011446 + , 0x01145E + , 0x0114C2 + , 0x0114C3 + , 0x0115BF + , 0x0115C0 + , 0x01163F + , 0x0116B6 + , 0x0116B7 + , 0x01172B + , 0x011839 + , 0x01183A + , 0x01193E + , 0x011943 + , 0x0119E0 + , 0x011A34 + , 0x011A47 + , 0x011A99 + , 0x011C3F + , 0x011D42 + , 0x011D45 + , 0x011D97 + , 0x011F42 + , 0x01612F + , 0x016AF4 + , 0x016B36 + , 0x016FF1 + , 0x01BC9E + , 0x01D166 + , 0x01D169 + , 0x01D16D + , 0x01D172 + , 0x01D182 + , 0x01D189 + , 0x01D18B + , 0x01D1AD + , 0x01D244 + , 0x01E006 + , 0x01E018 + , 0x01E021 + , 0x01E024 + , 0x01E02A + , 0x01E08F + , 0x01E136 + , 0x01E2AE + , 0x01E2EF + , 0x01E4ED + , 0x01E4EE + , 0x01E4EF + , 0x01E5EE + , 0x01E5EF + , 0x01E6E3 + , 0x01E6E6 + , 0x01E6EF + , 0x01E6F5 + , 0x01E8D6 + , 0x01E949 + , 0x01E94A + ] + +cccValues :: ByteArray +cccValues = PBA.byteArrayFromList + [ (230 :: Word8) + , 232 + , 220 + , 232 + , 216 + , 220 + , 202 + , 220 + , 202 + , 220 + , 1 + , 220 + , 230 + , 240 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 232 + , 220 + , 230 + , 233 + , 234 + , 233 + , 234 + , 233 + , 230 + , 230 + , 220 + , 230 + , 220 + , 230 + , 222 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 222 + , 228 + , 230 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 230 + , 220 + , 18 + , 230 + , 30 + , 31 + , 32 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 35 + , 230 + , 230 + , 220 + , 230 + , 230 + , 220 + , 230 + , 220 + , 36 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 230 + , 220 + , 230 + , 220 + , 230 + , 230 + , 230 + , 230 + , 220 + , 230 + , 220 + , 230 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 27 + , 28 + , 29 + , 230 + , 220 + , 230 + , 220 + , 230 + , 7 + , 9 + , 230 + , 220 + , 230 + , 7 + , 9 + , 230 + , 7 + , 9 + , 7 + , 9 + , 7 + , 9 + , 9 + , 7 + , 9 + , 84 + , 91 + , 7 + , 9 + , 9 + , 9 + , 9 + , 103 + , 9 + , 107 + , 118 + , 9 + , 122 + , 220 + , 220 + , 220 + , 216 + , 129 + , 130 + , 132 + , 130 + , 130 + , 230 + , 9 + , 230 + , 220 + , 7 + , 9 + , 220 + , 230 + , 9 + , 9 + , 9 + , 230 + , 228 + , 222 + , 230 + , 220 + , 230 + , 220 + , 9 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 234 + , 7 + , 9 + , 230 + , 220 + , 230 + , 9 + , 7 + , 9 + , 7 + , 230 + , 1 + , 220 + , 230 + , 220 + , 230 + , 1 + , 220 + , 230 + , 230 + , 230 + , 220 + , 230 + , 220 + , 230 + , 234 + , 214 + , 220 + , 202 + , 230 + , 232 + , 228 + , 220 + , 218 + , 230 + , 233 + , 220 + , 230 + , 220 + , 230 + , 1 + , 230 + , 1 + , 230 + , 230 + , 1 + , 230 + , 220 + , 230 + , 1 + , 220 + , 230 + , 230 + , 9 + , 230 + , 218 + , 228 + , 232 + , 222 + , 224 + , 8 + , 230 + , 230 + , 230 + , 230 + , 9 + , 9 + , 9 + , 230 + , 220 + , 9 + , 7 + , 9 + , 230 + , 230 + , 220 + , 230 + , 230 + , 230 + , 9 + , 9 + , 26 + , 230 + , 220 + , 230 + , 220 + , 220 + , 230 + , 220 + , 230 + , 230 + , 1 + , 220 + , 9 + , 230 + , 220 + , 230 + , 230 + , 230 + , 220 + , 220 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 230 + , 220 + , 9 + , 9 + , 9 + , 9 + , 7 + , 230 + , 9 + , 7 + , 9 + , 7 + , 9 + , 7 + , 7 + , 9 + , 7 + , 9 + , 230 + , 230 + , 9 + , 9 + , 7 + , 230 + , 9 + , 7 + , 9 + , 7 + , 9 + , 9 + , 7 + , 9 + , 9 + , 7 + , 9 + , 7 + , 9 + , 9 + , 9 + , 9 + , 9 + , 7 + , 9 + , 9 + , 9 + , 9 + , 1 + , 230 + , 6 + , 1 + , 216 + , 1 + , 226 + , 216 + , 220 + , 230 + , 220 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 230 + , 232 + , 220 + , 230 + , 230 + , 220 + , 230 + , 230 + , 230 + , 230 + , 220 + , 230 + , 7 + ] diff --git a/internal/Text/IDNA2008/Internal/Parse.hs b/internal/Text/IDNA2008/Internal/Parse.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Parse.hs @@ -0,0 +1,2401 @@ +-- | +-- Module : Text.IDNA2008.Internal.Parse +-- Description : IDNA-aware parser from presentation Text/UTF-8 to wire form +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Top-level parser that walks a UTF-8 byte view of a presentation-form +-- domain name, splits it into labels (handling DNS-style backslash +-- escapes), classifies each label into one of the 'LabelForm' singletons, +-- and writes the wire form into a freshly allocated 'Domain'. +-- +-- The parser shares a single output buffer and a single per-call +-- codepoint accumulator across all labels; allocations are constant in +-- the number of labels. +-- +-- IDNA validation: each U-label codepoint is looked up in the IANA +-- IDNA2008 derived-property table (see "Text.IDNA2008.Internal.Property") +-- and rejected unless its disposition is @PVALID@. @CONTEXTO@ +-- codepoints are admitted when 'checkContextO' accepts their context +-- (all of RFC 5892 Appendix A.3-A.9 are implemented). @CONTEXTJ@ +-- codepoints are admitted when 'checkContextJ' accepts their context +-- (RFC 5892 Appendix A.1\/A.2). +-- +-- With the 'NFCCHECK' flag, checks that the decoded codepoints are in NFC +-- (composed) form. With the 'BIDICHECK' flag, adds the per-label subset of +-- RFC 5893 Rules 1-6; the cross-label part of the rule set is treated as a +-- presentation-time concern and intentionally not enforced in the parser. +-- +-- Classification precedence: any codepoint @> 0xFF@ forces the U-label +-- path (since such a codepoint cannot fit in an octet); otherwise, any +-- non-LDH ASCII byte prefers the OCTET path (since the resulting +-- Punycode-encoded form would not be a valid LDH A-label); pure +-- Latin-1 input (codepoints @0x80..0xFF@ with no non-LDH ASCII) still +-- goes through the U-label path. +-- +-- A-label classification: under 'ALABELCHECK' (set in +-- 'defaultIdnaFlags'), an ACE-prefixed LDH label is reported +-- as 'ALABEL' only if its Punycode body decodes to a valid IDN label +-- and re-encodes to the same input bytes; on failure the label is +-- 'FAKEA' (when 'FAKEA' is in the caller's 'LabelFormSet') or +-- rejected with the underlying 'AceReason'. Without 'ALABELCHECK', +-- every syntactically valid ACE-prefixed LDH label is reported +-- as 'ALABEL'. +-- +-- U-label classification: a non-ASCII label that passes strict +-- IDNA2008 validation is reported as 'ULABEL'. When strict +-- validation fails (disposition, contextual rule, NFC, hyphen, +-- joining, or Bidi), the label is reported as 'LAXULABEL' if +-- 'LAXULABEL' is in the caller's 'LabelFormSet', else rejected. +-- 'LAXULABEL' and 'FAKEA' are the permissive companions to +-- 'ULABEL' and 'ALABEL': they admit labels that fail strict +-- validation in their respective input shapes. +{-# LANGUAGE DeriveLift #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE TemplateHaskell #-} + +module Text.IDNA2008.Internal.Parse + ( -- * Domain type + Domain(Domain) + , isValidWireForm + , toLabels + , wireBytes + , wireBytesShort + + -- * Domain parsers + , parseDomain + , parseDomainOpts + , parseDomainUtf8 + , parseDomainShort + + -- * Domain helpers (default options) + , mkDomain + , mkDomainStr + , mkDomainUtf8 + , mkDomainShort + + -- * Literal domains (TH splices) + , dnLit + + -- * Domain display forms + , domainToAscii + , domainToUnicode + , labelToAscii + , labelToUnicode + , unparseDomainOpts + , unparseLabelOpts + ) where + +import qualified Data.ByteString.Short as SBS +import qualified Data.Primitive.ByteArray as A +import qualified Data.Text as T +import qualified Data.Text.Array as TA +import qualified Language.Haskell.TH.Syntax as TH +import qualified Text.IDNA2008.Internal.Case as Case +import Text.IDNA2008.Internal.UTS46 (uts46Lookup) +import Control.Monad (when) +import Control.Monad.ST (ST, runST) +import Data.Array.Byte (ByteArray(..)) +import Data.Bits ((.|.), (.&.), testBit, unsafeShiftL, unsafeShiftR) +import Data.ByteString (ByteString) +import Data.ByteString.Short.Internal (ShortByteString(..)) +import Data.Char (chr, ord) +import Data.Foldable (foldMap') +import Data.Primitive.ByteArray + ( MutableByteArray + , copyMutableByteArray + , indexByteArray + , newByteArray + , readByteArray + , resizeMutableByteArray + , shrinkMutableByteArray + , unsafeFreezeByteArray + , writeByteArray + ) +import Data.Primitive.PrimArray + ( MutablePrimArray + , PrimArray + , indexPrimArray + , newPrimArray + , readPrimArray + , sizeofPrimArray + , unsafeFreezePrimArray + , writePrimArray + ) +import Data.Text.Internal (Text(Text)) +import Data.Word (Word8, Word64) + +import Text.IDNA2008.Internal.Bidi + ( BidiSummary + , addBidiTrigger + , bidiClassCp + , buildBidiSummary + , checkBidiSummary + , emptyBidiSummary + , extendBidiSummary + ) +import Text.IDNA2008.Internal.Combining (isCombiningMark) +import Text.IDNA2008.Internal.Flags + ( IdnaFlags(..) + , defaultIdnaFlags + , effectiveIdnaFlags + , meetsIdnaFlags + , withoutIdnaFlags + ) +import Text.IDNA2008.Internal.Emoji (isEmoji, isMappedEmoji) +import Text.IDNA2008.Internal.Error +import Text.IDNA2008.Internal.LabelForm + ( LabelForm + , pattern LDH, pattern RLDH, pattern FAKEA, pattern ALABEL + , pattern ULABEL, pattern LAXULABEL, pattern ATTRLEAF, pattern OCTET + , pattern WILDLABEL + ) +import Text.IDNA2008.Internal.LabelFormSet + ( LabelFormSet, labelFormToSet, memberLabelFormSet + , withoutLabelFormSet, idnLabelForms ) +import Text.IDNA2008.Internal.LabelInfo + ( LabelInfo, mkLabelInfo ) +import Text.IDNA2008.Internal.Property + ( IdnaDisposition(..), idnaDisposition ) +import Text.IDNA2008.Internal.Joining + ( isVirama + , jtIsLeftOrDual + , jtIsRightOrDual + , jtIsTransparent + ) +import Text.IDNA2008.Internal.NFC (isNFC, normalizeNFC) +import Text.IDNA2008.Internal.Punycode +import Text.IDNA2008.Internal.Script + ( isGreekCp + , isHebrewCp + , isHkhCp + ) +import Text.IDNA2008.Internal.Width (widthMapCp) + +---------------------------------------------------------------------- +-- Buffer sizes +---------------------------------------------------------------------- + +-- | Maximum wire form length, RFC 1035 section 3.1. +-- +-- Encoding of domain names allocates a buffer of this size +-- shrinking it to the actual size used when done. +maxWireLen :: Int +maxWireLen = 255 + +-- | Maximum wire octets in a single label. +maxLabelLen :: Int +maxLabelLen = 63 + +-- | Maximum codepoints we admit in a single label before encoding. +-- +-- This is the also the size of the mutable codepoint array given +-- to the Punycode decoder. The number of decoded codepoints +-- can't exceed the number of input bytes. +maxCpsPerLabel :: Int +maxCpsPerLabel = maxLabelLen + +-- | Output buffer capacity. +outBufSize :: Int +outBufSize = maxWireLen + 1 + +---------------------------------------------------------------------- +-- Per-label statistics (computed once at flush time from cpBuf). +---------------------------------------------------------------------- + +data LabelStat = LabelStat + { lsHasGt0xFF :: !Bool -- any codepoint > 0xFF + , lsHasNonAsciiCp :: !Bool -- any codepoint > 0x7F + , lsAllLdh :: !Bool -- every codepoint is LDH-ASCII + , lsLdhPastFirst :: !Bool -- every codepoint at index >= 1 is LDH-ASCII + , lsHasNonLdhAscii :: !Bool -- any codepoint < 0x80 that is non-LDH + , lsLeadingHyphen :: !Bool + , lsTrailingHyphen :: !Bool + , lsHy34 :: !Bool -- '-' at positions 3 AND 4 + , lsLeadingUnder :: !Bool -- first codepoint is '_' + , lsXnPrefix :: !Bool -- first two cps are 'x'/'X', 'n'/'N' + , lsIsWild :: !Bool -- the entire label is the single byte '*' + } + +---------------------------------------------------------------------- +-- Public entry points +---------------------------------------------------------------------- + +-- | Parse a 'Text' as a presentation-form domain name with the +-- strict default parser options ('defaultIdnaFlags'). See +-- 'parseDomainOpts' to override the flag set. +parseDomain + :: LabelFormSet -- ^ Permitted label forms + -> Text -- ^ Presentation form + -> Either IdnaError (Domain, LabelInfo) +parseDomain !allowed = parseDomainOpts allowed defaultIdnaFlags + +-- | Like 'parseDomain' but with a caller-supplied 'IdnaFlags' record. +parseDomainOpts + :: LabelFormSet -- ^ Permitted label forms + -> IdnaFlags -- ^ Parser options + -> Text -- ^ Presentation form + -> Either IdnaError (Domain, LabelInfo) +parseDomainOpts !allowed !opts (Text arr off len) = + parseDomainView (textArrayToByteArray arr) off len allowed opts + where + textArrayToByteArray :: TA.Array -> ByteArray + textArrayToByteArray (TA.ByteArray b) = ByteArray b + +-- | Parse a UTF-8 'ByteString' as a presentation-form domain name. +-- Ill-formed UTF-8 yields 'ErrInvalidUtf8'. +parseDomainUtf8 + :: LabelFormSet -- ^ Permitted label forms + -> IdnaFlags -- ^ Parser options + -> ByteString -- ^ UTF-8 presentation form + -> Either IdnaError (Domain, LabelInfo) +parseDomainUtf8 !allowed !flags !bs = + parseDomainShort allowed flags $! SBS.toShort bs + +-- | Parse a UTF-8 'ShortByteString' as a presentation-form domain +-- name. Ill-formed UTF-8 yields 'ErrInvalidUtf8'. +parseDomainShort + :: LabelFormSet -- ^ Permitted label forms + -> IdnaFlags -- ^ Parser options + -> ShortByteString -- ^ UTF-8 presentation form + -> Either IdnaError (Domain, LabelInfo) +parseDomainShort !allowed !flags sb@(SBS sba) = + parseDomainView (ByteArray sba) 0 (SBS.length sb) allowed flags + +-- | Core view-based entry point. Returns the parsed 'Domain' and a +-- per-label 'LabelInfo' classification result; fails if the input is +-- malformed or if any label's classification is not in the @allowed@ +-- set. +parseDomainView + :: ByteArray -- ^ Input UTF-8 bytes + -> Int -- ^ Slice start + -> Int -- ^ Slice length + -> LabelFormSet -- ^ Permitted label forms + -> IdnaFlags + -> Either IdnaError (Domain, LabelInfo) +parseDomainView !input !inOff !inLen !allowed !opts0 + | inLen < 0 = Left (ErrInvalidUtf8 (-1) Nothing) + | otherwise = runST do + outBuf <- newByteArray outBufSize + cpBuf <- newPrimArray maxCpsPerLabel + let !inEnd = inOff + inLen + !opts = effectiveIdnaFlags opts0 + res <- driver input cpBuf outBuf inEnd allowed opts + inOff 0 0 0 False 0 [] + case res of + Left e -> pure (Left e) + Right (outLen, produced) -> do + let !finalLen = outLen + 1 + if finalLen > maxWireLen + then pure (Left (ErrNameTooLong finalLen)) + else do + writeByteArray outBuf outLen (0 :: Word8) + resBA <- newByteArray finalLen + copyMutableByteArray resBA 0 outBuf 0 finalLen + (ByteArray frozen) <- unsafeFreezeByteArray resBA + let !info = mkLabelInfo (reverse produced) + -- The parser builds the wire form by construction + -- (length bytes capped at 63, trailing NUL appended, + -- total length <= maxWireLen). Use the bare data + -- constructor to skip the redundant validation that + -- the 'Domain' pattern would perform. + pure (Right (Domain_ (SBS frozen), info)) + +---------------------------------------------------------------------- +-- Driver +---------------------------------------------------------------------- + +-- | Codepoints that the 'MAPDOTS' option treats as additional label +-- separators alongside @U+002E@. See 'MAPDOTS' for the rationale. +isExtraDotCp :: Int -> Bool +isExtraDotCp !cp = + cp == 0x3002 -- IDEOGRAPHIC FULL STOP + || cp == 0xFF0E -- FULLWIDTH FULL STOP + || cp == 0xFF61 -- HALFWIDTH IDEOGRAPHIC FULL STOP +{-# INLINE isExtraDotCp #-} + +driver + :: ByteArray + -> MutablePrimArray s Int + -> MutableByteArray s + -> Int -- inEnd + -> LabelFormSet -- allowed + -> IdnaFlags + -> Int -- iPos + -> Int -- oPos + -> Int -- lStart + -> Int -- cpCount + -> Bool -- hasEscape + -> Int -- lIdx + -> [LabelForm] -- producedForms + -> ST s (Either IdnaError (Int, [LabelForm])) +driver !input !cpBuf !outBuf !inEnd !allowed !opts = go + where + go !iPos !oPos !lStart !cpCount !hasEscape !lIdx !produced + | iPos >= inEnd = + if cpCount == 0 + then pure (Right (oPos, produced)) + else flushLabel input cpBuf outBuf allowed opts + lStart cpCount hasEscape lIdx produced + | otherwise = do + let !b = indexByteArray input iPos :: Word8 + if | b == 0x5C -> + handleEscape iPos oPos lStart cpCount lIdx produced + | b == 0x2E -> + handleDot (iPos + 1) oPos lStart cpCount + hasEscape lIdx produced + | b < 0x80 -> + appendCp (fromIntegral b :: Int) (iPos + 1) + oPos lStart cpCount hasEscape lIdx produced + | otherwise -> + case decodeUtf8At input inEnd iPos of + Left e -> pure (Left e) + Right (cp, nextPos) + | MAPDOTS `meetsIdnaFlags` opts, isExtraDotCp cp -> + handleDot nextPos oPos lStart cpCount + hasEscape lIdx produced + | otherwise -> + appendCp cp nextPos oPos lStart cpCount + hasEscape lIdx produced + + appendCp !cp !iPos !oPos !lStart !cpCount !hasEscape !lIdx !produced + | cpCount >= maxCpsPerLabel = + pure (Left (ErrLabelTooLong lIdx (cpCount + 1))) + | otherwise = do + writePrimArray cpBuf cpCount cp + go iPos oPos lStart (cpCount + 1) hasEscape lIdx produced + + handleEscape !iPos !oPos !lStart !cpCount !lIdx !produced + | iPos + 1 >= inEnd = + pure (Left (ErrBadEscape lIdx (Just iPos))) + | otherwise = + let !b1 = indexByteArray input (iPos + 1) :: Word8 + in case asciiDigit b1 of + Just !v1 + -- \DDD numeric escape + | iPos + 4 > inEnd -> + pure (Left (ErrBadEscape lIdx (Just iPos))) + | otherwise -> + let !b2 = indexByteArray input (iPos + 2) :: Word8 + !b3 = indexByteArray input (iPos + 3) :: Word8 + in case (asciiDigit b2, asciiDigit b3) of + (Just v2, Just v3) -> + let !v = 100 * fromIntegral v1 + + 10 * fromIntegral v2 + + fromIntegral v3 :: Int + in if v > 0xFF + then pure (Left (ErrBadEscape lIdx (Just iPos))) + else + appendCp v (iPos + 4) oPos lStart cpCount + True lIdx produced + _ -> pure (Left (ErrBadEscape lIdx (Just iPos))) + Nothing + | b1 < 0x80 -> + appendCp (fromIntegral b1) (iPos + 2) + oPos lStart cpCount True lIdx produced + | otherwise -> + case decodeUtf8At input inEnd (iPos + 1) of + Left e -> pure (Left e) + Right (cp, nextPos) + | cp > 0xFF -> + pure (Left (ErrBadEscape lIdx (Just iPos))) + | otherwise -> + appendCp cp nextPos oPos lStart cpCount True + lIdx produced + + handleDot !nextPos !oPos !lStart !cpCount !hasEscape !lIdx !produced + | cpCount == 0 = + -- Empty label. Always an error here, with one special case: + -- the input is exactly a single separator codepoint, which + -- represents the root domain. We detect that as: no labels + -- emitted yet (oPos == 0, lIdx == 0) and this separator is + -- the last codepoint in the input. Trailing separators + -- after a real label are handled by the end-of-input branch + -- (where cpCount has already been reset to 0 by a previous + -- flush), so they never reach this function. + if oPos == 0 && lIdx == 0 && nextPos == inEnd + then pure (Right (oPos, produced)) + else pure (Left (ErrEmptyLabel lIdx)) + | otherwise = do + r <- flushLabel input cpBuf outBuf allowed opts + lStart cpCount hasEscape lIdx produced + case r of + Left e -> pure (Left e) + Right (oPos', produced') -> + go nextPos oPos' oPos' 0 False (lIdx + 1) produced' + +---------------------------------------------------------------------- +-- UTF-8 decoder +---------------------------------------------------------------------- + +-- | Decode the UTF-8 sequence starting at byte offset @p@. Returns +-- @(codepoint, advance)@ or an error. The @lIdx@ for error location +-- isn't threaded through; we use @-1@ here since the byte offset alone +-- already pinpoints the issue. +decodeUtf8At + :: ByteArray -- input + -> Int -- inEnd + -> Int -- p (offset to read) + -> Either IdnaError (Int, Int) +decodeUtf8At !input !inEnd !p + | p >= inEnd = Left (ErrInvalidUtf8 (-1) (Just p)) + | otherwise = + let !b0 = indexByteArray input p :: Word8 + in if | b0 < 0x80 -> Right (fromIntegral b0, p + 1) + | b0 < 0xC2 -> Left (ErrInvalidUtf8 (-1) (Just p)) + | b0 < 0xE0 -> two p b0 + | b0 < 0xF0 -> three p b0 + | b0 < 0xF5 -> four p b0 + | otherwise -> Left (ErrInvalidUtf8 (-1) (Just p)) + where + contBad b = b < 0x80 || b >= 0xC0 + + two !p0 !b0 + | p0 + 1 >= inEnd = + Left (ErrInvalidUtf8 (-1) (Just p0)) + | otherwise = + let !b1 = indexByteArray input (p0 + 1) :: Word8 + in if contBad b1 + then Left (ErrInvalidUtf8 (-1) (Just p0)) + else + let !cp = ((fromIntegral b0 - 0xC0) `unsafeShiftL` 6) + .|. (fromIntegral b1 - 0x80) :: Int + in Right (cp, p0 + 2) + + three !p0 !b0 + | p0 + 2 >= inEnd = + Left (ErrInvalidUtf8 (-1) (Just p0)) + | otherwise = + let !b1 = indexByteArray input (p0 + 1) :: Word8 + !b2 = indexByteArray input (p0 + 2) :: Word8 + in if contBad b1 || contBad b2 + then Left (ErrInvalidUtf8 (-1) (Just p0)) + else + let !cp = ((fromIntegral b0 - 0xE0) `unsafeShiftL` 12) + .|. ((fromIntegral b1 - 0x80) `unsafeShiftL` 6) + .|. (fromIntegral b2 - 0x80) :: Int + in if cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF) + then Left (ErrInvalidUtf8 (-1) (Just p0)) + else Right (cp, p0 + 3) + + four !p0 !b0 + | p0 + 3 >= inEnd = + Left (ErrInvalidUtf8 (-1) (Just p0)) + | otherwise = + let !b1 = indexByteArray input (p0 + 1) :: Word8 + !b2 = indexByteArray input (p0 + 2) :: Word8 + !b3 = indexByteArray input (p0 + 3) :: Word8 + in if contBad b1 || contBad b2 || contBad b3 + then Left (ErrInvalidUtf8 (-1) (Just p0)) + else + let !cp = ((fromIntegral b0 - 0xF0) `unsafeShiftL` 18) + .|. ((fromIntegral b1 - 0x80) `unsafeShiftL` 12) + .|. ((fromIntegral b2 - 0x80) `unsafeShiftL` 6) + .|. (fromIntegral b3 - 0x80) :: Int + in if cp < 0x10000 || cp > 0x10FFFF + then Left (ErrInvalidUtf8 (-1) (Just p0)) + else Right (cp, p0 + 4) + +---------------------------------------------------------------------- +-- Label flushing: classify, validate, emit. +-- +-- Decision tree, in order: +-- +-- 1. Escape produced any byte: OCTET (defensive: gt0xFF here is +-- impossible since \DDD is range-checked and \C requires cp <= 0xFF, +-- so a non-LDH ASCII byte that came from an escape simply lands +-- in OCTET). +-- +-- 2. Any codepoint > 0xFF: U-label path forced. Such a codepoint +-- cannot fit in a single octet, so neither OCTET nor LDH is an +-- option; failure to validate as a U-label is the only way out. +-- +-- 3. All-LDH and not bordered by hyphens: LDH (or RLDH/ALABEL via +-- the hy34 + xn prefix discriminator). +-- +-- 4. Leading underscore with the rest LDH: ATTRLEAF. +-- +-- 5. Has non-ASCII codepoints (0x80..0xFF) AND no non-LDH ASCII +-- byte: U-label path. Pure Latin-1 input. +-- +-- 6. Otherwise: OCTET. Catches "non-LDH ASCII present but no +-- codepoint > 0xFF" -- a label that cannot form a valid A-label +-- and is byte-fittable, so OCTET semantics apply. +---------------------------------------------------------------------- + +flushLabel + :: ByteArray -- input (unused; kept for future error context) + -> MutablePrimArray s Int -- cpBuf + -> MutableByteArray s -- outBuf + -> LabelFormSet -- allowed + -> IdnaFlags + -> Int -- lStart + -> Int -- cpCount + -> Bool -- hasEscape + -> Int -- lIdx + -> [LabelForm] -- producedForms accumulator + -> ST s (Either IdnaError (Int, [LabelForm])) +flushLabel !_input !cpBuf !outBuf !allowed !opts + !lStart !cpCount !hasEscape !lIdx !produced + | cpCount == 0 = pure (Left (ErrEmptyLabel lIdx)) + | otherwise = do + -- RFC 5895 section 2.1 mapping (ASCII portion): apply unconditional + -- lower-casing of A-Z to z+0x20 before classification. This + -- doesn't shift any label between forms (uppercase letters + -- are already part of the LDH alphabet for classification + -- purposes; cf. 'isLdhCp'), but it does ensure the wire form + -- emits lowercase ASCII regardless of input case. Unicode + -- toLower for non-ASCII codepoints is applied later, only on + -- the U-label path (see 'uLabelPath'). + when (MAPCASE `meetsIdnaFlags` opts) + (asciiDownCaseBuf cpBuf cpCount) + stat <- scanCpBuf cpBuf cpCount + -- WILDLABEL is a property of the wire content (the single byte + -- '*', 0x2A), not of the input spelling: '*', '\*', and '\042' + -- all produce the same wire byte and all classify the same way. + if lsIsWild stat + then commitBytes cpBuf outBuf allowed + lStart cpCount lIdx produced WILDLABEL + else if hasEscape + then if lsHasGt0xFF stat + -- Escape syntax (signalling OCTET-form + -- intent) appears alongside codepoints > + -- 0xFF that an OCTET label can't carry. + -- The label has no single coherent + -- presentation form; see 'ErrUnpresentableLabel'. + then pure (Left (ErrUnpresentableLabel lIdx)) + else commitBytes cpBuf outBuf allowed + lStart cpCount lIdx produced OCTET + else if lsHasGt0xFF stat + then uLabelPath cpBuf outBuf allowed opts lStart cpCount lIdx + produced + else if lsAllLdh stat + && not (lsLeadingHyphen stat || lsTrailingHyphen stat) + then if lsHy34 stat + then ldhRldhPath cpBuf outBuf allowed opts + lStart cpCount lIdx produced stat + else commitBytes cpBuf outBuf allowed + lStart cpCount lIdx produced LDH + else if lsLeadingUnder stat + && lsLdhPastFirst stat + && not (lsTrailingHyphen stat) + then commitBytes cpBuf outBuf allowed + lStart cpCount lIdx produced ATTRLEAF + else if lsHasNonAsciiCp stat + && not (lsHasNonLdhAscii stat) + then uLabelPath cpBuf outBuf allowed opts lStart cpCount lIdx + produced + else commitBytes cpBuf outBuf allowed + lStart cpCount lIdx produced OCTET + +-- | Scan cpBuf computing label characteristics in one pass. +scanCpBuf :: MutablePrimArray s Int -> Int -> ST s LabelStat +scanCpBuf !cpBuf !cnt = do + base <- scanBody 0 (LabelStat False False True True False + False False False False False False) + -- Boundary-position flags. + first <- if cnt >= 1 then readPrimArray cpBuf 0 else pure 0 + last_ <- if cnt >= 1 then readPrimArray cpBuf (cnt - 1) else pure 0 + p1 <- if cnt >= 2 then readPrimArray cpBuf 1 else pure 0 + p2 <- if cnt >= 4 then readPrimArray cpBuf 2 else pure 0 + p3 <- if cnt >= 4 then readPrimArray cpBuf 3 else pure 0 + let !leadHyp = first == 0x2D + !trailHyp = last_ == 0x2D + !hy34 = cnt >= 4 && p2 == 0x2D && p3 == 0x2D + !leadUnd = first == 0x5F + !xn = cnt >= 2 + && first .|. 0x20 == 0x78 + && p1 .|. 0x20 == 0x6E + !isWild = cnt == 1 && first == 0x2A + pure base { lsLeadingHyphen = leadHyp + , lsTrailingHyphen = trailHyp + , lsHy34 = hy34 + , lsLeadingUnder = leadUnd + , lsXnPrefix = xn + , lsIsWild = isWild + } + where + scanBody !i !st + | i >= cnt = pure st + | otherwise = do + cp <- readPrimArray cpBuf i + let !pastFirst = i >= 1 + !st' = + if | cp > 0xFF -> + st { lsHasGt0xFF = True + , lsHasNonAsciiCp = True + , lsAllLdh = False + , lsLdhPastFirst = lsLdhPastFirst st && not pastFirst + } + | cp >= 0x80 -> + st { lsHasNonAsciiCp = True + , lsAllLdh = False + , lsLdhPastFirst = lsLdhPastFirst st && not pastFirst + } + | isLdhCp cp -> st + | otherwise -> + st { lsAllLdh = False + , lsLdhPastFirst = lsLdhPastFirst st && not pastFirst + , lsHasNonLdhAscii = True + } + scanBody (i + 1) st' + +-- | Copy cpBuf[0..cnt) as raw bytes into outBuf starting at lStart+1, +-- check forms / lengths / wire-form caps, write the length byte, and +-- return the new output offset. +commitBytes + :: MutablePrimArray s Int + -> MutableByteArray s + -> LabelFormSet + -> Int -- lStart + -> Int -- cpCount + -> Int -- lIdx + -> [LabelForm] -- produced (in) + -> LabelForm -- form to assign + -> ST s (Either IdnaError (Int, [LabelForm])) +commitBytes !cpBuf !outBuf !allowed !lStart !cpCount !lIdx !produced !form + | not (form `memberLabelFormSet` allowed) = + pure (Left (ErrFormNotAllowed lIdx form)) + | otherwise = + let !newOff = lStart + 1 + cpCount + in if newOff >= maxWireLen + then pure (Left (ErrNameTooLong (newOff + 1))) + else do + writeCpBufAsBytes cpBuf outBuf (lStart + 1) cpCount + writeByteArray outBuf lStart (fromIntegral cpCount :: Word8) + pure (Right (newOff, form : produced)) + +writeCpBufAsBytes + :: MutablePrimArray s Int + -> MutableByteArray s + -> Int -- starting output offset + -> Int -- count + -> ST s () +writeCpBufAsBytes !cpBuf !outBuf !off !cnt = go 0 + where + go !i + | i >= cnt = pure () + | otherwise = do + cp <- readPrimArray cpBuf i + writeByteArray outBuf (off + i) (fromIntegral cp :: Word8) + go (i + 1) + +---------------------------------------------------------------------- +-- LDH / R-LDH / A-label decision. Loose by default; strict round- +-- trip classification (FAKEA on Punycode failure or non-round-trip) +-- is opt-in via the 'ALABELCHECK' option. +---------------------------------------------------------------------- + +ldhRldhPath + :: forall s + . MutablePrimArray s Int + -> MutableByteArray s + -> LabelFormSet + -> IdnaFlags + -> Int -- lStart + -> Int -- cpCount + -> Int -- lIdx + -> [LabelForm] -- produced + -> LabelStat + -> ST s (Either IdnaError (Int, [LabelForm])) +ldhRldhPath !cpBuf !outBuf !allowed !opts !lStart !cpCount !lIdx + !produced !stat + | not (lsXnPrefix stat) = + commitBytes cpBuf outBuf allowed lStart cpCount lIdx produced RLDH + | not (ALABELCHECK `meetsIdnaFlags` opts) = + -- Loose: every well-formed xn-- LDH label is ALABEL without + -- further checks. + commitBytes cpBuf outBuf allowed lStart cpCount lIdx produced ALABEL + | otherwise = do + -- Strict: copy the Punycode body (cpBuf[4..cpCount)) into + -- a fresh ByteArray, lower-casing any uppercase ASCII + -- letters, then run alabelRoundTrip. ALABEL on success; + -- on failure, FAKEA when the caller allows it, else the + -- specific 'AceReason' is surfaced via 'ErrAceInvalid'. + -- The case-normalisation here ensures all case variants + -- of the same ACE body decode identically through + -- Punycode (which is case-sensitive on basic codepoints). + let !bodyLen = cpCount - 4 + bodyBufM <- newByteArray bodyLen + let copyLc !i + | i >= bodyLen = pure () + | otherwise = do + cp <- readPrimArray cpBuf (4 + i) + let !b + | cp >= 0x41 && cp <= 0x5A = + fromIntegral (cp + 0x20) + | otherwise = + fromIntegral cp + writeByteArray bodyBufM i (b :: Word8) + copyLc (i + 1) + copyLc 0 + body <- unsafeFreezeByteArray bodyBufM + rt <- alabelRoundTrip body 0 bodyLen opts + case rt of + Right _ -> + commitBytes cpBuf outBuf allowed lStart cpCount lIdx + produced ALABEL + Left reason + | FAKEA `memberLabelFormSet` allowed -> + commitBytes cpBuf outBuf allowed lStart cpCount lIdx + produced FAKEA + | otherwise -> + pure (Left (ErrAceInvalid lIdx reason)) + +---------------------------------------------------------------------- +-- U-label path: validate codepoints, Punycode-encode, emit "xn--...". +---------------------------------------------------------------------- + +-- | Outcome of inspecting the codepoint buffer after the RFC 5895 +-- per-label mappings (case-fold + width + NFC) have run. An input +-- that originally had at least one non-ASCII codepoint may, after +-- mapping, contain only ASCII (typically when 'MAPWIDTH' collapsed +-- fullwidth Latin to ASCII). This type captures the four ways the +-- post-mapping buffer can shape up. +data PostMappingShape + = HasNonAscii -- ^ At least one codepoint @>= 0x80@; carry + -- on with the U-label path. + | AsciiNRLDH -- ^ All-ASCII, all-LDH, no leading or + -- trailing hyphen, no @\"--\"@ at + -- positions 3-4; reclassify as plain + -- LDH and emit bytes. + | AsciiBadCp !Int -- ^ All-ASCII so far, but at this codepoint + -- we hit a non-LDH ASCII byte (uppercase, + -- underscore, control byte, ...). The + -- carried 'Int' is the offending + -- codepoint. + | AsciiBadHyphens -- ^ All-ASCII LDH, but the hyphen pattern + -- disqualifies the buffer as plain LDH + -- (leading or trailing hyphen, or + -- @\"--\"@ at positions 3 and 4). + deriving (Eq, Show) + +-- | Single forward pass over @cpBuf[0..cnt)@ classifying the +-- post-mapping shape. Bails as soon as a non-ASCII codepoint or a +-- non-LDH ASCII codepoint is seen; otherwise inspects the hyphen +-- pattern at the boundary positions. +examinePostMapping + :: forall s. MutablePrimArray s Int -> Int -> ST s PostMappingShape +examinePostMapping !buf !cnt = scan 0 + where + scan :: Int -> ST s PostMappingShape + scan !i + | i >= cnt = checkShape + | otherwise = do + cp <- readPrimArray buf i + if | cp >= 0x80 -> pure HasNonAscii + | isLdhCp cp -> scan (i + 1) + | otherwise -> pure (AsciiBadCp cp) + + -- All-ASCII-LDH confirmed; check the boundary-position hyphen + -- rules. Caller's @cnt > 0@ invariant (flushLabel rejects + -- empty labels earlier) lets us read positions 0 and cnt-1 + -- unconditionally. + checkShape :: ST s PostMappingShape + checkShape = do + first <- readPrimArray buf 0 + last' <- readPrimArray buf (cnt - 1) + if first == 0x2D || last' == 0x2D + then pure AsciiBadHyphens + else if cnt >= 4 + then do + p2 <- readPrimArray buf 2 + p3 <- readPrimArray buf 3 + if p2 == 0x2D && p3 == 0x2D + then pure AsciiBadHyphens + else pure AsciiNRLDH + else pure AsciiNRLDH + +uLabelPath + :: MutablePrimArray s Int + -> MutableByteArray s + -> LabelFormSet + -> IdnaFlags + -> Int -- lStart + -> Int -- cpCount + -> Int -- lIdx + -> [LabelForm] -- produced + -> ST s (Either IdnaError (Int, [LabelForm])) +-- All RFC 5891 section 4.2.3.1 ("Hyphen Restrictions") prongs -- +-- leading, trailing, and @\"--\"@ at positions 3 and 4 -- are +-- enforced /after/ the per-label mappings, so that a fullwidth +-- hyphen U+FF0D that collapsed to ASCII @\'-\'@ under MAPWIDTH +-- (or any analogous mapping artifact) is caught. +-- 'examinePostMapping' handles them on the all-ASCII-after- +-- mapping branch; 'validateULabelCps' on the still-non-ASCII +-- branch. That keeps the check aligned with \"the Unicode +-- string\" as the RFC defines it (the post-mapping U-label, +-- not the raw input). +uLabelPath !cpBuf !outBuf !allowed !opts + !lStart !cpCount !lIdx !produced = do + -- RFC 5895 section 2.1 mapping (non-ASCII portion): under + -- 'MAPCASE', walk the codepoint buffer and apply Unicode + -- toLower to any non-ASCII codepoint with a defined case + -- mapping. ASCII A-Z were already lower-cased pre- + -- classification by 'flushLabel'; here we cover the rest. + when (MAPCASE `meetsIdnaFlags` opts) + (unicodeDownCaseBuf cpBuf cpCount) + -- RFC 5895 section 2.2 mapping: under 'MAPWIDTH', walk the + -- codepoint buffer and replace each fullwidth\/halfwidth + -- codepoint with its single-codepoint decomposition target. + -- Bits in the codepoint buffer that are not in the wide\/ + -- narrow set are left alone. Label-separator targets + -- (U+002E, U+3002) are handled at split time; 'MAPDOTS' + -- is implied by 'effectiveIdnaFlags' so they cannot reach + -- this pass. + when (MAPWIDTH `meetsIdnaFlags` opts) + (unicodeWidthBuf cpBuf cpCount) + -- Hand-curated UTS #46 subset (beyond IDNA2008): under + -- 'MAPUTS46', walk the codepoint buffer and expand each + -- source codepoint in 'uts46Lookup' to its target sequence. + -- 1:n expansion (era-name codepoints expand 1:2); buffer + -- growth past 'maxCpsPerLabel' raises 'ErrLabelTooLong'. + -- 'effectiveIdnaFlags' lifts 'MAPUTS46' to also enable + -- 'MAPCASE', 'MAPWIDTH', 'MAPDOTS', 'MAPNFC', so by the + -- time we get here the buffer has already been case- + -- folded and width-normalised. + !cnt0 <- + if MAPUTS46 `meetsIdnaFlags` opts + then unicodeUTS46Buf cpBuf cpCount + else pure cpCount + if cnt0 > maxCpsPerLabel + then pure (Left (ErrLabelTooLong lIdx cnt0)) + else uLabelMapped cpBuf outBuf allowed opts lStart cnt0 + lIdx produced + +-- | The remainder of 'uLabelPath' after the input-mapping passes: +-- apply 'MAPNFC', examine the post-mapping shape, and encode as +-- either ASCII-LDH or a U-label. Lifted out of 'uLabelPath' so +-- that 'uLabelPath' can short-circuit on 'MAPUTS46' overflow +-- without nesting the rest of the body in an @if-else@. +uLabelMapped + :: MutablePrimArray s Int + -> MutableByteArray s + -> LabelFormSet + -> IdnaFlags + -> Int -- lStart + -> Int -- cnt (post-MAPUTS46) + -> Int -- lIdx + -> [LabelForm] + -> ST s (Either IdnaError (Int, [LabelForm])) +uLabelMapped !cpBuf !outBuf !allowed !opts !lStart !cnt0 !lIdx !produced = do + -- RFC 5895 section 2.3 mapping: under 'MAPNFC', normalise + -- the codepoint buffer to NFC in place before validation, + -- so a decomposed input (e.g. @"a" + U+0301@) parses as + -- the precomposed equivalent (here @U+00E1@). Off by + -- default; when unset 'normalizeNFC' is not invoked. + -- + -- After normalising we strip 'NFCCHECK' from the opts + -- handed to 'validateULabelCps': the buffer is now in NFC + -- by construction, so the validator's 'isNFC' check would + -- be a no-op. A-labels (whose codepoints come from + -- 'alabelRoundTrip', which receives the original opts) + -- still get the 'NFCCHECK' check on their decoded form. + (!cnt, !opts') <- + if MAPNFC `meetsIdnaFlags` opts + then do + n <- normalizeNFC cpBuf cnt0 + pure (n, opts `withoutIdnaFlags` NFCCHECK) + else pure (cnt0, opts) + -- Post-mapping shape examination. If the mappings have + -- collapsed the buffer to all-ASCII, we either reclassify + -- as plain LDH (clean NR-LDH shape: a-z\/0-9\/hyphen, no + -- leading or trailing hyphen, no @\"--\"@ at positions + -- 3-4) or reject (any non-LDH ASCII codepoint, or a + -- disqualifying hyphen pattern). We do not try to + -- recover into ALABEL or RLDH from this position -- + -- that's a bridge too far for what was originally a + -- U-label. Buffers that still contain at least one + -- non-ASCII codepoint continue down the U-label path. + shape <- examinePostMapping cpBuf cnt + case shape of + AsciiBadCp cp -> + pure (Left (ErrLabelInvalid lIdx (DisallowedCodepoint cp))) + AsciiBadHyphens -> + pure (Left (ErrLabelInvalid lIdx HyphenViolation)) + AsciiNRLDH -> + commitBytes cpBuf outBuf allowed lStart cnt lIdx + produced LDH + HasNonAscii -> uLabelEncode cpBuf outBuf allowed opts' + lStart cnt lIdx produced + +uLabelEncode + :: forall s + . MutablePrimArray s Int + -> MutableByteArray s + -> LabelFormSet + -> IdnaFlags + -> Int -- lStart + -> Int -- cnt (post-mapping codepoint count) + -> Int -- lIdx + -> [LabelForm] -- produced + -> ST s (Either IdnaError (Int, [LabelForm])) +uLabelEncode !cpBuf !outBuf !allowed !opts !lStart !cnt !lIdx !produced = do + rv <- validateULabelCps cpBuf cnt opts + rv' <- case rv of + Just _ -> pure rv + Nothing + | BIDICHECK `meetsIdnaFlags` opts -> do + !summary <- buildBidiSummary cpBuf cnt + pure (checkBidiLabel summary) + | otherwise -> pure Nothing + case rv' of + Just reason + -- 'LAXULABEL' not even in the set: nothing to consider, + -- propagate the rejection unchanged. + | not (LAXULABEL `memberLabelFormSet` allowed) -> + pure (Left (ErrLabelInvalid lIdx reason)) + -- First failure is already an ASCII codepoint: we know + -- the label has non-LDH ASCII content (the offending + -- codepoint itself). Hard reject; 'LAXULABEL' must not + -- rescue --- such a byte cannot appear in a valid + -- Punycode-encoded wire form. + | DisallowedCodepoint cp <- reason + , cp < 0x80 -> + pure (Left (ErrLabelInvalid lIdx reason)) + -- First failure is non-ASCII (or non-codepoint-specific): + -- scan the rest of the post-mapping buffer for any + -- non-LDH ASCII that would also disqualify 'LAXULABEL' + -- admission. + | otherwise -> do + !badAscii <- hasNonLdhAsciiCps cpBuf cnt + if badAscii + then pure (Left (ErrLabelInvalid lIdx reason)) + else encodeAs LAXULABEL + Nothing + | not (ULABEL `memberLabelFormSet` allowed) -> + pure (Left (ErrFormNotAllowed lIdx ULABEL)) + | otherwise -> encodeAs ULABEL + where + -- Punycode-encode the post-mapping codepoint buffer and emit + -- the ACE-prefixed wire form, tagging the result with + -- the caller-chosen 'LabelForm' (either 'ULABEL' for a label + -- that passed strict validation, or 'LAXULABEL' for one + -- admitted via the permissive fallback). + encodeAs :: LabelForm -> ST s (Either IdnaError (Int, [LabelForm])) + encodeAs !tag = do + let !prefAt = lStart + 1 + writeByteArray @Word8 outBuf prefAt 0x78 -- 'x' + writeByteArray @Word8 outBuf (prefAt + 1) 0x6E -- 'n' + writeByteArray @Word8 outBuf (prefAt + 2) 0x2D -- '-' + writeByteArray @Word8 outBuf (prefAt + 3) 0x2D -- '-' + cps <- freezeCps cpBuf cnt + let !bodyAt = prefAt + 4 + !bodyCap = maxLabelLen - 4 + enc <- punycodeEncode cps cnt outBuf bodyAt bodyCap + case enc of + Left _ -> + pure (Left (ErrPunycodeOverflow lIdx)) + Right encLen -> + let !contentLen = 4 + encLen + !newOff = lStart + 1 + contentLen + in if contentLen > maxLabelLen + then pure (Left (ErrLabelTooLong lIdx contentLen)) + else if newOff >= maxWireLen + then pure (Left (ErrNameTooLong (newOff + 1))) + else do + writeByteArray outBuf lStart + (fromIntegral contentLen :: Word8) + pure (Right (newOff, tag : produced)) + +freezeCps :: MutablePrimArray s Int -> Int -> ST s (PrimArray Int) +freezeCps !cpBuf !cnt = do + tmp <- newPrimArray cnt + copyN tmp 0 + unsafeFreezePrimArray tmp + where + copyN dst !i + | i >= cnt = pure () + | otherwise = do + v <- readPrimArray cpBuf i + writePrimArray dst i v + copyN dst (i + 1) + +---------------------------------------------------------------------- +-- U-label codepoint validation +---------------------------------------------------------------------- + +-- | U-label codepoint validator. Walks the codepoint sequence and +-- consults 'idnaDisposition' for each. +-- +-- Currently enforced: +-- +-- * Each codepoint must have IDNA disposition @PVALID@. +-- * @CONTEXTO@ codepoints are admitted only when 'checkContextO' +-- accepts their context. All seven contextual rules from +-- RFC 5892 Appendix A.3-A.9 are implemented (the four +-- Script-based rules consult +-- "Text.IDNA2008.Internal.Script"). +-- * @CONTEXTJ@ codepoints are admitted only when 'checkContextJ' +-- accepts their context (RFC 5892 Appendix A.1 ZWNJ and A.2 +-- ZWJ). Both rules consult +-- "Text.IDNA2008.Internal.Joining" for Virama membership; +-- A.1 also consults Joining_Type. +-- * Under 'NFCCHECK', full canonical normalisation is performed +-- and the result compared against the input; the label is +-- rejected if it differs. Off by default. +-- * @DISALLOWED@ \/ @UNASSIGNED@ codepoints are rejected. Under +-- 'EMOJIOK', a non-ASCII @DISALLOWED@ codepoint with +-- @Emoji=Yes@ is admitted as if @PVALID@ (a diagnostic +-- relaxation; @UNASSIGNED@ is still rejected). +-- +-- Note: the per-label RFC 5893 (Bidi) check, gated on 'BIDICHECK', +-- runs separately from this function — see 'checkBidiLabel' in +-- the Bidi section below. It is invoked from 'uLabelEncode' +-- after this validator returns clean. +validateULabelCps + :: forall s + . MutablePrimArray s Int + -> Int + -> IdnaFlags + -> ST s (Maybe LabelReason) +validateULabelCps !cpBuf !cnt !opts + | cnt <= 0 = + if NFCCHECK `meetsIdnaFlags` opts + then nfcCheck + else pure Nothing + | otherwise = do + -- RFC 5891 section 4.2.3.2: a U-label must not begin + -- with a combining mark (General_Category in {Mn, Mc, + -- Me}). Many such codepoints are PVALID per RFC 5892, + -- so the per-codepoint validator alone wouldn't catch + -- this; we check the first codepoint explicitly here. + first <- readPrimArray cpBuf 0 + if isCombiningMark first + then pure (Just (LeadingCombiningMark first)) + else do + -- RFC 5891 section 4.2.3.1 ("Hyphen Restrictions"), + -- all three prongs, enforced /post-mapping/ so that + -- a fullwidth-hyphen U+FF0D that collapsed to ASCII + -- @\'-\'@ under MAPWIDTH is caught here even though + -- 'scanCpBuf' (which runs pre-mapping) did not see + -- it. The all-ASCII-after-mapping branch is + -- handled in 'examinePostMapping'; this is the + -- still-non-ASCII branch. + -- + -- 1. Leading hyphen: @cpBuf[0] == 0x2D@. + -- 2. Trailing hyphen: @cpBuf[cnt-1] == 0x2D@. + -- 3. @\"--\"@ at positions 3 and 4: a U-label + -- that would visually mimic an A-label + -- (e.g. Cyrillic @\"\\x445n----...\"@ vs Latin + -- @\"xn----...\"@) without being one. The + -- ACE-escape namespace is reserved for that + -- shape. + last' <- readPrimArray cpBuf (cnt - 1) + hy34 <- if cnt >= 4 + then do c2 <- readPrimArray cpBuf 2 + c3 <- readPrimArray cpBuf 3 + pure (c2 == 0x2D && c3 == 0x2D) + else pure False + if first == 0x2D || last' == 0x2D || hy34 + then pure (Just HyphenViolation) + else go 0 + where + go :: Int -> ST s (Maybe LabelReason) + go !i + | i >= cnt = + if NFCCHECK `meetsIdnaFlags` opts + then nfcCheck + else pure Nothing + | otherwise = do + cp <- readPrimArray cpBuf i + case idnaDisposition cp of + IdnaPVALID -> go (i + 1) + IdnaCONTEXTJ -> do + ok <- checkContextJ cpBuf cnt i cp + if ok + then go (i + 1) + else pure (Just (ContextRule cp)) + IdnaCONTEXTO -> do + ok <- checkContextO cpBuf cnt i cp + if ok + then go (i + 1) + else pure (Just (ContextRule cp)) + IdnaDISALLOWED + -- Diagnostic relaxation: under 'EMOJIOK' admit a + -- non-ASCII DISALLOWED codepoint that carries the + -- Unicode @Emoji=Yes@ property AND is not in the + -- UTS #46-mapped subset of emoji codepoints. + -- + -- Restricted to non-ASCII so that ASCII punctuation + -- that happens to be @Emoji=Yes@ (e.g. @\'#\'@ + -- U+0023, @\'*\'@ U+002A) stays disallowed. + -- + -- The @isMappedEmoji@ exclusion drops the small + -- subset of emoji codepoints whose UTS #46 status + -- is @mapped@: these resolve ambiguously across the + -- ecosystem (browsers following UTS #46 mapping reach + -- the fold target; this library would admit the + -- codepoint as-is). When the mapped form and the + -- admit-as-is form belong to separately registered + -- domains under different operators — e.g. (xn--q97h.ws) + -- and its UTS #46 target (xn--uny.ws), seen in the wild + -- silently routing to either is a security concern. We + -- refuse both interpretations under 'EMOJIOK'; a + -- caller who needs to address such a label can + -- construct its wire form directly. + -- + -- Emoji codepoints are not @CONTEXTJ@/@CONTEXTO@, + -- so no contextual check is needed on the relaxation + -- path. + | cp >= 0x80, EMOJIOK `meetsIdnaFlags` opts + , isEmoji cp, not (isMappedEmoji cp) -> + go (i + 1) + | otherwise -> pure (Just (DisallowedCodepoint cp)) + IdnaUNASSIGNED -> pure (Just (DisallowedCodepoint cp)) + + -- | Precise NFC pass via 'isNFC': Quick_Check fast-path on + -- the all-@Yes@ common case, otherwise full canonical + -- decompose / reorder / compose and byte-compare against + -- the input. + nfcCheck :: ST s (Maybe LabelReason) + nfcCheck = do + ok <- isNFC cpBuf cnt + if ok then pure Nothing else pure (Just NotNFC) + +---------------------------------------------------------------------- +-- CONTEXTO contextual rules (RFC 5892 Appendix A.3-A.9) +---------------------------------------------------------------------- + +-- | Decide whether a 'IdnaCONTEXTO' codepoint at position @i@ in a +-- candidate U-label is admissible in its surrounding context. +-- Returns 'True' iff the codepoint's contextual rule is satisfied. +-- +-- All seven contextual rules from RFC 5892 Appendix A.3-A.9 are +-- implemented: +-- +-- * A.3 Middle Dot @U+00B7@ -- preceded /and/ followed by +-- @U+006C@ (lowercase ASCII @\'l\'@); the Catalan +-- @\"l\\u00B7l\"@ (ela geminada). +-- * A.4 Greek Lower Numeral Sign @U+0375@ -- followed by a +-- codepoint in the Greek script. +-- * A.5 Hebrew Punctuation Geresh @U+05F3@ -- preceded by a +-- codepoint in the Hebrew script. +-- * A.6 Hebrew Punctuation Gershayim @U+05F4@ -- preceded by +-- a codepoint in the Hebrew script. +-- * A.7 Katakana Middle Dot @U+30FB@ -- the label contains at +-- least one codepoint in Hiragana, Katakana, or Han. +-- * A.8 Arabic-Indic Digits @U+0660..U+0669@ -- the label +-- contains no Extended Arabic-Indic digit. +-- * A.9 Extended Arabic-Indic Digits @U+06F0..U+06F9@ -- the +-- label contains no Arabic-Indic digit. +-- +-- The Script-based rules (A.4-A.7) consult +-- "Text.IDNA2008.Internal.Script". +checkContextO + :: MutablePrimArray s Int + -> Int -- cnt + -> Int -- position of cp + -> Int -- cp + -> ST s Bool +checkContextO !cpBuf !cnt !i !cp + -- A.3: Middle Dot, surrounded by 'l' on both sides. + | cp == 0x00B7 = + if i == 0 || i + 1 >= cnt + then pure False + else do + prev <- readPrimArray cpBuf (i - 1) + next <- readPrimArray cpBuf (i + 1) + pure (prev == 0x6C && next == 0x6C) + -- A.4: Greek Lower Numeral Sign, followed by a Greek codepoint. + | cp == 0x0375 = + if i + 1 >= cnt + then pure False + else do + next <- readPrimArray cpBuf (i + 1) + pure (isGreekCp next) + -- A.5 / A.6: Hebrew Geresh / Gershayim, preceded by a Hebrew + -- codepoint. + | cp == 0x05F3 || cp == 0x05F4 = + if i == 0 + then pure False + else do + prev <- readPrimArray cpBuf (i - 1) + pure (isHebrewCp prev) + -- A.7: Katakana Middle Dot, with at least one Hira / Kata / Han + -- codepoint somewhere in the label. + | cp == 0x30FB = labelHasAny cpBuf cnt isHkhCp + -- A.8: Arabic-Indic digits and Extended Arabic-Indic digits + -- must not appear in the same label. + | cp >= 0x0660 && cp <= 0x0669 = + labelLacks cpBuf cnt 0x06F0 0x06F9 + -- A.9: same rule, mirrored. + | cp >= 0x06F0 && cp <= 0x06F9 = + labelLacks cpBuf cnt 0x0660 0x0669 + -- No other CONTEXTO codepoints are defined by RFC 5892. + -- Reject defensively in case the IDNA disposition table + -- starts to advertise one we do not yet recognise. + | otherwise = pure False + +-- | Walk a codepoint buffer and return 'True' iff no codepoint in +-- @[lo, hi]@ appears anywhere in @[0, cnt)@. +labelLacks + :: MutablePrimArray s Int + -> Int -- cnt + -> Int -- forbidden lo + -> Int -- forbidden hi + -> ST s Bool +labelLacks !cpBuf !cnt !lo !hi = go 0 + where + go !j + | j >= cnt = pure True + | otherwise = do + c <- readPrimArray cpBuf j + if c >= lo && c <= hi + then pure False + else go (j + 1) + +-- | Walk a codepoint buffer and return 'True' iff @p@ accepts +-- any codepoint in @[0, cnt)@. +labelHasAny + :: MutablePrimArray s Int + -> Int -- cnt + -> (Int -> Bool) -- predicate + -> ST s Bool +labelHasAny !cpBuf !cnt !p = go 0 + where + go !j + | j >= cnt = pure False + | otherwise = do + c <- readPrimArray cpBuf j + if p c + then pure True + else go (j + 1) + +---------------------------------------------------------------------- +-- CONTEXTJ contextual rules (RFC 5892 Appendix A.1, A.2) +---------------------------------------------------------------------- + +-- | Decide whether a 'IdnaCONTEXTJ' codepoint at position @i@ in a +-- candidate U-label is admissible in its surrounding context. +-- Returns 'True' iff the codepoint's contextual rule is satisfied. +-- +-- * A.1 ZERO WIDTH NON-JOINER @U+200C@ -- admitted when either +-- (a) the immediately preceding codepoint is a Virama +-- (Canonical_Combining_Class = 9), or (b) the label +-- structure matches the regex +-- @(L|D) T* U+200C T* (R|D)@ over the Joining_Type +-- property. +-- * A.2 ZERO WIDTH JOINER @U+200D@ -- admitted only when the +-- immediately preceding codepoint is a Virama. +-- +-- Both arms consult "Text.IDNA2008.Internal.Joining". +checkContextJ + :: MutablePrimArray s Int + -> Int -- cnt + -> Int -- position of cp + -> Int -- cp + -> ST s Bool +checkContextJ !cpBuf !cnt !i !cp + -- A.2: ZWJ. Only the Virama-prefix arm. + | cp == 0x200D = viramaPrefix cpBuf i + -- A.1: ZWNJ. Virama-prefix arm OR the joining-type regex. + | cp == 0x200C = do + vp <- viramaPrefix cpBuf i + if vp + then pure True + else zwnjJoinContext cpBuf cnt i + -- No other CONTEXTJ codepoints are defined by RFC 5892; + -- reject defensively if the disposition table grows one. + | otherwise = pure False + +-- | Is the codepoint immediately preceding position @i@ a Virama +-- (Canonical_Combining_Class = 9)? False at @i == 0@. +viramaPrefix + :: MutablePrimArray s Int + -> Int -- position + -> ST s Bool +viramaPrefix !cpBuf !i + | i == 0 = pure False + | otherwise = do + prev <- readPrimArray cpBuf (i - 1) + pure (isVirama prev) + +-- | The A.1 ZWNJ join-context regex: +-- +-- > (Joining_Type:{L,D}) (Joining_Type:T)* U+200C (Joining_Type:T)* (Joining_Type:{R,D}) +-- +-- Walk left from @i - 1@ skipping any 'jtIsTransparent' codepoints +-- and require the first non-transparent to be 'jtIsLeftOrDual'; +-- walk right from @i + 1@ skipping any 'jtIsTransparent' and +-- require the first non-transparent to be 'jtIsRightOrDual'. +-- Either side hitting the label boundary fails the rule. +zwnjJoinContext + :: forall s + . MutablePrimArray s Int + -> Int -- cnt + -> Int -- position of ZWNJ + -> ST s Bool +zwnjJoinContext !cpBuf !cnt !i = do + leftOk <- scanLeft (i - 1) + if not leftOk + then pure False + else scanRight (i + 1) + where + scanLeft :: Int -> ST s Bool + scanLeft !j + | j < 0 = pure False + | otherwise = do + c <- readPrimArray cpBuf j + if jtIsTransparent c + then scanLeft (j - 1) + else pure (jtIsLeftOrDual c) + + scanRight :: Int -> ST s Bool + scanRight !j + | j >= cnt = pure False + | otherwise = do + c <- readPrimArray cpBuf j + if jtIsTransparent c + then scanRight (j + 1) + else pure (jtIsRightOrDual c) + +---------------------------------------------------------------------- +-- Per-label Bidi check (RFC 5893) +---------------------------------------------------------------------- + +-- | Per-label RFC 5893 check. Thin wrapper over +-- 'checkBidiSummary' (in trigger-on mode — per-label semantics) +-- that lifts the rule-violation payload into 'LabelReason' for +-- the parser-time error path. See 'checkBidiSummary' for the +-- semantics of the rule application. +checkBidiLabel :: BidiSummary -> Maybe LabelReason +checkBidiLabel !s = LabelBidi <$> checkBidiSummary False s + +---------------------------------------------------------------------- +-- A-label strict round-trip +---------------------------------------------------------------------- + +-- | Strict A-label round-trip check. Given the lower-cased Punycode +-- body of an ACE-prefixed label (i.e. the bytes /after/ the +-- @\"xn--\"@ prefix), return the decoded codepoints if they form a +-- valid U-label whose re-encoding matches the input body +-- byte-for-byte. Otherwise return the reason the label failed: +-- +-- * 'BadPunycode' -- the body is not valid Punycode +-- (truncated, bad delimiter, non-base-36 byte, internal overflow). +-- * 'DecodedInvalid' -- the decoded codepoints have an +-- IDNA2008 disposition that is not @PVALID@. The wrapped +-- 'LabelReason' identifies the first offending codepoint. +-- * 'RoundTripMismatch' -- the decoded form is a valid +-- U-label, but its re-encoding to ACE does not match the input +-- body byte-for-byte. +-- +-- The input body is expected to be lower-cased by the caller; the +-- comparison after re-encoding is exact byte-equality. Punycode +-- encoding always emits lower-case digits, and basic codepoints in +-- a valid U-label are themselves already lower-case ASCII (uppercase +-- ASCII has IDNA2008 disposition @DISALLOWED@), so a lower-cased +-- input body will round-trip exactly when the label is a real +-- A-label. +alabelRoundTrip + :: forall s + . ByteArray -- ^ Lower-cased Punycode body bytes + -> Int -- ^ Body start offset + -> Int -- ^ Body length + -> IdnaFlags -- ^ Threaded through to the U-label + -- validator (e.g., 'NFCCHECK', when + -- set, also constrains the decoded + -- form). + -> ST s (Either AceReason (PrimArray Int)) +alabelRoundTrip !body !bStart !bLen !opts = do + cpBuf <- newPrimArray maxCpsPerLabel + rDec <- punycodeDecode body bStart bLen cpBuf 0 maxCpsPerLabel + case rDec of + Left _ -> pure (Left BadPunycode) + Right cpCount -> do + rVal <- validateULabelCps cpBuf cpCount opts + rVal' <- case rVal of + Just _ -> pure rVal + Nothing + | BIDICHECK `meetsIdnaFlags` opts -> do + !summary <- buildBidiSummary cpBuf cpCount + pure (checkBidiLabel summary) + | otherwise -> pure Nothing + case rVal' of + Just reason -> pure (Left (DecodedInvalid reason)) + Nothing -> do + cps <- freezeCps cpBuf cpCount + outBuf <- newByteArray maxLabelLen + rEnc <- punycodeEncode cps cpCount outBuf 0 maxLabelLen + case rEnc of + Left _ -> pure (Left RoundTripMismatch) + Right encLen + | encLen /= bLen -> + pure (Left RoundTripMismatch) + | otherwise -> do + eq <- byteRangeEqMutImm outBuf 0 body bStart bLen + if eq + then pure (Right cps) + else pure (Left RoundTripMismatch) + +-- | Compare @[mOff, mOff+len)@ of a 'MutableByteArray' against +-- @[iOff, iOff+len)@ of a 'ByteArray' for exact byte equality. +byteRangeEqMutImm + :: forall s + . MutableByteArray s + -> Int + -> ByteArray + -> Int + -> Int + -> ST s Bool +byteRangeEqMutImm !mBuf !mOff !iBuf !iOff !len = go 0 + where + go !k + | k >= len = pure True + | otherwise = do + mb <- readByteArray @Word8 mBuf (mOff + k) + let !ib = indexByteArray iBuf (iOff + k) :: Word8 + if mb /= ib + then pure False + else go (k + 1) + +---------------------------------------------------------------------- +-- Domain -> Unicode display form +---------------------------------------------------------------------- + +-- | Render a 'Domain' as 'Text' in Unicode presentation form. +-- Valid labels decode to U-labels. Any other labels render as +-- ASCII (with escapes for bytes that need them). The root domain +-- renders as @\".\"@, other domains have /no/ trailing dots. +-- +-- See 'unparseDomainOpts' for finer control. +domainToUnicode :: Domain -> Text +domainToUnicode dom = case unparseDomainOpts forms flags dom of + Right (t, _) -> t + Left _ -> error "domainToUnicode: unreachable" + where + !flags = defaultIdnaFlags <> ASCIIFALLBACK + !forms = foldMap' labelFormToSet + [LDH, RLDH, ULABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + +-- | Render a 'Domain' as 'Text' in ASCII presentation form. +-- Every A-label label stays in its ASCII form (no Unicode +-- decoding). Labels with non-LDH characters are output with +-- escapes as needed. The root domain renders as @\".\"@, +-- other domains have /no/ trailing dots. +-- +-- See 'unparseDomainOpts' for finer control. +domainToAscii :: Domain -> Text +domainToAscii dom = case unparseDomainOpts forms mempty dom of + Right (t, _) -> t + Left _ -> error "domainToAscii: unreachable" + where + !forms = foldMap' labelFormToSet + [LDH, RLDH, ALABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + +-- | Grow if needed, then write a single raw byte at the cursor. +writeRawByte + :: MutableByteArray s -> Int -> Int -> Word8 + -> ST s (MutableByteArray s, Int, Int) +writeRawByte !mba !cap !cur !b = do + (mba', cap') <- ensureCapacity mba cap cur 1 + writeByteArray mba' cur b + pure (mba', cap', cur + 1) + +-- | Reserve at least @need@ bytes of headroom in the buffer, +-- growing it via 'resizeMutableByteArray' if necessary. The +-- growth branch adds 50% more capacity (or jumps directly to +-- @cur + need@, whichever is larger). +ensureCapacity + :: MutableByteArray s -> Int -> Int -> Int + -> ST s (MutableByteArray s, Int) +ensureCapacity !mba !cap !cur !need + | cur + need <= cap = pure (mba, cap) + | otherwise = do + let !cap' = max (cur + need) (cap + cap `div` 2) + mba' <- resizeMutableByteArray mba cap' + pure (mba', cap') + +-- | Walk a label's wire-bytes slice @sbs[off .. off+len)@, +-- writing each byte (with presentation-form escaping as needed) +-- into the buffer. The slice form lets the domain-level walker +-- emit each label in place against the parent @Domain@'s wire +-- bytes without per-label allocation. +writeEscapedLabel + :: MutableByteArray s -> Int -> Int + -> ShortByteString -> Int -> Int + -> ST s (MutableByteArray s, Int, Int) +writeEscapedLabel !mba0 !cap0 !cur0 (SBS sba) !off !len = + step mba0 cap0 cur0 0 + where + !arr = ByteArray sba + step !mba !cap !cur !i + | i >= len = pure (mba, cap, cur) + | otherwise = do + let !b = indexByteArray arr (off + i) :: Word8 + (mba', cap', cur') <- writeEscapedByte mba cap cur b + step mba' cap' cur' (i + 1) + +-- | Escape one byte into the buffer. Printable ASCII syntactic +-- specials (per 'special') become @\\C@ (two bytes); controls, +-- whitespace, DEL, and high bytes become @\\DDD@ (four bytes); +-- other printable ASCII passes through unchanged. This is the +-- single source of truth for the byte-level escape policy -- +-- both 'writeEscapedLabel' and the codepoint emitter +-- 'writeCpUtf8' route ASCII bytes through here. +writeEscapedByte + :: MutableByteArray s -> Int -> Int -> Word8 + -> ST s (MutableByteArray s, Int, Int) +writeEscapedByte !mba !cap !cur !b + | special b = do + (mba', cap') <- ensureCapacity mba cap cur 2 + writeByteArray mba' cur (0x5C :: Word8) + writeByteArray mba' (cur + 1) b + pure (mba', cap', cur + 2) + | b <= 0x20 || b >= 0x7F = do + (mba', cap') <- ensureCapacity mba cap cur 4 + let !d100 = b `quot` 100 + !d10 = (b `quot` 10) `rem` 10 + !d1 = b `rem` 10 + writeByteArray mba' cur (0x5C :: Word8) + writeByteArray mba' (cur + 1) (0x30 + d100) + writeByteArray mba' (cur + 2) (0x30 + d10) + writeByteArray mba' (cur + 3) (0x30 + d1) + pure (mba', cap', cur + 4) + | otherwise = do + (mba', cap') <- ensureCapacity mba cap cur 1 + writeByteArray mba' cur b + pure (mba', cap', cur + 1) + +special :: Word8 -> Bool +special b = testBit tab $ fromIntegral (b - 0x22) + where + -- Status of 59 bytes from '"' to '\\', bit set means "special". + tab :: Word64 + tab = 0b0100_0000_0000_0000_0000_0000_0000_0100_0010_0000_0000_0001_0000_1100_0101 + +-- | Walk a 'PrimArray' of codepoints and emit each as 1--4 UTF-8 +-- bytes into the buffer. +writeCodepointsUtf8 + :: MutableByteArray s -> Int -> Int -> PrimArray Int + -> ST s (MutableByteArray s, Int, Int) +writeCodepointsUtf8 !mba0 !cap0 !cur0 !cps = step mba0 cap0 cur0 0 + where + !n = sizeofPrimArray cps + step !mba !cap !cur !i + | i >= n = pure (mba, cap, cur) + | otherwise = do + let !cp = indexPrimArray cps i + (mba', cap', cur') <- writeCpUtf8 mba cap cur cp + step mba' cap' cur' (i + 1) + +-- | Encode a single codepoint as UTF-8 into the buffer. Up to +-- four bytes for codepoints @<= U+10FFFF@. Caller guarantees +-- the codepoint is in range (the decoder enforces this). +-- +-- Codepoints @< 0x80@ are routed through 'writeEscapedByte' so +-- that ASCII bytes with zone-file significance get the usual +-- @\\C@ or @\\DDD@ treatment. This matters whenever a FAKEA +-- Punycode body decodes (via 'LAXULABEL' admission) to codepoints +-- like @\'.\'@, @\';\'@, or @\'\@\'@ that strict round-trip would +-- have rejected. Multi-byte UTF-8 encodings of codepoints +-- @>= 0x80@ are emitted verbatim; continuation bytes are all +-- @>= 0x80@ and carry no per-byte zone-file meaning. +writeCpUtf8 + :: MutableByteArray s -> Int -> Int -> Int + -> ST s (MutableByteArray s, Int, Int) +writeCpUtf8 !mba !cap !cur !cp + | cp < 0x80 = writeEscapedByte mba cap cur (fromIntegral cp) + | cp < 0x800 = do + (mba', cap') <- ensureCapacity mba cap cur 2 + writeByteArray mba' cur + (fromIntegral (0xC0 .|. (cp `unsafeShiftR` 6)) :: Word8) + writeByteArray mba' (cur + 1) + (fromIntegral (0x80 .|. (cp .&. 0x3F)) :: Word8) + pure (mba', cap', cur + 2) + | cp < 0x10000 = do + (mba', cap') <- ensureCapacity mba cap cur 3 + writeByteArray mba' cur + (fromIntegral (0xE0 .|. (cp `unsafeShiftR` 12)) :: Word8) + writeByteArray mba' (cur + 1) + (fromIntegral (0x80 .|. ((cp `unsafeShiftR` 6) .&. 0x3F)) :: Word8) + writeByteArray mba' (cur + 2) + (fromIntegral (0x80 .|. (cp .&. 0x3F)) :: Word8) + pure (mba', cap', cur + 3) + | otherwise = do + (mba', cap') <- ensureCapacity mba cap cur 4 + writeByteArray mba' cur + (fromIntegral (0xF0 .|. (cp `unsafeShiftR` 18)) :: Word8) + writeByteArray mba' (cur + 1) + (fromIntegral (0x80 .|. ((cp `unsafeShiftR` 12) .&. 0x3F)) :: Word8) + writeByteArray mba' (cur + 2) + (fromIntegral (0x80 .|. ((cp `unsafeShiftR` 6) .&. 0x3F)) :: Word8) + writeByteArray mba' (cur + 3) + (fromIntegral (0x80 .|. (cp .&. 0x3F)) :: Word8) + pure (mba', cap', cur + 4) + +---------------------------------------------------------------------- +-- Wire-bytes label classification (unparse side) +---------------------------------------------------------------------- + +-- | Byte-shape classification of a wire-form label, before any +-- @\"xn--\"@ round-trip check. Used by 'unparseDomainOpts' to +-- decide which 'LabelForm' applies to each wire label. +data WireShape + = WSWild -- ^ Single byte @\'*\'@. Renders as 'WILDLABEL'. + | WSLdh -- ^ All-LDH, no leading or trailing hyphen, no + -- @\"--\"@ at positions 3-4. Renders as 'LDH'. + | WSRldh -- ^ LDH with @\"--\"@ at positions 3-4 and a + -- non-@\"xn\"@ first pair. Renders as 'RLDH'. + | WSXnPrefix -- ^ LDH with @\"--\"@ at positions 3-4 and an + -- @\"xn\"@ first pair. The 'ALABEL' / + -- 'FAKEA' decision is made by 'alabelRoundTrip' + -- under the active 'IdnaFlags'. + | WSAttrLeaf -- ^ Leading @\'_\'@, bytes @1..n-1@ all LDH, + -- no trailing hyphen. Renders as 'ATTRLEAF'. + | WSOctet -- ^ Anything else. Renders as 'OCTET'. + deriving (Eq, Show) + +-- | Classify a wire-form label's raw bytes into a 'WireShape'. +-- Pure, single pass over @sbs[off .. off+len)@. The slice form +-- lets a single 'ShortByteString' carry either a stand-alone +-- label (called with @off = 0@ and @len = 'SBS.length' sbs@) or +-- the full wire form of a 'Domain' (the domain-level walker +-- supplies each label's offset and length in turn), so no +-- per-label 'ShortByteString' allocation is needed. +-- +-- The case of LDH letters is preserved on the wire and matched +-- case-insensitively here, so an uppercase @\"XN--ABC\"@ slice +-- classifies as 'WSXnPrefix' just like the canonical lowercase +-- form. +classifyWireBytes :: ShortByteString -> Int -> Int -> WireShape +classifyWireBytes (SBS sba) !off !len = case len of + 0 -> WSOctet + 1 | byteAt 0 == 0x2A -> WSWild + n -> classify n + where + !arr = ByteArray sba + byteAt !i = indexByteArray arr (off + i) :: Word8 + classify !n + | b0 == 0x5F = + -- Leading underscore: ATTRLEAF iff every byte past the + -- underscore is LDH and the last byte is not a hyphen. + if allLdh 1 n && bn1 /= 0x2D + then WSAttrLeaf + else WSOctet + | b0 /= 0x2D && bn1 /= 0x2D && allLdh 0 n = + -- LDH-shaped: distinguish RLDH / xn-- / plain LDH by the + -- positions 3-4 hyphen pair and the first-pair letters. + if n >= 4 && byteAt 2 == 0x2D && byteAt 3 == 0x2D + then if isXnPair then WSXnPrefix else WSRldh + else WSLdh + | otherwise = WSOctet + where + !b0 = byteAt 0 + !bn1 = byteAt (n - 1) + !isXnPair = (b0 .|. 0x20) == 0x78 && (byteAt 1 .|. 0x20 == 0x6E) + allLdh !i !lim + | i >= lim = True + | otherwise = isLdhByte (byteAt i) && allLdh (i + 1) lim + +-- | The LDH alphabet on the wire: @a-z@, @A-Z@, @0-9@, @-@. +-- Lowercase is the canonical wire form; uppercase letters can +-- appear when the input was upper-case and 'MAPCASE' was not +-- applied. +isLdhByte :: Word8 -> Bool +isLdhByte b + | (b .|. 0x20) - 0x61 < 26 = True -- A-Z, a-z + | b - 0x30 < 10 = True -- 0-9 + | otherwise = b == 0x2D -- hyphen +{-# INLINE isLdhByte #-} + +-- | Render a single wire-form label as 'Text' in presentation +-- form and report that form's classification. +-- +-- Non A-labels are classified by their byte content ('LDH', +-- 'RLDH', 'ATTRLEAF', 'WILDLABEL', or 'OCTET') and render as +-- ASCII, with escapes applied as needed. +-- +-- The presentation form of a wire-form ACE-prefixed LDH-label +-- can have four possible classifications: +-- +-- * 'ULABEL': The label passed validation and was decoded to Unicode. +-- * 'LAXULABEL': The label failed (lax) validation, and was decoded to Unicode. +-- * 'ALABEL': The label passed validation and left in ASCII form. +-- * 'FAKEA': The label failed validation and left in ASCII form. +-- +-- The specified 'LabelFormSet' selects which of the above forms are +-- allowed, and, when a label can be rendered as either Unicode +-- or ASCII, the Unicode-output form is chosen. +-- +-- 'IdnaFlags' determine whether the label content is validated +-- ('ALABELCHECK'), and what the validation entails ('NFCCHECK', +-- 'BIDICHECK'). +-- +-- Returns 'Left' 'ErrFormNotAllowed' when the label's content is +-- not compatible with any of the specified forms. +unparseLabelOpts + :: LabelFormSet + -> IdnaFlags + -> ShortByteString + -> Either IdnaError (Text, LabelForm) +unparseLabelOpts !forms !flagsIn sbs = runST do + let !flags = effectiveIdnaFlags flagsIn + !labelLen = SBS.length sbs + !cap0 = max 16 labelLen + mba0 <- newByteArray cap0 + r <- unparseLabelInto forms flags 0 mba0 cap0 0 sbs 0 labelLen + case r of + Left err -> pure (Left err) + Right (mba, _cap, !finalLen, !form) -> do + shrinkMutableByteArray mba finalLen + ByteArray b <- unsafeFreezeByteArray mba + pure (Right (Text (TA.ByteArray b) 0 finalLen, form)) + +-- | Render one label into the supplied output buffer, choosing a +-- 'LabelForm' from the supplied set. Used by both the single-label +-- 'unparseLabelOpts' and the domain-level 'unparseDomainOpts'. +-- +-- The 'IdnaFlags' argument is assumed to have already had its +-- implication lifts applied (the wrappers do this once, callers +-- in a loop need not repeat). The @lIdx@ argument is the label +-- index used when reporting 'ErrFormNotAllowed'. The +-- @sbs@\/@off@\/@len@ triple is the label's wire-byte slice +-- within its containing 'ShortByteString' -- the parent 'Domain' +-- for 'unparseDomainOpts', the single-label argument for +-- 'unparseLabelOpts'. +unparseLabelInto + :: forall s. + LabelFormSet + -> IdnaFlags + -> Int -- ^ lIdx + -> MutableByteArray s -> Int -> Int -- ^ output: mba, cap, cur + -> ShortByteString -> Int -> Int -- ^ input: sbs, off, len + -> ST s (Either IdnaError (MutableByteArray s, Int, Int, LabelForm)) +unparseLabelInto !forms !flags !lIdx + !mba0 !cap0 !cur0 + sbs !off !len = + case classifyWireBytes sbs off len of + WSWild -> admitAscii WILDLABEL + WSLdh -> admitAscii LDH + WSRldh -> admitAscii RLDH + WSAttrLeaf -> admitAscii ATTRLEAF + WSOctet -> admitAscii OCTET + WSXnPrefix -> dispatchXn + where + admitAscii !form + | not (form `memberLabelFormSet` forms) = + pure (Left (ErrFormNotAllowed lIdx form)) + | otherwise = do + (mba, cap, cur) <- writeEscapedLabel mba0 cap0 cur0 sbs off len + pure (Right (mba, cap, cur, form)) + + emitUnicode !form !cps = do + (mba, cap, cur) <- writeCodepointsUtf8 mba0 cap0 cur0 cps + pure (Right (mba, cap, cur, form)) + + -- @\"xn--\"@ branch. Lower-case the body bytes into a fresh + -- buffer, then dispatch on 'ALABELCHECK' and the underlying + -- classification. + dispatchXn = do + let !bodyLen = len - 4 + bodyBufM <- newByteArray bodyLen + let copyLc !i + | i >= bodyLen = pure () + | otherwise = do + let !b = SBS.index sbs (off + 4 + i) + !c | b - 0x41 < 26 = b + 0x20 + | otherwise = b + writeByteArray bodyBufM i c + copyLc (i + 1) + copyLc 0 + body <- unsafeFreezeByteArray bodyBufM + if ALABELCHECK `meetsIdnaFlags` flags + then do + rt <- alabelRoundTrip body 0 bodyLen flags + case rt of + Right cps -> renderAlabel cps + Left _ -> renderFakea body bodyLen + else + -- 'ALABELCHECK' off: the label is 'ALABEL' by fiat. + -- A 'ULABEL' render uses Punycode-only (lax) decode + -- since no strict round-trip was requested; if even + -- Punycode fails, fall through to the literal branch. + if ULABEL `memberLabelFormSet` forms + then do + d <- laxDecode body bodyLen + case d of + Right cps -> emitUnicode ULABEL cps + Left _ -> admitXnLit ALABEL + else admitXnLit ALABEL + + -- Render the original @\"xn--\"@ wire bytes as ASCII literal + -- under the given classification. Pure LDH on the wire, so + -- 'writeEscapedLabel' here just copies bytes. + admitXnLit !form + | not (form `memberLabelFormSet` forms) = + pure (Left (ErrFormNotAllowed lIdx form)) + | otherwise = do + (mba, cap, cur) <- writeEscapedLabel mba0 cap0 cur0 sbs off len + pure (Right (mba, cap, cur, form)) + + -- Underlying 'ALABEL': choose 'ULABEL' (Unicode) when + -- requested, else 'ALABEL' (literal), else reject. + renderAlabel !cps + | ULABEL `memberLabelFormSet` forms = emitUnicode ULABEL cps + | ALABEL `memberLabelFormSet` forms = admitXnLit ALABEL + | otherwise = pure (Left (ErrFormNotAllowed lIdx ALABEL)) + + -- Underlying 'FAKEA': choose 'LAXULABEL' (Unicode via lax + -- decode) when requested, else 'FAKEA' (literal), else + -- reject. When 'LAXULABEL' is requested but Punycode itself + -- fails so there are no codepoints to render, fall through + -- to the literal branch. + renderFakea !body !bodyLen + | LAXULABEL `memberLabelFormSet` forms = do + d <- laxDecode body bodyLen + case d of + Right cps -> emitUnicode LAXULABEL cps + Left _ -> admitXnLit FAKEA + | FAKEA `memberLabelFormSet` forms = admitXnLit FAKEA + | otherwise = pure (Left (ErrFormNotAllowed lIdx FAKEA)) + +-- | Render a 'Domain' as 'Text' in presentation form, paired with +-- the per-label 'LabelInfo'. The output has no trailing dot, +-- except for the root domain which renders as @\".\"@. +-- +-- See 'unparseLabelOpts' for how 'LabelFormSet' and 'IdnaFlags' +-- affect the validation and output form of each label. +-- +-- Under 'BIDICHECK', the RFC 5893 cross-label rules are enforced +-- whenever any label carries an RTL or AL codepoint. Failure +-- yields 'Left' 'ErrCrossLabelBidi'. 'ASCIIFALLBACK' degrades +-- that failure into a successful retry with 'ULABEL' and +-- 'LAXULABEL' dropped from the set, so every label renders in +-- its literal ASCII form. +-- +-- The cross-label check is suppressed only when at least one +-- label is /actually/ classified as 'LAXULABEL' in the returned +-- 'LabelInfo' — not merely because 'LAXULABEL' is in the +-- supplied set. A caller who admits LAXULABEL has already +-- accepted per-label validation failures, so a cross-label +-- rejection on top would be contradictory. A domain whose +-- labels all classify cleanly but whose cross-label rules +-- fail still surfaces as 'ErrCrossLabelBidi'. +-- +-- If any label's classification falls outside the supplied set, +-- the call returns 'Left' 'ErrFormNotAllowed'. +unparseDomainOpts + :: LabelFormSet + -> IdnaFlags + -> Domain + -> Either IdnaError (Text, LabelInfo) +unparseDomainOpts !forms !flagsIn (Domain sbs) + | sblen <= 1 = Right (T.singleton '.', mkLabelInfo []) + | otherwise = case attempt forms of + Right res -> Right res + Left err@(ErrCrossLabelBidi _ _) + | ASCIIFALLBACK `meetsIdnaFlags` flags -> + attempt (forms `withoutLabelFormSet` uOutForms) + | otherwise -> Left err + Left err -> Left err + where + !flags = effectiveIdnaFlags flagsIn + !sblen = SBS.length sbs + !checkBidi = BIDICHECK `meetsIdnaFlags` flags + !uOutForms = foldMap' labelFormToSet [ULABEL, LAXULABEL] + + -- One render pass under the given 'LabelFormSet'. Allocates a + -- buffer, walks labels, and on completion either returns the + -- ('Text', 'LabelInfo') pair, or surfaces a cross-label Bidi + -- violation (when 'BIDICHECK' is on and both an RTL/AL/AN + -- trigger and a per-label Rules 1-6 violation were observed), + -- or surfaces the per-label error from 'unparseLabelInto'. + -- + -- Cross-label Bidi enforcement is suppressed when at least + -- one label in the output was actually classified as + -- 'LAXULABEL': admitting a LAXULABEL is the caller's opt-in + -- to per-label permissiveness, so surfacing a cross-label + -- rejection on top of that opt-in would be contradictory. + -- When no label needed LAXULABEL admission --- even if + -- 'LAXULABEL' /is/ in the supplied set --- the cross-label + -- check still fires, so a domain whose individual labels + -- happen to pass per-label rules but whose cross-label rules + -- fail (e.g. @\"_tcp.אב.example\"@: ATTRLEAF + clean + -- ULABEL + LDH, with the cross-label rule catching @\"_tcp\"@ + -- in RTL context) still surfaces as 'ErrCrossLabelBidi'. + attempt :: LabelFormSet -> Either IdnaError (Text, LabelInfo) + attempt !activeForms = runST do + let !cap0 = max 32 sblen + mba0 <- newByteArray cap0 + r <- walk activeForms mba0 cap0 0 0 0 [] False Nothing + case r of + Left err -> pure (Left err) + Right (mba, _cap, !finalLen, !revForms, !triggered, !mViol) -> do + let !laxAdmitted = any (== LAXULABEL) revForms + case mViol of + Just (idx, rule) + | triggered && checkBidi && not laxAdmitted -> + pure (Left (ErrCrossLabelBidi idx rule)) + _ -> do + shrinkMutableByteArray mba finalLen + ByteArray b <- unsafeFreezeByteArray mba + let !text = Text (TA.ByteArray b) 0 finalLen + !info = mkLabelInfo (reverse revForms) + pure (Right (text, info)) + + -- Walk labels left-to-right, rendering each into the shared + -- output buffer via 'unparseLabelInto' and scanning the + -- just-written bytes (under 'BIDICHECK') to update the + -- cross-label Bidi state. Carries the buffer state, the + -- in-reverse 'LabelForm' accumulator, and the Bidi state + -- (whether an RTL/AL/AN codepoint has been seen anywhere, and + -- the index of the first label that failed Rules 1-6 under + -- forced check). + walk :: forall s. LabelFormSet + -> MutableByteArray s -> Int -> Int -> Int -> Int + -> [LabelForm] -> Bool -> Maybe (Int, BidiRuleViolation) + -> ST s ( Either IdnaError + ( MutableByteArray s, Int, Int, [LabelForm] + , Bool, Maybe (Int, BidiRuleViolation) ) ) + walk !activeForms !mba !cap !cur !off !lIdx + !revForms !triggered !mViol = do + let !lLen = fromIntegral (SBS.index sbs off) :: Int + if lLen == 0 + then pure (Right (mba, cap, cur, revForms, triggered, mViol)) + else do + -- @\'.\'@ separator before all labels after the first. + (mba1, cap1, cur1) <- + if lIdx > 0 + then writeRawByte mba cap cur 0x2E + else pure (mba, cap, cur) + r <- unparseLabelInto activeForms flags lIdx + mba1 cap1 cur1 + sbs (off + 1) lLen + case r of + Left err -> pure (Left err) + Right (mba2, cap2, cur2, !form) -> do + (!triggered', !mViol') <- + if checkBidi + then do + !sm <- bidiSummaryFromMutableUtf8Range mba2 cur1 cur2 + let !trig' = addBidiTrigger triggered sm + !mViol' + | Just _ <- mViol = mViol + | otherwise = case checkBidiSummary True sm of + Just rule -> Just (lIdx, rule) + Nothing -> Nothing + pure (trig', mViol') + else pure (triggered, mViol) + walk activeForms mba2 cap2 cur2 + (off + 1 + lLen) (lIdx + 1) + (form : revForms) triggered' mViol' + +-- | Build a 'BidiSummary' by walking a UTF-8-encoded byte range +-- within a 'MutableByteArray'. Used by 'unparseDomainOpts' to +-- score each label's Bidi contribution by scanning the bytes +-- it just emitted, without freezing the buffer (which is still +-- being built into). The bytes are well-formed UTF-8 by +-- construction (our writers produced them); the decoder branches +-- on the leading-byte prefix and assumes the indicated number of +-- continuation bytes follow. +bidiSummaryFromMutableUtf8Range + :: MutableByteArray s -> Int -> Int -> ST s BidiSummary +bidiSummaryFromMutableUtf8Range !mba !start !end = go start emptyBidiSummary + where + go !i !sm + | i >= end = pure sm + | otherwise = do + !b0 <- readByteArray @Word8 mba i + if | b0 < 0x80 -> + let !cp = fromIntegral b0 + in go (i + 1) (extendBidiSummary sm (bidiClassCp cp)) + | b0 < 0xE0 -> do + !b1 <- readByteArray @Word8 mba (i + 1) + let !cp = ((fromIntegral b0 .&. 0x1F) `unsafeShiftL` 6) + .|. (fromIntegral b1 .&. 0x3F) + go (i + 2) (extendBidiSummary sm (bidiClassCp cp)) + | b0 < 0xF0 -> do + !b1 <- readByteArray @Word8 mba (i + 1) + !b2 <- readByteArray @Word8 mba (i + 2) + let !cp = ((fromIntegral b0 .&. 0x0F) `unsafeShiftL` 12) + .|. ((fromIntegral b1 .&. 0x3F) `unsafeShiftL` 6) + .|. (fromIntegral b2 .&. 0x3F) + go (i + 3) (extendBidiSummary sm (bidiClassCp cp)) + | otherwise -> do + !b1 <- readByteArray @Word8 mba (i + 1) + !b2 <- readByteArray @Word8 mba (i + 2) + !b3 <- readByteArray @Word8 mba (i + 3) + let !cp = ((fromIntegral b0 .&. 0x07) `unsafeShiftL` 18) + .|. ((fromIntegral b1 .&. 0x3F) `unsafeShiftL` 12) + .|. ((fromIntegral b2 .&. 0x3F) `unsafeShiftL` 6) + .|. (fromIntegral b3 .&. 0x3F) + go (i + 4) (extendBidiSummary sm (bidiClassCp cp)) + +-- | The single-label counterpart to 'domainToUnicode'. Valid +-- A-labels decode to U-labels. Any other label renders as ASCII +-- (with escapes for bytes that need them). See 'unparseLabelOpts' +-- for finer control. +labelToUnicode :: ShortByteString -> Text +labelToUnicode sbs = case unparseLabelOpts forms defaultIdnaFlags sbs of + Right (t, _) -> t + Left _ -> error "labelToUnicode: unreachable" + where + !forms = foldMap' labelFormToSet + [LDH, RLDH, ULABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + +-- | The single-label counterpart to 'domainToAscii'. Output is +-- pure ASCII, A-labels stay literal. Other labels' wire bytes +-- are output with escapes as needed. See 'unparseLabelOpts' for +-- finer control. +labelToAscii :: ShortByteString -> Text +labelToAscii sbs = case unparseLabelOpts forms mempty sbs of + Right (t, _) -> t + Left _ -> error "labelToAscii: unreachable" + where + !forms = foldMap' labelFormToSet + [LDH, RLDH, ALABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + +-- | Lax A-label decoder: Punycode-decode the body and surface +-- the resulting codepoints unconditionally, /without/ U-label +-- validation or re-encode-and-compare. Returns @'Left' +-- 'BadPunycode'@ only when the Punycode body itself fails to +-- decode -- no other failure mode is possible without +-- validation. +laxDecode :: forall s. ByteArray -> Int -> ST s (Either AceReason (PrimArray Int)) +laxDecode !body !bodyLen = do + cpBuf <- newPrimArray maxCpsPerLabel + rDec <- punycodeDecode body 0 bodyLen cpBuf 0 maxCpsPerLabel + case rDec of + Left _ -> pure (Left BadPunycode) + Right cpCount -> Right <$> freezeCps cpBuf cpCount + +---------------------------------------------------------------------- +-- Domain helpers (default strict IDNA2008 options) +---------------------------------------------------------------------- + +-- | Parse a 'Text' as a domain name and return either the parsed +-- 'Domain' or an 'IdnaError'. Uses 'idnLabelForms' as the +-- permitted set of label forms @('LDH' | 'ALABEL' | 'ULABEL')@ and +-- 'defaultIdnaFlags' for the parser options. +mkDomain :: Text -> Either IdnaError Domain +mkDomain = fmap fst . parseDomain idnLabelForms + +-- | 'mkDomain' for 'String' input. Equivalent to @'mkDomain' . 'T.pack'@. +mkDomainStr :: String -> Either IdnaError Domain +mkDomainStr = mkDomain . T.pack + +-- | 'mkDomain' for a UTF-8 'ByteString'. +mkDomainUtf8 :: ByteString -> Either IdnaError Domain +mkDomainUtf8 = fmap fst . parseDomainUtf8 idnLabelForms defaultIdnaFlags + +-- | 'mkDomain' for a UTF-8 'ShortByteString'. +mkDomainShort :: ShortByteString -> Either IdnaError Domain +mkDomainShort = fmap fst . parseDomainShort idnLabelForms defaultIdnaFlags + +---------------------------------------------------------------------- +-- Tiny utilities +---------------------------------------------------------------------- + +-- | If @w@ is an ASCII decimal digit, return its numeric value (0..9); +-- otherwise return 'Nothing'. Returning the value avoids a redundant +-- subtraction at every call site. +asciiDigit :: Word8 -> Maybe Word8 +asciiDigit !w + | d <= 9 = Just d + | otherwise = Nothing + where + !d = w - 0x30 +{-# INLINE asciiDigit #-} + +isLdhCp :: Int -> Bool +isLdhCp !cp + | cp >= 0x30, cp <= 0x39 = True + | cp >= 0x41, cp <= 0x5A = True + | cp >= 0x61, cp <= 0x7A = True + | cp == 0x2D = True + | otherwise = False +{-# INLINE isLdhCp #-} + +-- | Walk @cpBuf[0..cnt)@ and return 'True' iff any codepoint is +-- ASCII (@< 0x80@) and not in the LDH alphabet. Used by +-- 'uLabelEncode' to detect labels whose presentation form +-- contains non-LDH ASCII that 'LAXULABEL' must /not/ rescue, +-- regardless of which codepoint the validator surfaced as the +-- first failure. +hasNonLdhAsciiCps :: MutablePrimArray s Int -> Int -> ST s Bool +hasNonLdhAsciiCps !cpBuf !cnt = go 0 + where + go !i + | i >= cnt = pure False + | otherwise = do + !cp <- readPrimArray cpBuf i + if cp < 0x80 && not (isLdhCp cp) + then pure True + else go (i + 1) + +---------------------------------------------------------------------- +-- Case-fold helpers (used by 'MAPCASE') +---------------------------------------------------------------------- + +-- | Lowercase ASCII A-Z in @cpBuf[0..cnt)@ in place. Idempotent; +-- non-ASCII codepoints and non-letter ASCII bytes are left alone. +asciiDownCaseBuf :: forall s. MutablePrimArray s Int -> Int -> ST s () +asciiDownCaseBuf !buf !cnt = go 0 + where + go :: Int -> ST s () + go !i + | i >= cnt = pure () + | otherwise = do + cp <- readPrimArray buf i + when (cp >= 0x41 && cp <= 0x5A) + (writePrimArray buf i (cp + 0x20)) + go (i + 1) + +-- | Apply Unicode simple lowercase mapping to every non-ASCII +-- codepoint in @cpBuf[0..cnt)@ in place, using +-- "Text.IDNA2008.Internal.Case"'s 'Case.toLower' -- the +-- @Simple_Lowercase_Mapping@ table pinned to the UCD version this +-- library tracks (see @x-unicode-version@ in @idna2008.cabal@), +-- independent of the host GHC's @base@ vintage. ASCII codepoints +-- are skipped: the pre-classification 'asciiDownCaseBuf' pass +-- already handled them, and 'Case.toLower' on already-lowercase +-- ASCII is identity. +unicodeDownCaseBuf :: forall s. MutablePrimArray s Int -> Int -> ST s () +unicodeDownCaseBuf !buf !cnt = go 0 + where + go :: Int -> ST s () + go !i + | i >= cnt = pure () + | otherwise = do + cp <- readPrimArray buf i + when (cp >= 0x80) $ do + let !cp' = ord (Case.toLower (chr cp)) + when (cp' /= cp) (writePrimArray buf i cp') + go (i + 1) + +-- | Apply 'widthMapCp' to every codepoint in @cpBuf[0..cnt)@ in +-- place: codepoints with @\@ or @\@ decomposition +-- are replaced by their single-codepoint target. Used by the +-- 'MAPWIDTH' option (RFC 5895 section 2.2). +unicodeWidthBuf :: forall s. MutablePrimArray s Int -> Int -> ST s () +unicodeWidthBuf !buf !cnt = go 0 + where + go :: Int -> ST s () + go !i + | i >= cnt = pure () + | otherwise = do + cp <- readPrimArray buf i + let !cp' = widthMapCp cp + when (cp' /= cp) (writePrimArray buf i cp') + go (i + 1) + +-- | Apply the 'MAPUTS46' allow-list ('uts46Lookup') to every +-- codepoint in @cpBuf[0..cnt)@ in place. Source codepoints in the +-- allow-list are replaced by their target sequence (1:n, where +-- @n >= 1@); other codepoints are left alone. Returns the new +-- codepoint count. +-- +-- If the post-expansion count exceeds 'maxCpsPerLabel', the buffer +-- is left untouched and the returned count is the would-be size, +-- which the caller should treat as 'ErrLabelTooLong'. Otherwise +-- expansion happens in place, walking right-to-left so the write +-- head never overruns the read head — positions @[0..i-1]@ of the +-- input each contribute @>= 1@ codepoint to the output, so the +-- total output prefix is @>= i@ at all times. +unicodeUTS46Buf + :: forall s. MutablePrimArray s Int -> Int -> ST s Int +unicodeUTS46Buf !cpBuf !cnt = do + !hit <- scan 0 + if not hit + then pure cnt + else do + !out <- sizeOut 0 0 + if out > maxCpsPerLabel + then pure out + else do + expandInPlace (cnt - 1) (out - 1) + pure out + where + -- Fast scan: any source codepoint in the allow-list? + scan :: Int -> ST s Bool + scan !i + | i >= cnt = pure False + | otherwise = do + cp <- readPrimArray cpBuf i + case uts46Lookup cp of + Just _ -> pure True + Nothing -> scan (i + 1) + + -- Total post-expansion codepoint count. + sizeOut :: Int -> Int -> ST s Int + sizeOut !i !acc + | i >= cnt = pure acc + | otherwise = do + cp <- readPrimArray cpBuf i + let !n = case uts46Lookup cp of + Just tgts -> length tgts + Nothing -> 1 + sizeOut (i + 1) (acc + n) + + -- Walk right-to-left, expanding in place. + expandInPlace :: Int -> Int -> ST s () + expandInPlace !inIdx !outIdx + | inIdx < 0 = pure () + | otherwise = do + cp <- readPrimArray cpBuf inIdx + case uts46Lookup cp of + Nothing -> do + writePrimArray cpBuf outIdx cp + expandInPlace (inIdx - 1) (outIdx - 1) + Just tgts -> do + let !n = length tgts + writeTargets cpBuf (outIdx - n + 1) tgts + expandInPlace (inIdx - 1) (outIdx - n) + + writeTargets :: MutablePrimArray s Int -> Int -> [Int] -> ST s () + writeTargets !_ !_ [] = pure () + writeTargets !buf !i (x:xs) = do + writePrimArray buf i x + writeTargets buf (i + 1) xs + +---------------------------------------------------------------------- +-- Domain data type +---------------------------------------------------------------------- + +-- | A wire-form fully-qualified domain name. +-- +-- Stored as a 'ShortByteString' to avoid pinning, since the +-- typical name is short and the buffer outlives the parser +-- run. The encoding is the standard DNS uncompressed wire +-- form: each label is one length byte followed by that many +-- content bytes, with a final zero-length label terminating +-- the name. +-- +newtype Domain = Domain_ ShortByteString + deriving (Eq, Ord, TH.Lift) + +instance Show Domain where + showsPrec p = showsPrec p . T.unpack . domainToAscii + +-- | Bidirectional pattern synonym for 'Domain'. +-- +-- As a pattern (@case dom of {Domain sbs -> ... }@) it matches any +-- 'Domain' and binds the underlying wire-form 'ShortByteString'. +-- +-- As an expression (@Domain sbs@) it builds a 'Domain' from the +-- supplied bytes after checking that they form a well-formed DNS +-- wire encoding. Malformed input raises an error; the input can +-- be pre-validated via 'isValidWireForm'. The parser functions +-- never produce malformed wire bytes. +pattern Domain :: ShortByteString -> Domain +pattern Domain sbs <- Domain_ sbs + where + Domain sbs + | isValidWireForm sbs = Domain_ sbs + | otherwise = errorWithoutStackTrace + "Text.IDNA2008.Domain: malformed wire bytes" + +{-# COMPLETE Domain #-} + +-- | The whole wire-form buffer as a 'ShortByteString'. +-- Zero-copy. +wireBytesShort :: Domain -> ShortByteString +wireBytesShort (Domain sbs) = sbs + +-- | The whole wire-form buffer as a regular 'ByteString'. +-- Zero-copy. +wireBytes :: Domain -> ByteString +wireBytes (Domain sbs) = SBS.fromShort sbs + +-- | Walk the wire form and yield each label's content bytes, +-- skipping the leading length byte and stopping at the +-- terminal zero-length root label. +toLabels :: Domain -> [ShortByteString] +toLabels (Domain sbs) = toLabelsFromWire sbs + +-- | Wire-bytes variant of 'toLabels': accepts a 'ShortByteString' +-- assumed to be valid DNS wire form (caller's responsibility, no +-- validation performed) and yields each label's content bytes. +toLabelsFromWire :: ShortByteString -> [ShortByteString] +toLabelsFromWire sbs@(SBS sba) = go 0 + where + sblen = SBS.length sbs + ba = ByteArray sba + go !off + | off < sblen + , llen <- fromIntegral (A.indexByteArray ba off :: Word8) + , off' <- off + llen + 1 + = if | llen > 0 && llen < 64 && off' < sblen + , (ByteArray lba) <- A.cloneByteArray ba (off + 1) llen + -> SBS lba : go off' + | llen == 0 && off' == sblen -> [] + | otherwise -> error "Invalid wire form domain" + | otherwise = error "Invalid wire form domain" + +-- | Check that the supplied 'ShortByteString' is a well-formed +-- DNS wire-form name: +-- +-- * overall length is in @[1, 255]@ (DNS message-section cap +-- on a single name); +-- * the final byte is the @NUL@ root terminator; +-- * every other length byte is in @[1, 63]@ (no compression +-- pointers, no reserved length-byte values, no zero-length +-- non-root labels); +-- * the label walk lands exactly on the terminator (no +-- truncation, no overrun, no trailing junk past the root). +-- +isValidWireForm :: ShortByteString -> Bool +isValidWireForm sbs@(SBS sba) = + n >= 1 && n <= 255 && walk 0 + where + !ba = ByteArray sba + !n = SBS.length sbs + walk !off + | off >= n = False + | llen == 0 = off == n - 1 + | llen <= 63 = walk (off + llen + 1) + | otherwise = False + where + !llen = fromIntegral (A.indexByteArray ba off :: Word8) :: Int + +---------------------------------------------------------------------- +-- Literal domains (TH splices) +---------------------------------------------------------------------- + +-- | Template-Haskell typed splice for a compile-time 'Domain' +-- literal. The parser argument is any +-- @'T.Text' -> Either e Domain@ function, so a single primitive +-- subsumes every kind of literal the library can express. +-- Typical idioms: +-- +-- > -- Strict default: use 'mkDomain' directly. +-- > dom :: Domain +-- > dom = $$(dnLit mkDomain "example.com") +-- +-- > -- Custom flags via @(parseDomainOpts forms flags)@: +-- > dom :: Domain +-- > dom = $$(let forms = idnLabelForms +-- > flags = defaultIdnaFlags <> allIdnaMappings +-- > parse = fmap fst . parseDomainOpts forms flags +-- > in dnLit parse "Example.COM") +-- +-- > -- Permit an attrleaf or other less-restrictive form set: +-- > dom :: Domain +-- > dom = $$(let forms = idnLabelForms <+> ATTRLEAF +-- > flags = defaultIdnaFlags +-- > parse = fmap fst . parseDomainOpts forms flags +-- > in dnLit parse "_dmarc.example.com") +-- +-- The parser only performs whatever checks its own flags enable. +-- The cross-label half of RFC 5893 is a presentation-time +-- concern and is not enforced by 'parseDomain' / +-- 'parseDomainOpts'; callers wanting it at literal time should +-- wrap their parser with a post-parse call to 'unparseDomainOpts' +-- and raise any error as a parse failure. +-- +-- The emitted splice is a constant 'Domain' value: no runtime +-- IDNA work. +dnLit :: forall e m. (Show e, MonadFail m, TH.Quote m) + => (T.Text -> Either e Domain) + -> String + -> TH.Code m Domain +dnLit parse s = TH.joinCode case parse (T.pack s) of + Left e -> fail $ "Invalid domain-name literal " ++ show s + ++ ": " ++ show e + Right d -> pure (TH.liftTyped d) diff --git a/internal/Text/IDNA2008/Internal/Property.hs b/internal/Text/IDNA2008/Internal/Property.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Property.hs @@ -0,0 +1,124 @@ +-- | +-- Module : Text.IDNA2008.Internal.Property +-- Description : IDNA2008 codepoint disposition lookup +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- The IDNA disposition of a Unicode codepoint per +-- (and as +-- summarised by the IANA IDNA Parameters registry). Each codepoint +-- falls into exactly one of: +-- +-- * @PVALID@ -- permitted in U-labels with no further conditions +-- * @CONTEXTJ@ -- permitted only when the per-codepoint contextual +-- /joining/ rule is satisfied (currently U+200C +-- ZWNJ and U+200D ZWJ; rules in RFC 5892 section A.1, section A.2) +-- * @CONTEXTO@ -- permitted only when the per-codepoint contextual +-- /other/ rule is satisfied (rules in RFC 5892 section A.3+) +-- * @DISALLOWED@ -- not permitted in any U-label +-- * @UNASSIGNED@ -- not permitted; reserved for future Unicode +-- assignments +-- +-- The lookup is implemented as a binary search over a sorted array of +-- range starts. Each entry says \"from this codepoint until the next +-- entry's start, the disposition is /D/\". Total per-call cost is +-- @O(log N)@ array reads and zero allocations. +-- +-- == Coverage +-- +-- The full per-codepoint table is loaded from +-- "Text.IDNA2008.Internal.Property.Data", which is machine-generated +-- from the IANA @idna-tables-properties.csv@ registry. As of +-- 2024-04-26 the most recent IANA-curated derived table is aligned with +-- /Unicode 12.0.0/; the IETF position is that applications which need +-- newer Unicode coverage may compute derived properties themselves per +-- RFC 5892 section 2 -- only the contextual rules are normative across versions. +-- +-- Codepoints assigned in Unicode 12.1.0 or later that are not covered +-- by the 12.0.0 derivation will be reported as @UNASSIGNED@ here, which +-- this validator rejects as not-a-U-label. That is consistent with the +-- IETF reference and conservative for DNS use; if you specifically need +-- to admit newer codepoints, regenerate the data module from a derived +-- table you compute against the desired Unicode version. +module Text.IDNA2008.Internal.Property + ( -- * Disposition + IdnaDisposition(..) + -- * Lookup + , idnaDisposition + ) where + +import Data.Bits (unsafeShiftR) +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word8, Word32) + +import Text.IDNA2008.Internal.Property.Data + ( rangeCount + , rangeStarts + , rangeTags + ) + +---------------------------------------------------------------------- +-- Disposition type +---------------------------------------------------------------------- + +-- | IDNA2008 codepoint disposition. +data IdnaDisposition + = IdnaPVALID + | IdnaCONTEXTJ + | IdnaCONTEXTO + | IdnaDISALLOWED + | IdnaUNASSIGNED + deriving (Eq, Show, Enum, Bounded) + +-- | Map an internal 'Word8' tag back to a disposition. Tags +-- @0..4@ correspond to the constructors in declaration order. +{-# INLINE word8ToDisp #-} +word8ToDisp :: Word8 -> IdnaDisposition +word8ToDisp w = case w of + 0 -> IdnaPVALID + 1 -> IdnaCONTEXTJ + 2 -> IdnaCONTEXTO + 3 -> IdnaDISALLOWED + _ -> IdnaUNASSIGNED + +---------------------------------------------------------------------- +-- Public lookup +---------------------------------------------------------------------- + +-- | Return the IDNA2008 disposition of a codepoint. Out-of-range +-- inputs (negative or greater than @0x10FFFF@) report +-- 'IdnaDISALLOWED'. +idnaDisposition :: Int -> IdnaDisposition +idnaDisposition !cp + | cp < 0 || cp > 0x10FFFF = IdnaDISALLOWED + | otherwise = word8ToDisp (lookupTag cp) +{-# INLINE idnaDisposition #-} + +---------------------------------------------------------------------- +-- Binary search over the codegen'd range table +---------------------------------------------------------------------- + +-- | Binary search for the largest index @i@ in @[0 .. rangeCount-1]@ +-- such that @rangeStarts[i] <= cp@; return the disposition tag at that +-- index. Precondition: @cp >= 0 && cp <= 0x10FFFF@; combined with the +-- table starting at @0x000000@ this guarantees a hit. +{-# INLINE lookupTag #-} +lookupTag :: Int -> Word8 +lookupTag !cp = go 0 (rangeCount - 1) + where + !target = fromIntegral cp :: Word32 + + go !lo !hi + | lo >= hi = readTag lo + | otherwise = + let !mid = (lo + hi + 1) `unsafeShiftR` 1 + !midKey = readStart mid + in if midKey <= target + then go mid hi + else go lo (mid - 1) + + readStart i = indexByteArray rangeStarts i :: Word32 + readTag i = indexByteArray rangeTags i :: Word8 diff --git a/internal/Text/IDNA2008/Internal/Property/Data.hs b/internal/Text/IDNA2008/Internal/Property/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Property/Data.hs @@ -0,0 +1,525 @@ +-- | +-- Module : Text.IDNA2008.Internal.Property.Data +-- Description : Codegen'd IDNA2008 codepoint disposition table +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED FROM idna-tables-properties.csv. +-- Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaProperty.py +-- +-- Source: https://www.iana.org/assignments/idna-tables-properties/ +-- Aligned with: Unicode 17.0.0 +-- Range count: 3066 +-- +-- Each disposition tag is a small Word8: +-- +-- * 0 -- PVALID +-- * 1 -- CONTEXTJ +-- * 2 -- CONTEXTO +-- * 3 -- DISALLOWED +-- * 4 -- UNASSIGNED +module Text.IDNA2008.Internal.Property.Data + ( rangeCount + , rangeStarts + , rangeTags + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +rangeCount :: Int +rangeCount = 3066 + +rangeStarts :: ByteArray +rangeStarts = PBA.byteArrayFromList + [ (0x000000 :: Word32), 0x00002D, 0x00002E, 0x000030, 0x00003A, 0x000061, 0x00007B, 0x0000B7 + , 0x0000B8, 0x0000DF, 0x0000F7, 0x0000F8, 0x000100, 0x000101, 0x000102, 0x000103 + , 0x000104, 0x000105, 0x000106, 0x000107, 0x000108, 0x000109, 0x00010A, 0x00010B + , 0x00010C, 0x00010D, 0x00010E, 0x00010F, 0x000110, 0x000111, 0x000112, 0x000113 + , 0x000114, 0x000115, 0x000116, 0x000117, 0x000118, 0x000119, 0x00011A, 0x00011B + , 0x00011C, 0x00011D, 0x00011E, 0x00011F, 0x000120, 0x000121, 0x000122, 0x000123 + , 0x000124, 0x000125, 0x000126, 0x000127, 0x000128, 0x000129, 0x00012A, 0x00012B + , 0x00012C, 0x00012D, 0x00012E, 0x00012F, 0x000130, 0x000131, 0x000132, 0x000135 + , 0x000136, 0x000137, 0x000139, 0x00013A, 0x00013B, 0x00013C, 0x00013D, 0x00013E + , 0x00013F, 0x000142, 0x000143, 0x000144, 0x000145, 0x000146, 0x000147, 0x000148 + , 0x000149, 0x00014B, 0x00014C, 0x00014D, 0x00014E, 0x00014F, 0x000150, 0x000151 + , 0x000152, 0x000153, 0x000154, 0x000155, 0x000156, 0x000157, 0x000158, 0x000159 + , 0x00015A, 0x00015B, 0x00015C, 0x00015D, 0x00015E, 0x00015F, 0x000160, 0x000161 + , 0x000162, 0x000163, 0x000164, 0x000165, 0x000166, 0x000167, 0x000168, 0x000169 + , 0x00016A, 0x00016B, 0x00016C, 0x00016D, 0x00016E, 0x00016F, 0x000170, 0x000171 + , 0x000172, 0x000173, 0x000174, 0x000175, 0x000176, 0x000177, 0x000178, 0x00017A + , 0x00017B, 0x00017C, 0x00017D, 0x00017E, 0x00017F, 0x000180, 0x000181, 0x000183 + , 0x000184, 0x000185, 0x000186, 0x000188, 0x000189, 0x00018C, 0x00018E, 0x000192 + , 0x000193, 0x000195, 0x000196, 0x000199, 0x00019C, 0x00019E, 0x00019F, 0x0001A1 + , 0x0001A2, 0x0001A3, 0x0001A4, 0x0001A5, 0x0001A6, 0x0001A8, 0x0001A9, 0x0001AA + , 0x0001AC, 0x0001AD, 0x0001AE, 0x0001B0, 0x0001B1, 0x0001B4, 0x0001B5, 0x0001B6 + , 0x0001B7, 0x0001B9, 0x0001BC, 0x0001BD, 0x0001C4, 0x0001CE, 0x0001CF, 0x0001D0 + , 0x0001D1, 0x0001D2, 0x0001D3, 0x0001D4, 0x0001D5, 0x0001D6, 0x0001D7, 0x0001D8 + , 0x0001D9, 0x0001DA, 0x0001DB, 0x0001DC, 0x0001DE, 0x0001DF, 0x0001E0, 0x0001E1 + , 0x0001E2, 0x0001E3, 0x0001E4, 0x0001E5, 0x0001E6, 0x0001E7, 0x0001E8, 0x0001E9 + , 0x0001EA, 0x0001EB, 0x0001EC, 0x0001ED, 0x0001EE, 0x0001EF, 0x0001F1, 0x0001F5 + , 0x0001F6, 0x0001F9, 0x0001FA, 0x0001FB, 0x0001FC, 0x0001FD, 0x0001FE, 0x0001FF + , 0x000200, 0x000201, 0x000202, 0x000203, 0x000204, 0x000205, 0x000206, 0x000207 + , 0x000208, 0x000209, 0x00020A, 0x00020B, 0x00020C, 0x00020D, 0x00020E, 0x00020F + , 0x000210, 0x000211, 0x000212, 0x000213, 0x000214, 0x000215, 0x000216, 0x000217 + , 0x000218, 0x000219, 0x00021A, 0x00021B, 0x00021C, 0x00021D, 0x00021E, 0x00021F + , 0x000220, 0x000221, 0x000222, 0x000223, 0x000224, 0x000225, 0x000226, 0x000227 + , 0x000228, 0x000229, 0x00022A, 0x00022B, 0x00022C, 0x00022D, 0x00022E, 0x00022F + , 0x000230, 0x000231, 0x000232, 0x000233, 0x00023A, 0x00023C, 0x00023D, 0x00023F + , 0x000241, 0x000242, 0x000243, 0x000247, 0x000248, 0x000249, 0x00024A, 0x00024B + , 0x00024C, 0x00024D, 0x00024E, 0x00024F, 0x0002B0, 0x0002B9, 0x0002C2, 0x0002C6 + , 0x0002D2, 0x0002EC, 0x0002ED, 0x0002EE, 0x0002EF, 0x000300, 0x000340, 0x000342 + , 0x000343, 0x000346, 0x00034F, 0x000350, 0x000370, 0x000371, 0x000372, 0x000373 + , 0x000374, 0x000375, 0x000376, 0x000377, 0x000378, 0x00037A, 0x00037B, 0x00037E + , 0x000380, 0x000384, 0x00038B, 0x00038C, 0x00038D, 0x00038E, 0x000390, 0x000391 + , 0x0003A2, 0x0003A3, 0x0003AC, 0x0003CF, 0x0003D7, 0x0003D8, 0x0003D9, 0x0003DA + , 0x0003DB, 0x0003DC, 0x0003DD, 0x0003DE, 0x0003DF, 0x0003E0, 0x0003E1, 0x0003E2 + , 0x0003E3, 0x0003E4, 0x0003E5, 0x0003E6, 0x0003E7, 0x0003E8, 0x0003E9, 0x0003EA + , 0x0003EB, 0x0003EC, 0x0003ED, 0x0003EE, 0x0003EF, 0x0003F0, 0x0003F3, 0x0003F4 + , 0x0003F8, 0x0003F9, 0x0003FB, 0x0003FD, 0x000430, 0x000460, 0x000461, 0x000462 + , 0x000463, 0x000464, 0x000465, 0x000466, 0x000467, 0x000468, 0x000469, 0x00046A + , 0x00046B, 0x00046C, 0x00046D, 0x00046E, 0x00046F, 0x000470, 0x000471, 0x000472 + , 0x000473, 0x000474, 0x000475, 0x000476, 0x000477, 0x000478, 0x000479, 0x00047A + , 0x00047B, 0x00047C, 0x00047D, 0x00047E, 0x00047F, 0x000480, 0x000481, 0x000482 + , 0x000483, 0x000488, 0x00048B, 0x00048C, 0x00048D, 0x00048E, 0x00048F, 0x000490 + , 0x000491, 0x000492, 0x000493, 0x000494, 0x000495, 0x000496, 0x000497, 0x000498 + , 0x000499, 0x00049A, 0x00049B, 0x00049C, 0x00049D, 0x00049E, 0x00049F, 0x0004A0 + , 0x0004A1, 0x0004A2, 0x0004A3, 0x0004A4, 0x0004A5, 0x0004A6, 0x0004A7, 0x0004A8 + , 0x0004A9, 0x0004AA, 0x0004AB, 0x0004AC, 0x0004AD, 0x0004AE, 0x0004AF, 0x0004B0 + , 0x0004B1, 0x0004B2, 0x0004B3, 0x0004B4, 0x0004B5, 0x0004B6, 0x0004B7, 0x0004B8 + , 0x0004B9, 0x0004BA, 0x0004BB, 0x0004BC, 0x0004BD, 0x0004BE, 0x0004BF, 0x0004C0 + , 0x0004C2, 0x0004C3, 0x0004C4, 0x0004C5, 0x0004C6, 0x0004C7, 0x0004C8, 0x0004C9 + , 0x0004CA, 0x0004CB, 0x0004CC, 0x0004CD, 0x0004CE, 0x0004D0, 0x0004D1, 0x0004D2 + , 0x0004D3, 0x0004D4, 0x0004D5, 0x0004D6, 0x0004D7, 0x0004D8, 0x0004D9, 0x0004DA + , 0x0004DB, 0x0004DC, 0x0004DD, 0x0004DE, 0x0004DF, 0x0004E0, 0x0004E1, 0x0004E2 + , 0x0004E3, 0x0004E4, 0x0004E5, 0x0004E6, 0x0004E7, 0x0004E8, 0x0004E9, 0x0004EA + , 0x0004EB, 0x0004EC, 0x0004ED, 0x0004EE, 0x0004EF, 0x0004F0, 0x0004F1, 0x0004F2 + , 0x0004F3, 0x0004F4, 0x0004F5, 0x0004F6, 0x0004F7, 0x0004F8, 0x0004F9, 0x0004FA + , 0x0004FB, 0x0004FC, 0x0004FD, 0x0004FE, 0x0004FF, 0x000500, 0x000501, 0x000502 + , 0x000503, 0x000504, 0x000505, 0x000506, 0x000507, 0x000508, 0x000509, 0x00050A + , 0x00050B, 0x00050C, 0x00050D, 0x00050E, 0x00050F, 0x000510, 0x000511, 0x000512 + , 0x000513, 0x000514, 0x000515, 0x000516, 0x000517, 0x000518, 0x000519, 0x00051A + , 0x00051B, 0x00051C, 0x00051D, 0x00051E, 0x00051F, 0x000520, 0x000521, 0x000522 + , 0x000523, 0x000524, 0x000525, 0x000526, 0x000527, 0x000528, 0x000529, 0x00052A + , 0x00052B, 0x00052C, 0x00052D, 0x00052E, 0x00052F, 0x000530, 0x000531, 0x000557 + , 0x000559, 0x00055A, 0x000560, 0x000587, 0x000588, 0x000589, 0x00058B, 0x00058D + , 0x000590, 0x000591, 0x0005BE, 0x0005BF, 0x0005C0, 0x0005C1, 0x0005C3, 0x0005C4 + , 0x0005C6, 0x0005C7, 0x0005C8, 0x0005D0, 0x0005EB, 0x0005EF, 0x0005F3, 0x0005F5 + , 0x000600, 0x000610, 0x00061B, 0x000620, 0x000640, 0x000641, 0x000660, 0x00066A + , 0x00066E, 0x000675, 0x000679, 0x0006D4, 0x0006D5, 0x0006DD, 0x0006DF, 0x0006E9 + , 0x0006EA, 0x0006F0, 0x0006FA, 0x000700, 0x00070E, 0x00070F, 0x000710, 0x00074B + , 0x00074D, 0x0007B2, 0x0007C0, 0x0007F6, 0x0007FB, 0x0007FD, 0x0007FE, 0x000800 + , 0x00082E, 0x000830, 0x00083F, 0x000840, 0x00085C, 0x00085E, 0x00085F, 0x000860 + , 0x00086B, 0x000870, 0x000888, 0x000889, 0x000890, 0x000892, 0x000897, 0x0008E2 + , 0x0008E3, 0x000958, 0x000960, 0x000964, 0x000966, 0x000970, 0x000971, 0x000984 + , 0x000985, 0x00098D, 0x00098F, 0x000991, 0x000993, 0x0009A9, 0x0009AA, 0x0009B1 + , 0x0009B2, 0x0009B3, 0x0009B6, 0x0009BA, 0x0009BC, 0x0009C5, 0x0009C7, 0x0009C9 + , 0x0009CB, 0x0009CF, 0x0009D7, 0x0009D8, 0x0009DC, 0x0009DE, 0x0009DF, 0x0009E0 + , 0x0009E4, 0x0009E6, 0x0009F2, 0x0009FC, 0x0009FD, 0x0009FE, 0x0009FF, 0x000A01 + , 0x000A04, 0x000A05, 0x000A0B, 0x000A0F, 0x000A11, 0x000A13, 0x000A29, 0x000A2A + , 0x000A31, 0x000A32, 0x000A33, 0x000A34, 0x000A35, 0x000A36, 0x000A37, 0x000A38 + , 0x000A3A, 0x000A3C, 0x000A3D, 0x000A3E, 0x000A43, 0x000A47, 0x000A49, 0x000A4B + , 0x000A4E, 0x000A51, 0x000A52, 0x000A59, 0x000A5C, 0x000A5D, 0x000A5E, 0x000A5F + , 0x000A66, 0x000A76, 0x000A77, 0x000A81, 0x000A84, 0x000A85, 0x000A8E, 0x000A8F + , 0x000A92, 0x000A93, 0x000AA9, 0x000AAA, 0x000AB1, 0x000AB2, 0x000AB4, 0x000AB5 + , 0x000ABA, 0x000ABC, 0x000AC6, 0x000AC7, 0x000ACA, 0x000ACB, 0x000ACE, 0x000AD0 + , 0x000AD1, 0x000AE0, 0x000AE4, 0x000AE6, 0x000AF0, 0x000AF2, 0x000AF9, 0x000B00 + , 0x000B01, 0x000B04, 0x000B05, 0x000B0D, 0x000B0F, 0x000B11, 0x000B13, 0x000B29 + , 0x000B2A, 0x000B31, 0x000B32, 0x000B34, 0x000B35, 0x000B3A, 0x000B3C, 0x000B45 + , 0x000B47, 0x000B49, 0x000B4B, 0x000B4E, 0x000B55, 0x000B58, 0x000B5C, 0x000B5E + , 0x000B5F, 0x000B64, 0x000B66, 0x000B70, 0x000B71, 0x000B72, 0x000B78, 0x000B82 + , 0x000B84, 0x000B85, 0x000B8B, 0x000B8E, 0x000B91, 0x000B92, 0x000B96, 0x000B99 + , 0x000B9B, 0x000B9C, 0x000B9D, 0x000B9E, 0x000BA0, 0x000BA3, 0x000BA5, 0x000BA8 + , 0x000BAB, 0x000BAE, 0x000BBA, 0x000BBE, 0x000BC3, 0x000BC6, 0x000BC9, 0x000BCA + , 0x000BCE, 0x000BD0, 0x000BD1, 0x000BD7, 0x000BD8, 0x000BE6, 0x000BF0, 0x000BFB + , 0x000C00, 0x000C0D, 0x000C0E, 0x000C11, 0x000C12, 0x000C29, 0x000C2A, 0x000C3A + , 0x000C3C, 0x000C45, 0x000C46, 0x000C49, 0x000C4A, 0x000C4E, 0x000C55, 0x000C57 + , 0x000C58, 0x000C5B, 0x000C5C, 0x000C5E, 0x000C60, 0x000C64, 0x000C66, 0x000C70 + , 0x000C77, 0x000C80, 0x000C84, 0x000C85, 0x000C8D, 0x000C8E, 0x000C91, 0x000C92 + , 0x000CA9, 0x000CAA, 0x000CB4, 0x000CB5, 0x000CBA, 0x000CBC, 0x000CC5, 0x000CC6 + , 0x000CC9, 0x000CCA, 0x000CCE, 0x000CD5, 0x000CD7, 0x000CDC, 0x000CDF, 0x000CE0 + , 0x000CE4, 0x000CE6, 0x000CF0, 0x000CF1, 0x000CF4, 0x000D00, 0x000D0D, 0x000D0E + , 0x000D11, 0x000D12, 0x000D45, 0x000D46, 0x000D49, 0x000D4A, 0x000D4F, 0x000D50 + , 0x000D54, 0x000D58, 0x000D5F, 0x000D64, 0x000D66, 0x000D70, 0x000D7A, 0x000D80 + , 0x000D81, 0x000D84, 0x000D85, 0x000D97, 0x000D9A, 0x000DB2, 0x000DB3, 0x000DBC + , 0x000DBD, 0x000DBE, 0x000DC0, 0x000DC7, 0x000DCA, 0x000DCB, 0x000DCF, 0x000DD5 + , 0x000DD6, 0x000DD7, 0x000DD8, 0x000DE0, 0x000DE6, 0x000DF0, 0x000DF2, 0x000DF4 + , 0x000DF5, 0x000E01, 0x000E33, 0x000E34, 0x000E3B, 0x000E3F, 0x000E40, 0x000E4F + , 0x000E50, 0x000E5A, 0x000E5C, 0x000E81, 0x000E83, 0x000E84, 0x000E85, 0x000E86 + , 0x000E8B, 0x000E8C, 0x000EA4, 0x000EA5, 0x000EA6, 0x000EA7, 0x000EB3, 0x000EB4 + , 0x000EBE, 0x000EC0, 0x000EC5, 0x000EC6, 0x000EC7, 0x000EC8, 0x000ECF, 0x000ED0 + , 0x000EDA, 0x000EDC, 0x000EDE, 0x000EE0, 0x000F00, 0x000F01, 0x000F0B, 0x000F0C + , 0x000F18, 0x000F1A, 0x000F20, 0x000F2A, 0x000F35, 0x000F36, 0x000F37, 0x000F38 + , 0x000F39, 0x000F3A, 0x000F3E, 0x000F43, 0x000F44, 0x000F48, 0x000F49, 0x000F4D + , 0x000F4E, 0x000F52, 0x000F53, 0x000F57, 0x000F58, 0x000F5C, 0x000F5D, 0x000F69 + , 0x000F6A, 0x000F6D, 0x000F71, 0x000F73, 0x000F74, 0x000F75, 0x000F7A, 0x000F81 + , 0x000F82, 0x000F85, 0x000F86, 0x000F93, 0x000F94, 0x000F98, 0x000F99, 0x000F9D + , 0x000F9E, 0x000FA2, 0x000FA3, 0x000FA7, 0x000FA8, 0x000FAC, 0x000FAD, 0x000FB9 + , 0x000FBA, 0x000FBD, 0x000FBE, 0x000FC6, 0x000FC7, 0x000FCD, 0x000FCE, 0x000FDB + , 0x001000, 0x00104A, 0x001050, 0x00109E, 0x0010C6, 0x0010C7, 0x0010C8, 0x0010CD + , 0x0010CE, 0x0010D0, 0x0010FB, 0x0010FD, 0x001100, 0x001200, 0x001249, 0x00124A + , 0x00124E, 0x001250, 0x001257, 0x001258, 0x001259, 0x00125A, 0x00125E, 0x001260 + , 0x001289, 0x00128A, 0x00128E, 0x001290, 0x0012B1, 0x0012B2, 0x0012B6, 0x0012B8 + , 0x0012BF, 0x0012C0, 0x0012C1, 0x0012C2, 0x0012C6, 0x0012C8, 0x0012D7, 0x0012D8 + , 0x001311, 0x001312, 0x001316, 0x001318, 0x00135B, 0x00135D, 0x001360, 0x00137D + , 0x001380, 0x001390, 0x00139A, 0x0013A0, 0x0013F6, 0x0013F8, 0x0013FE, 0x001400 + , 0x001401, 0x00166D, 0x00166F, 0x001680, 0x001681, 0x00169B, 0x00169D, 0x0016A0 + , 0x0016EB, 0x0016F1, 0x0016F9, 0x001700, 0x001716, 0x00171F, 0x001735, 0x001737 + , 0x001740, 0x001754, 0x001760, 0x00176D, 0x00176E, 0x001771, 0x001772, 0x001774 + , 0x001780, 0x0017B4, 0x0017B6, 0x0017D4, 0x0017D7, 0x0017D8, 0x0017DC, 0x0017DE + , 0x0017E0, 0x0017EA, 0x0017F0, 0x0017FA, 0x001800, 0x001810, 0x00181A, 0x001820 + , 0x001879, 0x001880, 0x0018AB, 0x0018B0, 0x0018F6, 0x001900, 0x00191F, 0x001920 + , 0x00192C, 0x001930, 0x00193C, 0x001940, 0x001941, 0x001944, 0x001946, 0x00196E + , 0x001970, 0x001975, 0x001980, 0x0019AC, 0x0019B0, 0x0019CA, 0x0019D0, 0x0019DA + , 0x0019DB, 0x0019DE, 0x001A00, 0x001A1C, 0x001A1E, 0x001A20, 0x001A5F, 0x001A60 + , 0x001A7D, 0x001A7F, 0x001A8A, 0x001A90, 0x001A9A, 0x001AA0, 0x001AA7, 0x001AA8 + , 0x001AAE, 0x001AB0, 0x001ABE, 0x001ABF, 0x001ADE, 0x001AE0, 0x001AEC, 0x001B00 + , 0x001B4D, 0x001B4E, 0x001B50, 0x001B5A, 0x001B6B, 0x001B74, 0x001B80, 0x001BF4 + , 0x001BFC, 0x001C00, 0x001C38, 0x001C3B, 0x001C40, 0x001C4A, 0x001C4D, 0x001C7E + , 0x001C8A, 0x001C8B, 0x001C90, 0x001CBB, 0x001CBD, 0x001CC8, 0x001CD0, 0x001CD3 + , 0x001CD4, 0x001CFB, 0x001D00, 0x001D2C, 0x001D2F, 0x001D30, 0x001D3B, 0x001D3C + , 0x001D4E, 0x001D4F, 0x001D6B, 0x001D78, 0x001D79, 0x001D9B, 0x001DC0, 0x001E00 + , 0x001E01, 0x001E02, 0x001E03, 0x001E04, 0x001E05, 0x001E06, 0x001E07, 0x001E08 + , 0x001E09, 0x001E0A, 0x001E0B, 0x001E0C, 0x001E0D, 0x001E0E, 0x001E0F, 0x001E10 + , 0x001E11, 0x001E12, 0x001E13, 0x001E14, 0x001E15, 0x001E16, 0x001E17, 0x001E18 + , 0x001E19, 0x001E1A, 0x001E1B, 0x001E1C, 0x001E1D, 0x001E1E, 0x001E1F, 0x001E20 + , 0x001E21, 0x001E22, 0x001E23, 0x001E24, 0x001E25, 0x001E26, 0x001E27, 0x001E28 + , 0x001E29, 0x001E2A, 0x001E2B, 0x001E2C, 0x001E2D, 0x001E2E, 0x001E2F, 0x001E30 + , 0x001E31, 0x001E32, 0x001E33, 0x001E34, 0x001E35, 0x001E36, 0x001E37, 0x001E38 + , 0x001E39, 0x001E3A, 0x001E3B, 0x001E3C, 0x001E3D, 0x001E3E, 0x001E3F, 0x001E40 + , 0x001E41, 0x001E42, 0x001E43, 0x001E44, 0x001E45, 0x001E46, 0x001E47, 0x001E48 + , 0x001E49, 0x001E4A, 0x001E4B, 0x001E4C, 0x001E4D, 0x001E4E, 0x001E4F, 0x001E50 + , 0x001E51, 0x001E52, 0x001E53, 0x001E54, 0x001E55, 0x001E56, 0x001E57, 0x001E58 + , 0x001E59, 0x001E5A, 0x001E5B, 0x001E5C, 0x001E5D, 0x001E5E, 0x001E5F, 0x001E60 + , 0x001E61, 0x001E62, 0x001E63, 0x001E64, 0x001E65, 0x001E66, 0x001E67, 0x001E68 + , 0x001E69, 0x001E6A, 0x001E6B, 0x001E6C, 0x001E6D, 0x001E6E, 0x001E6F, 0x001E70 + , 0x001E71, 0x001E72, 0x001E73, 0x001E74, 0x001E75, 0x001E76, 0x001E77, 0x001E78 + , 0x001E79, 0x001E7A, 0x001E7B, 0x001E7C, 0x001E7D, 0x001E7E, 0x001E7F, 0x001E80 + , 0x001E81, 0x001E82, 0x001E83, 0x001E84, 0x001E85, 0x001E86, 0x001E87, 0x001E88 + , 0x001E89, 0x001E8A, 0x001E8B, 0x001E8C, 0x001E8D, 0x001E8E, 0x001E8F, 0x001E90 + , 0x001E91, 0x001E92, 0x001E93, 0x001E94, 0x001E95, 0x001E9A, 0x001E9C, 0x001E9E + , 0x001E9F, 0x001EA0, 0x001EA1, 0x001EA2, 0x001EA3, 0x001EA4, 0x001EA5, 0x001EA6 + , 0x001EA7, 0x001EA8, 0x001EA9, 0x001EAA, 0x001EAB, 0x001EAC, 0x001EAD, 0x001EAE + , 0x001EAF, 0x001EB0, 0x001EB1, 0x001EB2, 0x001EB3, 0x001EB4, 0x001EB5, 0x001EB6 + , 0x001EB7, 0x001EB8, 0x001EB9, 0x001EBA, 0x001EBB, 0x001EBC, 0x001EBD, 0x001EBE + , 0x001EBF, 0x001EC0, 0x001EC1, 0x001EC2, 0x001EC3, 0x001EC4, 0x001EC5, 0x001EC6 + , 0x001EC7, 0x001EC8, 0x001EC9, 0x001ECA, 0x001ECB, 0x001ECC, 0x001ECD, 0x001ECE + , 0x001ECF, 0x001ED0, 0x001ED1, 0x001ED2, 0x001ED3, 0x001ED4, 0x001ED5, 0x001ED6 + , 0x001ED7, 0x001ED8, 0x001ED9, 0x001EDA, 0x001EDB, 0x001EDC, 0x001EDD, 0x001EDE + , 0x001EDF, 0x001EE0, 0x001EE1, 0x001EE2, 0x001EE3, 0x001EE4, 0x001EE5, 0x001EE6 + , 0x001EE7, 0x001EE8, 0x001EE9, 0x001EEA, 0x001EEB, 0x001EEC, 0x001EED, 0x001EEE + , 0x001EEF, 0x001EF0, 0x001EF1, 0x001EF2, 0x001EF3, 0x001EF4, 0x001EF5, 0x001EF6 + , 0x001EF7, 0x001EF8, 0x001EF9, 0x001EFA, 0x001EFB, 0x001EFC, 0x001EFD, 0x001EFE + , 0x001EFF, 0x001F08, 0x001F10, 0x001F16, 0x001F18, 0x001F1E, 0x001F20, 0x001F28 + , 0x001F30, 0x001F38, 0x001F40, 0x001F46, 0x001F48, 0x001F4E, 0x001F50, 0x001F58 + , 0x001F59, 0x001F5A, 0x001F5B, 0x001F5C, 0x001F5D, 0x001F5E, 0x001F5F, 0x001F60 + , 0x001F68, 0x001F70, 0x001F71, 0x001F72, 0x001F73, 0x001F74, 0x001F75, 0x001F76 + , 0x001F77, 0x001F78, 0x001F79, 0x001F7A, 0x001F7B, 0x001F7C, 0x001F7D, 0x001F7E + , 0x001F80, 0x001FB0, 0x001FB2, 0x001FB5, 0x001FB6, 0x001FB7, 0x001FC5, 0x001FC6 + , 0x001FC7, 0x001FD0, 0x001FD3, 0x001FD4, 0x001FD6, 0x001FD8, 0x001FDC, 0x001FDD + , 0x001FE0, 0x001FE3, 0x001FE4, 0x001FE8, 0x001FF0, 0x001FF2, 0x001FF5, 0x001FF6 + , 0x001FF7, 0x001FFF, 0x002000, 0x00200C, 0x00200E, 0x002065, 0x002066, 0x002072 + , 0x002074, 0x00208F, 0x002090, 0x00209D, 0x0020A0, 0x0020C2, 0x0020D0, 0x0020F1 + , 0x002100, 0x00214E, 0x00214F, 0x002184, 0x002185, 0x00218C, 0x002190, 0x00242A + , 0x002440, 0x00244B, 0x002460, 0x002B74, 0x002B76, 0x002C30, 0x002C60, 0x002C61 + , 0x002C62, 0x002C65, 0x002C67, 0x002C68, 0x002C69, 0x002C6A, 0x002C6B, 0x002C6C + , 0x002C6D, 0x002C71, 0x002C72, 0x002C73, 0x002C75, 0x002C76, 0x002C7C, 0x002C81 + , 0x002C82, 0x002C83, 0x002C84, 0x002C85, 0x002C86, 0x002C87, 0x002C88, 0x002C89 + , 0x002C8A, 0x002C8B, 0x002C8C, 0x002C8D, 0x002C8E, 0x002C8F, 0x002C90, 0x002C91 + , 0x002C92, 0x002C93, 0x002C94, 0x002C95, 0x002C96, 0x002C97, 0x002C98, 0x002C99 + , 0x002C9A, 0x002C9B, 0x002C9C, 0x002C9D, 0x002C9E, 0x002C9F, 0x002CA0, 0x002CA1 + , 0x002CA2, 0x002CA3, 0x002CA4, 0x002CA5, 0x002CA6, 0x002CA7, 0x002CA8, 0x002CA9 + , 0x002CAA, 0x002CAB, 0x002CAC, 0x002CAD, 0x002CAE, 0x002CAF, 0x002CB0, 0x002CB1 + , 0x002CB2, 0x002CB3, 0x002CB4, 0x002CB5, 0x002CB6, 0x002CB7, 0x002CB8, 0x002CB9 + , 0x002CBA, 0x002CBB, 0x002CBC, 0x002CBD, 0x002CBE, 0x002CBF, 0x002CC0, 0x002CC1 + , 0x002CC2, 0x002CC3, 0x002CC4, 0x002CC5, 0x002CC6, 0x002CC7, 0x002CC8, 0x002CC9 + , 0x002CCA, 0x002CCB, 0x002CCC, 0x002CCD, 0x002CCE, 0x002CCF, 0x002CD0, 0x002CD1 + , 0x002CD2, 0x002CD3, 0x002CD4, 0x002CD5, 0x002CD6, 0x002CD7, 0x002CD8, 0x002CD9 + , 0x002CDA, 0x002CDB, 0x002CDC, 0x002CDD, 0x002CDE, 0x002CDF, 0x002CE0, 0x002CE1 + , 0x002CE2, 0x002CE3, 0x002CE5, 0x002CEC, 0x002CED, 0x002CEE, 0x002CF2, 0x002CF3 + , 0x002CF4, 0x002CF9, 0x002D00, 0x002D26, 0x002D27, 0x002D28, 0x002D2D, 0x002D2E + , 0x002D30, 0x002D68, 0x002D6F, 0x002D71, 0x002D7F, 0x002D97, 0x002DA0, 0x002DA7 + , 0x002DA8, 0x002DAF, 0x002DB0, 0x002DB7, 0x002DB8, 0x002DBF, 0x002DC0, 0x002DC7 + , 0x002DC8, 0x002DCF, 0x002DD0, 0x002DD7, 0x002DD8, 0x002DDF, 0x002DE0, 0x002E00 + , 0x002E2F, 0x002E30, 0x002E5E, 0x002E80, 0x002E9A, 0x002E9B, 0x002EF4, 0x002F00 + , 0x002FD6, 0x002FF0, 0x003005, 0x003008, 0x00302A, 0x00302E, 0x00303C, 0x00303D + , 0x003040, 0x003041, 0x003097, 0x003099, 0x00309B, 0x00309D, 0x00309F, 0x0030A1 + , 0x0030FB, 0x0030FC, 0x0030FF, 0x003100, 0x003105, 0x003130, 0x003131, 0x00318F + , 0x003190, 0x0031A0, 0x0031C0, 0x0031E6, 0x0031EF, 0x0031F0, 0x003200, 0x00321F + , 0x003220, 0x003400, 0x004DC0, 0x004E00, 0x00A48D, 0x00A490, 0x00A4C7, 0x00A4D0 + , 0x00A4FE, 0x00A500, 0x00A60D, 0x00A610, 0x00A62C, 0x00A640, 0x00A641, 0x00A642 + , 0x00A643, 0x00A644, 0x00A645, 0x00A646, 0x00A647, 0x00A648, 0x00A649, 0x00A64A + , 0x00A64B, 0x00A64C, 0x00A64D, 0x00A64E, 0x00A64F, 0x00A650, 0x00A651, 0x00A652 + , 0x00A653, 0x00A654, 0x00A655, 0x00A656, 0x00A657, 0x00A658, 0x00A659, 0x00A65A + , 0x00A65B, 0x00A65C, 0x00A65D, 0x00A65E, 0x00A65F, 0x00A660, 0x00A661, 0x00A662 + , 0x00A663, 0x00A664, 0x00A665, 0x00A666, 0x00A667, 0x00A668, 0x00A669, 0x00A66A + , 0x00A66B, 0x00A66C, 0x00A66D, 0x00A670, 0x00A674, 0x00A67E, 0x00A67F, 0x00A680 + , 0x00A681, 0x00A682, 0x00A683, 0x00A684, 0x00A685, 0x00A686, 0x00A687, 0x00A688 + , 0x00A689, 0x00A68A, 0x00A68B, 0x00A68C, 0x00A68D, 0x00A68E, 0x00A68F, 0x00A690 + , 0x00A691, 0x00A692, 0x00A693, 0x00A694, 0x00A695, 0x00A696, 0x00A697, 0x00A698 + , 0x00A699, 0x00A69A, 0x00A69B, 0x00A69C, 0x00A69E, 0x00A6E6, 0x00A6F0, 0x00A6F2 + , 0x00A6F8, 0x00A700, 0x00A717, 0x00A720, 0x00A723, 0x00A724, 0x00A725, 0x00A726 + , 0x00A727, 0x00A728, 0x00A729, 0x00A72A, 0x00A72B, 0x00A72C, 0x00A72D, 0x00A72E + , 0x00A72F, 0x00A732, 0x00A733, 0x00A734, 0x00A735, 0x00A736, 0x00A737, 0x00A738 + , 0x00A739, 0x00A73A, 0x00A73B, 0x00A73C, 0x00A73D, 0x00A73E, 0x00A73F, 0x00A740 + , 0x00A741, 0x00A742, 0x00A743, 0x00A744, 0x00A745, 0x00A746, 0x00A747, 0x00A748 + , 0x00A749, 0x00A74A, 0x00A74B, 0x00A74C, 0x00A74D, 0x00A74E, 0x00A74F, 0x00A750 + , 0x00A751, 0x00A752, 0x00A753, 0x00A754, 0x00A755, 0x00A756, 0x00A757, 0x00A758 + , 0x00A759, 0x00A75A, 0x00A75B, 0x00A75C, 0x00A75D, 0x00A75E, 0x00A75F, 0x00A760 + , 0x00A761, 0x00A762, 0x00A763, 0x00A764, 0x00A765, 0x00A766, 0x00A767, 0x00A768 + , 0x00A769, 0x00A76A, 0x00A76B, 0x00A76C, 0x00A76D, 0x00A76E, 0x00A76F, 0x00A770 + , 0x00A771, 0x00A779, 0x00A77A, 0x00A77B, 0x00A77C, 0x00A77D, 0x00A77F, 0x00A780 + , 0x00A781, 0x00A782, 0x00A783, 0x00A784, 0x00A785, 0x00A786, 0x00A787, 0x00A789 + , 0x00A78C, 0x00A78D, 0x00A78E, 0x00A790, 0x00A791, 0x00A792, 0x00A793, 0x00A796 + , 0x00A797, 0x00A798, 0x00A799, 0x00A79A, 0x00A79B, 0x00A79C, 0x00A79D, 0x00A79E + , 0x00A79F, 0x00A7A0, 0x00A7A1, 0x00A7A2, 0x00A7A3, 0x00A7A4, 0x00A7A5, 0x00A7A6 + , 0x00A7A7, 0x00A7A8, 0x00A7A9, 0x00A7AA, 0x00A7AF, 0x00A7B0, 0x00A7B5, 0x00A7B6 + , 0x00A7B7, 0x00A7B8, 0x00A7B9, 0x00A7BA, 0x00A7BB, 0x00A7BC, 0x00A7BD, 0x00A7BE + , 0x00A7BF, 0x00A7C0, 0x00A7C1, 0x00A7C2, 0x00A7C3, 0x00A7C4, 0x00A7C8, 0x00A7C9 + , 0x00A7CA, 0x00A7CB, 0x00A7CD, 0x00A7CE, 0x00A7CF, 0x00A7D0, 0x00A7D1, 0x00A7D2 + , 0x00A7D3, 0x00A7D4, 0x00A7D5, 0x00A7D6, 0x00A7D7, 0x00A7D8, 0x00A7D9, 0x00A7DA + , 0x00A7DB, 0x00A7DC, 0x00A7DD, 0x00A7F1, 0x00A7F6, 0x00A7F8, 0x00A7FA, 0x00A828 + , 0x00A82C, 0x00A82D, 0x00A830, 0x00A83A, 0x00A840, 0x00A874, 0x00A878, 0x00A880 + , 0x00A8C6, 0x00A8CE, 0x00A8D0, 0x00A8DA, 0x00A8E0, 0x00A8F8, 0x00A8FB, 0x00A8FC + , 0x00A8FD, 0x00A92E, 0x00A930, 0x00A954, 0x00A95F, 0x00A97D, 0x00A980, 0x00A9C1 + , 0x00A9CE, 0x00A9CF, 0x00A9DA, 0x00A9DE, 0x00A9E0, 0x00A9FF, 0x00AA00, 0x00AA37 + , 0x00AA40, 0x00AA4E, 0x00AA50, 0x00AA5A, 0x00AA5C, 0x00AA60, 0x00AA77, 0x00AA7A + , 0x00AAC3, 0x00AADB, 0x00AADE, 0x00AAE0, 0x00AAF0, 0x00AAF2, 0x00AAF7, 0x00AB01 + , 0x00AB07, 0x00AB09, 0x00AB0F, 0x00AB11, 0x00AB17, 0x00AB20, 0x00AB27, 0x00AB28 + , 0x00AB2F, 0x00AB30, 0x00AB5B, 0x00AB60, 0x00AB69, 0x00AB6C, 0x00AB70, 0x00ABC0 + , 0x00ABEB, 0x00ABEC, 0x00ABEE, 0x00ABF0, 0x00ABFA, 0x00AC00, 0x00D7A4, 0x00D7B0 + , 0x00D7C7, 0x00D7CB, 0x00D7FC, 0x00D800, 0x00FA0E, 0x00FA10, 0x00FA11, 0x00FA12 + , 0x00FA13, 0x00FA15, 0x00FA1F, 0x00FA20, 0x00FA21, 0x00FA22, 0x00FA23, 0x00FA25 + , 0x00FA27, 0x00FA2A, 0x00FA6E, 0x00FA70, 0x00FADA, 0x00FB00, 0x00FB07, 0x00FB13 + , 0x00FB18, 0x00FB1D, 0x00FB1E, 0x00FB1F, 0x00FB37, 0x00FB38, 0x00FB3D, 0x00FB3E + , 0x00FB3F, 0x00FB40, 0x00FB42, 0x00FB43, 0x00FB45, 0x00FB46, 0x00FE1A, 0x00FE20 + , 0x00FE30, 0x00FE53, 0x00FE54, 0x00FE67, 0x00FE68, 0x00FE6C, 0x00FE70, 0x00FE73 + , 0x00FE74, 0x00FE75, 0x00FE76, 0x00FEFD, 0x00FEFF, 0x00FF00, 0x00FF01, 0x00FFBF + , 0x00FFC2, 0x00FFC8, 0x00FFCA, 0x00FFD0, 0x00FFD2, 0x00FFD8, 0x00FFDA, 0x00FFDD + , 0x00FFE0, 0x00FFE7, 0x00FFE8, 0x00FFEF, 0x00FFF9, 0x010000, 0x01000C, 0x01000D + , 0x010027, 0x010028, 0x01003B, 0x01003C, 0x01003E, 0x01003F, 0x01004E, 0x010050 + , 0x01005E, 0x010080, 0x0100FB, 0x010100, 0x010103, 0x010107, 0x010134, 0x010137 + , 0x01018F, 0x010190, 0x01019D, 0x0101A0, 0x0101A1, 0x0101D0, 0x0101FD, 0x0101FE + , 0x010280, 0x01029D, 0x0102A0, 0x0102D1, 0x0102E0, 0x0102E1, 0x0102FC, 0x010300 + , 0x010320, 0x010324, 0x01032D, 0x010341, 0x010342, 0x01034A, 0x01034B, 0x010350 + , 0x01037B, 0x010380, 0x01039E, 0x01039F, 0x0103A0, 0x0103C4, 0x0103C8, 0x0103D0 + , 0x0103D6, 0x010400, 0x010428, 0x01049E, 0x0104A0, 0x0104AA, 0x0104B0, 0x0104D4 + , 0x0104D8, 0x0104FC, 0x010500, 0x010528, 0x010530, 0x010564, 0x01056F, 0x01057B + , 0x01057C, 0x01058B, 0x01058C, 0x010593, 0x010594, 0x010596, 0x010597, 0x0105A2 + , 0x0105A3, 0x0105B2, 0x0105B3, 0x0105BA, 0x0105BB, 0x0105BD, 0x0105C0, 0x0105F4 + , 0x010600, 0x010737, 0x010740, 0x010756, 0x010760, 0x010768, 0x010780, 0x010781 + , 0x010786, 0x010787, 0x0107B1, 0x0107B2, 0x0107BB, 0x010800, 0x010806, 0x010808 + , 0x010809, 0x01080A, 0x010836, 0x010837, 0x010839, 0x01083C, 0x01083D, 0x01083F + , 0x010856, 0x010857, 0x010860, 0x010877, 0x010880, 0x01089F, 0x0108A7, 0x0108B0 + , 0x0108E0, 0x0108F3, 0x0108F4, 0x0108F6, 0x0108FB, 0x010900, 0x010916, 0x01091C + , 0x01091F, 0x010920, 0x01093A, 0x01093F, 0x010940, 0x01095A, 0x010980, 0x0109B8 + , 0x0109BC, 0x0109BE, 0x0109C0, 0x0109D0, 0x0109D2, 0x010A00, 0x010A04, 0x010A05 + , 0x010A07, 0x010A0C, 0x010A14, 0x010A15, 0x010A18, 0x010A19, 0x010A36, 0x010A38 + , 0x010A3B, 0x010A3F, 0x010A40, 0x010A49, 0x010A50, 0x010A59, 0x010A60, 0x010A7D + , 0x010A80, 0x010A9D, 0x010AA0, 0x010AC0, 0x010AC8, 0x010AC9, 0x010AE7, 0x010AEB + , 0x010AF7, 0x010B00, 0x010B36, 0x010B39, 0x010B40, 0x010B56, 0x010B58, 0x010B60 + , 0x010B73, 0x010B78, 0x010B80, 0x010B92, 0x010B99, 0x010B9D, 0x010BA9, 0x010BB0 + , 0x010C00, 0x010C49, 0x010C80, 0x010CB3, 0x010CC0, 0x010CF3, 0x010CFA, 0x010D00 + , 0x010D28, 0x010D30, 0x010D3A, 0x010D40, 0x010D50, 0x010D66, 0x010D69, 0x010D6E + , 0x010D6F, 0x010D86, 0x010D8E, 0x010D90, 0x010E60, 0x010E7F, 0x010E80, 0x010EAA + , 0x010EAB, 0x010EAD, 0x010EAE, 0x010EB0, 0x010EB2, 0x010EC2, 0x010EC8, 0x010ED0 + , 0x010ED9, 0x010EFA, 0x010F1D, 0x010F27, 0x010F28, 0x010F30, 0x010F51, 0x010F5A + , 0x010F70, 0x010F86, 0x010F8A, 0x010FB0, 0x010FC5, 0x010FCC, 0x010FE0, 0x010FF7 + , 0x011000, 0x011047, 0x01104E, 0x011052, 0x011066, 0x011076, 0x01107F, 0x0110BB + , 0x0110C2, 0x0110C3, 0x0110CD, 0x0110CE, 0x0110D0, 0x0110E9, 0x0110F0, 0x0110FA + , 0x011100, 0x011135, 0x011136, 0x011140, 0x011144, 0x011148, 0x011150, 0x011174 + , 0x011176, 0x011177, 0x011180, 0x0111C5, 0x0111C9, 0x0111CD, 0x0111CE, 0x0111DB + , 0x0111DC, 0x0111DD, 0x0111E0, 0x0111E1, 0x0111F5, 0x011200, 0x011212, 0x011213 + , 0x011238, 0x01123E, 0x011242, 0x011280, 0x011287, 0x011288, 0x011289, 0x01128A + , 0x01128E, 0x01128F, 0x01129E, 0x01129F, 0x0112A9, 0x0112AA, 0x0112B0, 0x0112EB + , 0x0112F0, 0x0112FA, 0x011300, 0x011304, 0x011305, 0x01130D, 0x01130F, 0x011311 + , 0x011313, 0x011329, 0x01132A, 0x011331, 0x011332, 0x011334, 0x011335, 0x01133A + , 0x01133B, 0x011345, 0x011347, 0x011349, 0x01134B, 0x01134E, 0x011350, 0x011351 + , 0x011357, 0x011358, 0x01135D, 0x011364, 0x011366, 0x01136D, 0x011370, 0x011375 + , 0x011380, 0x01138A, 0x01138B, 0x01138C, 0x01138E, 0x01138F, 0x011390, 0x0113B6 + , 0x0113B7, 0x0113C1, 0x0113C2, 0x0113C3, 0x0113C5, 0x0113C6, 0x0113C7, 0x0113CB + , 0x0113CC, 0x0113D4, 0x0113D6, 0x0113D7, 0x0113D9, 0x0113E1, 0x0113E3, 0x011400 + , 0x01144B, 0x011450, 0x01145A, 0x01145C, 0x01145D, 0x01145E, 0x011462, 0x011480 + , 0x0114C6, 0x0114C7, 0x0114C8, 0x0114D0, 0x0114DA, 0x011580, 0x0115B6, 0x0115B8 + , 0x0115C1, 0x0115D8, 0x0115DE, 0x011600, 0x011641, 0x011644, 0x011645, 0x011650 + , 0x01165A, 0x011660, 0x01166D, 0x011680, 0x0116B9, 0x0116BA, 0x0116C0, 0x0116CA + , 0x0116D0, 0x0116E4, 0x011700, 0x01171B, 0x01171D, 0x01172C, 0x011730, 0x01173A + , 0x011740, 0x011747, 0x011800, 0x01183B, 0x01183C, 0x0118A0, 0x0118C0, 0x0118EA + , 0x0118F3, 0x0118FF, 0x011907, 0x011909, 0x01190A, 0x01190C, 0x011914, 0x011915 + , 0x011917, 0x011918, 0x011936, 0x011937, 0x011939, 0x01193B, 0x011944, 0x011947 + , 0x011950, 0x01195A, 0x0119A0, 0x0119A8, 0x0119AA, 0x0119D8, 0x0119DA, 0x0119E2 + , 0x0119E3, 0x0119E5, 0x011A00, 0x011A3F, 0x011A47, 0x011A48, 0x011A50, 0x011A9A + , 0x011A9D, 0x011A9E, 0x011AA3, 0x011AB0, 0x011AF9, 0x011B00, 0x011B0A, 0x011B60 + , 0x011B68, 0x011BC0, 0x011BE1, 0x011BE2, 0x011BF0, 0x011BFA, 0x011C00, 0x011C09 + , 0x011C0A, 0x011C37, 0x011C38, 0x011C41, 0x011C46, 0x011C50, 0x011C5A, 0x011C6D + , 0x011C70, 0x011C72, 0x011C90, 0x011C92, 0x011CA8, 0x011CA9, 0x011CB7, 0x011D00 + , 0x011D07, 0x011D08, 0x011D0A, 0x011D0B, 0x011D37, 0x011D3A, 0x011D3B, 0x011D3C + , 0x011D3E, 0x011D3F, 0x011D48, 0x011D50, 0x011D5A, 0x011D60, 0x011D66, 0x011D67 + , 0x011D69, 0x011D6A, 0x011D8F, 0x011D90, 0x011D92, 0x011D93, 0x011D99, 0x011DA0 + , 0x011DAA, 0x011DB0, 0x011DDC, 0x011DE0, 0x011DEA, 0x011EE0, 0x011EF7, 0x011EF9 + , 0x011F00, 0x011F11, 0x011F12, 0x011F3B, 0x011F3E, 0x011F43, 0x011F50, 0x011F5B + , 0x011FB0, 0x011FB1, 0x011FC0, 0x011FF2, 0x011FFF, 0x012000, 0x01239A, 0x012400 + , 0x01246F, 0x012470, 0x012475, 0x012480, 0x012544, 0x012F90, 0x012FF1, 0x012FF3 + , 0x013000, 0x013430, 0x013440, 0x013456, 0x013460, 0x0143FB, 0x014400, 0x014647 + , 0x016100, 0x01613A, 0x016800, 0x016A39, 0x016A40, 0x016A5F, 0x016A60, 0x016A6A + , 0x016A6E, 0x016A70, 0x016ABF, 0x016AC0, 0x016ACA, 0x016AD0, 0x016AEE, 0x016AF0 + , 0x016AF5, 0x016AF6, 0x016B00, 0x016B37, 0x016B40, 0x016B44, 0x016B46, 0x016B50 + , 0x016B5A, 0x016B5B, 0x016B62, 0x016B63, 0x016B78, 0x016B7D, 0x016B90, 0x016D40 + , 0x016D6D, 0x016D70, 0x016D7A, 0x016E40, 0x016E60, 0x016E80, 0x016E9B, 0x016EA0 + , 0x016EB9, 0x016EBB, 0x016ED4, 0x016F00, 0x016F4B, 0x016F4F, 0x016F88, 0x016F8F + , 0x016FA0, 0x016FE0, 0x016FE2, 0x016FE3, 0x016FE5, 0x016FF0, 0x016FF4, 0x016FF7 + , 0x017000, 0x018CD6, 0x018CFF, 0x018D1F, 0x018D80, 0x018DF3, 0x01AFF0, 0x01AFF4 + , 0x01AFF5, 0x01AFFC, 0x01AFFD, 0x01AFFF, 0x01B000, 0x01B123, 0x01B132, 0x01B133 + , 0x01B150, 0x01B153, 0x01B155, 0x01B156, 0x01B164, 0x01B168, 0x01B170, 0x01B2FC + , 0x01BC00, 0x01BC6B, 0x01BC70, 0x01BC7D, 0x01BC80, 0x01BC89, 0x01BC90, 0x01BC9A + , 0x01BC9C, 0x01BC9D, 0x01BC9F, 0x01BCA4, 0x01CC00, 0x01CCFD, 0x01CD00, 0x01CEB4 + , 0x01CEBA, 0x01CED1, 0x01CEE0, 0x01CEF1, 0x01CF00, 0x01CF2E, 0x01CF30, 0x01CF47 + , 0x01CF50, 0x01CFC4, 0x01D000, 0x01D0F6, 0x01D100, 0x01D127, 0x01D129, 0x01D1EB + , 0x01D200, 0x01D246, 0x01D2C0, 0x01D2D4, 0x01D2E0, 0x01D2F4, 0x01D300, 0x01D357 + , 0x01D360, 0x01D379, 0x01D400, 0x01D455, 0x01D456, 0x01D49D, 0x01D49E, 0x01D4A0 + , 0x01D4A2, 0x01D4A3, 0x01D4A5, 0x01D4A7, 0x01D4A9, 0x01D4AD, 0x01D4AE, 0x01D4BA + , 0x01D4BB, 0x01D4BC, 0x01D4BD, 0x01D4C4, 0x01D4C5, 0x01D506, 0x01D507, 0x01D50B + , 0x01D50D, 0x01D515, 0x01D516, 0x01D51D, 0x01D51E, 0x01D53A, 0x01D53B, 0x01D53F + , 0x01D540, 0x01D545, 0x01D546, 0x01D547, 0x01D54A, 0x01D551, 0x01D552, 0x01D6A6 + , 0x01D6A8, 0x01D7CC, 0x01D7CE, 0x01DA00, 0x01DA37, 0x01DA3B, 0x01DA6D, 0x01DA75 + , 0x01DA76, 0x01DA84, 0x01DA85, 0x01DA8C, 0x01DA9B, 0x01DAA0, 0x01DAA1, 0x01DAB0 + , 0x01DF00, 0x01DF1F, 0x01DF25, 0x01DF2B, 0x01E000, 0x01E007, 0x01E008, 0x01E019 + , 0x01E01B, 0x01E022, 0x01E023, 0x01E025, 0x01E026, 0x01E02B, 0x01E030, 0x01E06E + , 0x01E08F, 0x01E090, 0x01E100, 0x01E12D, 0x01E130, 0x01E13E, 0x01E140, 0x01E14A + , 0x01E14E, 0x01E14F, 0x01E150, 0x01E290, 0x01E2AF, 0x01E2C0, 0x01E2FA, 0x01E2FF + , 0x01E300, 0x01E4D0, 0x01E4FA, 0x01E5D0, 0x01E5FB, 0x01E5FF, 0x01E600, 0x01E6C0 + , 0x01E6DF, 0x01E6E0, 0x01E6F6, 0x01E6FE, 0x01E700, 0x01E7E0, 0x01E7E7, 0x01E7E8 + , 0x01E7EC, 0x01E7ED, 0x01E7EF, 0x01E7F0, 0x01E7FF, 0x01E800, 0x01E8C5, 0x01E8C7 + , 0x01E8D0, 0x01E8D7, 0x01E900, 0x01E922, 0x01E94C, 0x01E950, 0x01E95A, 0x01E95E + , 0x01E960, 0x01EC71, 0x01ECB5, 0x01ED01, 0x01ED3E, 0x01EE00, 0x01EE04, 0x01EE05 + , 0x01EE20, 0x01EE21, 0x01EE23, 0x01EE24, 0x01EE25, 0x01EE27, 0x01EE28, 0x01EE29 + , 0x01EE33, 0x01EE34, 0x01EE38, 0x01EE39, 0x01EE3A, 0x01EE3B, 0x01EE3C, 0x01EE42 + , 0x01EE43, 0x01EE47, 0x01EE48, 0x01EE49, 0x01EE4A, 0x01EE4B, 0x01EE4C, 0x01EE4D + , 0x01EE50, 0x01EE51, 0x01EE53, 0x01EE54, 0x01EE55, 0x01EE57, 0x01EE58, 0x01EE59 + , 0x01EE5A, 0x01EE5B, 0x01EE5C, 0x01EE5D, 0x01EE5E, 0x01EE5F, 0x01EE60, 0x01EE61 + , 0x01EE63, 0x01EE64, 0x01EE65, 0x01EE67, 0x01EE6B, 0x01EE6C, 0x01EE73, 0x01EE74 + , 0x01EE78, 0x01EE79, 0x01EE7D, 0x01EE7E, 0x01EE7F, 0x01EE80, 0x01EE8A, 0x01EE8B + , 0x01EE9C, 0x01EEA1, 0x01EEA4, 0x01EEA5, 0x01EEAA, 0x01EEAB, 0x01EEBC, 0x01EEF0 + , 0x01EEF2, 0x01F000, 0x01F02C, 0x01F030, 0x01F094, 0x01F0A0, 0x01F0AF, 0x01F0B1 + , 0x01F0C0, 0x01F0C1, 0x01F0D0, 0x01F0D1, 0x01F0F6, 0x01F100, 0x01F1AE, 0x01F1E6 + , 0x01F203, 0x01F210, 0x01F23C, 0x01F240, 0x01F249, 0x01F250, 0x01F252, 0x01F260 + , 0x01F266, 0x01F300, 0x01F6D9, 0x01F6DC, 0x01F6ED, 0x01F6F0, 0x01F6FD, 0x01F700 + , 0x01F7DA, 0x01F7E0, 0x01F7EC, 0x01F7F0, 0x01F7F1, 0x01F800, 0x01F80C, 0x01F810 + , 0x01F848, 0x01F850, 0x01F85A, 0x01F860, 0x01F888, 0x01F890, 0x01F8AE, 0x01F8B0 + , 0x01F8BC, 0x01F8C0, 0x01F8C2, 0x01F8D0, 0x01F8D9, 0x01F900, 0x01FA58, 0x01FA60 + , 0x01FA6E, 0x01FA70, 0x01FA7D, 0x01FA80, 0x01FA8B, 0x01FA8E, 0x01FAC7, 0x01FAC8 + , 0x01FAC9, 0x01FACD, 0x01FADD, 0x01FADF, 0x01FAEB, 0x01FAEF, 0x01FAF9, 0x01FB00 + , 0x01FB93, 0x01FB94, 0x01FBFB, 0x01FFFE, 0x020000, 0x02A6E0, 0x02A700, 0x02B81E + , 0x02B820, 0x02CEAE, 0x02CEB0, 0x02EBE1, 0x02EBF0, 0x02EE5E, 0x02F800, 0x02FA1E + , 0x02FFFE, 0x030000, 0x03134B, 0x031350, 0x03347A, 0x03FFFE, 0x040000, 0x04FFFE + , 0x050000, 0x05FFFE, 0x060000, 0x06FFFE, 0x070000, 0x07FFFE, 0x080000, 0x08FFFE + , 0x090000, 0x09FFFE, 0x0A0000, 0x0AFFFE, 0x0B0000, 0x0BFFFE, 0x0C0000, 0x0CFFFE + , 0x0D0000, 0x0DFFFE, 0x0E0000, 0x0E0001, 0x0E0002, 0x0E0020, 0x0E0080, 0x0E0100 + , 0x0E01F0, 0x0EFFFE + ] + +rangeTags :: ByteArray +rangeTags = PBA.byteArrayFromList @Word8 + [ 3, 0, 3, 0, 3, 0, 3, 2, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 2, 3, 0, 4, 3, 0, 3, 4, 3, 4, 3, 4, 3, 0, 3 + , 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4, 3, 4, 0, 3, 0, 3, 0, 3, 4, 3, 4, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 4, 0, 4, 0, 2, 4, 3, 0, 3, 0, 3, 0, 2, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 2, 0, 3, 4, 3, 0, 4 + , 0, 4, 0, 3, 4, 0, 3, 0, 4, 3, 4, 0, 4, 3, 4, 0, 4, 0, 3, 0, 3, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 3, 0, 4, 0, 3, 0, 3, 0, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 4, 3, 4 + , 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 0, 4, 0, 3, 0, 3, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 3, 0, 4, 0, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4 + , 0, 3, 0, 4, 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3 + , 4, 0, 3, 0, 4, 3, 0, 3, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0 + , 4, 3, 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4, 3, 0, 3, 4, 3, 4 + , 0, 3, 0, 3, 4, 3, 4, 3, 4, 0, 3, 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 3, 4, 0, 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 4, 0 + , 3, 0, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 0, 4, 0, 4, 3, 4, 3, 0, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 3, 0, 4, 3, 0, 4, 0 + , 4, 0, 4, 0, 4, 3, 0, 3, 4, 0, 3, 0, 4, 0, 4, 0, 4, 3, 0, 3, 0, 3, 0, 4, 3, 0, 4, 3, 0, 4, 0, 3 + , 0, 4, 3, 4, 3, 4, 0, 3, 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 4, 3, 4, 0, 3, 0, 3, 0, 4, 3, 4, 0, 4 + , 3, 4, 3, 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 4, 3, 0, 3, 4, 0, 3, 4, 0 + , 3, 0, 3, 4, 0, 3, 4, 3, 0, 3, 0, 3, 4, 3, 4, 0, 3, 4, 3, 1, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4 + , 3, 0, 3, 0, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0 + , 3, 0, 3, 0, 3, 0, 3, 0, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4 + , 0, 4, 0, 4, 0, 4, 0, 3, 0, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 4, 0, 4, 0, 3, 0, 3, 0 + , 2, 0, 3, 4, 0, 4, 3, 4, 3, 0, 3, 4, 3, 0, 3, 4, 3, 0, 3, 0, 4, 3, 4, 0, 3, 0, 3, 0, 4, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3 + , 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 4, 3, 0, 3, 0, 3, 0, 4, 3, 4, 0, 3, 4, 0 + , 4, 3, 0, 4, 0, 3, 0, 3, 0, 3, 0, 4, 3, 4, 0, 3, 4, 0, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 3, 0 + , 4, 0, 3, 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 0, 3, 4, 3, 0, 3, 0, 4, 0, 4, 0, 4, 3 + , 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 0, 3, 4, 3, 4, 3, 4, 3, 0, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4 + , 3, 4, 3, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 4 + , 0, 4, 0, 4, 0, 3, 4, 0, 3, 4, 0, 3, 0, 3, 4, 0, 4, 0, 4, 3, 0, 4, 0, 3, 4, 3, 0, 4, 0, 4, 3, 4 + , 0, 4, 0, 4, 0, 4, 3, 4, 3, 4, 3, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3 + , 4, 3, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 3, 0, 4, 3, 4, 0, 4, 0, 4, 3, 0, 3, 4 + , 3, 0, 4, 3, 0, 4, 0, 4, 3, 0, 3, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 3, 4, 0, 3 + , 0, 3, 4, 0, 3, 0, 4, 3, 4, 0, 4, 3, 0, 4, 3, 0, 4, 3, 0, 4, 3, 4, 3, 4, 0, 4, 3, 4, 0, 4, 3, 0 + , 4, 0, 4, 0, 3, 4, 0, 3, 0, 4, 3, 4, 3, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 3, 4, 0, 3, 0, 4, 0, 3, 4 + , 0, 3, 4, 0, 3, 4, 0, 4, 0, 3, 4, 3, 0, 4, 0, 3, 0, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 0, 3 + , 0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 4, 3, 4, 0, 4, 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 3, 4, 0, 4, 0, 3, 0, 3, 4, 3, 0, 4, 0 + , 3, 0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 0, 3, 0, 4, 0, 4, 3, 4, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3 + , 0, 4, 0, 3, 4, 3, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 3 + , 0, 4, 0, 3, 0, 4, 0, 3, 0, 3, 4, 0, 4, 3, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 3, 4 + , 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0 + , 4, 0, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 0, 3, 0, 4, 0, 4, 3, 4, 3, 0, 4, 3, 4, 3, 4, 0, 4, 0, 3, 4 + , 0, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 3, 0, 3, 4, 0 + , 4, 3, 4, 0, 4, 0, 4, 0, 3, 0, 4, 3, 0, 3, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 0, 3, 4 + , 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4 + , 3, 0, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 0, 4, 0, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4 + , 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4 + , 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 3, 0, 3, 0, 3, 0, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4 + , 0, 4, 0, 4, 0, 4, 3, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 3, 4, 0, 4, 0, 4, 3, 4, 0, 4, 0, 4, 3, 4, 0 + , 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 0, 4, 3, 0, 4, 0, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 0, 4, 0, 4, 0, 4, 0, 4, 0, 4, 3, 4, 3, 0, 4, 0, 4, 3, 4, 3 + , 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3 + ] diff --git a/internal/Text/IDNA2008/Internal/Ranges.hs b/internal/Text/IDNA2008/Internal/Ranges.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Ranges.hs @@ -0,0 +1,62 @@ +-- | +-- Module : Text.IDNA2008.Internal.Ranges +-- Description : Shared binary-search helper for sorted codepoint +-- range tables packed as @[start, end, start, end, ...]@ +-- 'Word32' entries in a 'ByteArray'. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- One canonical implementation of @inRanges@ -- a binary search +-- over a sorted, coalesced array of @[start, end]@ pairs packed +-- into a 'ByteArray' as alternating 'Word32' entries. Consumed +-- by the property-table wrappers ("Text.IDNA2008.Internal.Emoji", +-- "Text.IDNA2008.Internal.Joining", +-- "Text.IDNA2008.Internal.NFC", "Text.IDNA2008.Internal.Script") +-- when their range data is shaped that way. +-- +-- The function is intentionally marked @INLINE@; callers +-- partially-apply it to a specific @arr@ and @cnt@ at the top +-- level, and GHC specialises the worker per call site. +{-# LANGUAGE BangPatterns #-} + +module Text.IDNA2008.Internal.Ranges + ( inRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word32) + +-- | Test whether @cp@ falls within any of the @cnt@ +-- @[start_i, end_i]@ ranges packed into @arr@ as alternating +-- 'Word32' entries, sorted ascending by @start_i@. +-- +-- Returns 'False' when @cnt == 0@, and short-circuits @'False'@ +-- when @cp@ precedes the first range's start. +inRanges :: ByteArray -> Int -> Int -> Bool +inRanges !arr !cnt !cp + | cnt == 0 || cp < startAt 0 = False + | otherwise = + let !idx = bsearch 0 (cnt - 1) + in cp <= endAt idx + where + startAt :: Int -> Int + startAt !i = fromIntegral (indexByteArray arr (2 * i) :: Word32) + {-# INLINE startAt #-} + + endAt :: Int -> Int + endAt !i = fromIntegral (indexByteArray arr (2 * i + 1) :: Word32) + {-# INLINE endAt #-} + + bsearch :: Int -> Int -> Int + bsearch !lo !hi + | lo == hi = lo + | otherwise = + let !mid = (lo + hi + 1) `quot` 2 + in if startAt mid <= cp + then bsearch mid hi + else bsearch lo (mid - 1) +{-# INLINE inRanges #-} diff --git a/internal/Text/IDNA2008/Internal/Script.hs b/internal/Text/IDNA2008/Internal/Script.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Script.hs @@ -0,0 +1,55 @@ +-- | +-- Module : Text.IDNA2008.Internal.Script +-- Description : Predicates for the Unicode Scripts consulted by the +-- CONTEXTO contextual rules (RFC 5892 A.4-A.7). +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Membership predicates over the Unicode @sc@ (Script) property +-- for the four scripts named by RFC 5892 Appendix A.4-A.7: +-- Greek, Hebrew, and the Hiragana \/ Katakana \/ Han trio (the +-- last collapsed into one predicate because A.7 only ever needs +-- the union). Backed by the compact range tables in +-- "Text.IDNA2008.Internal.Script.Data". + +module Text.IDNA2008.Internal.Script + ( isGreekCp + , isHebrewCp + , isHkhCp + ) where + +import Text.IDNA2008.Internal.Ranges (inRanges) +import Text.IDNA2008.Internal.Script.Data + ( greekRangeCount + , greekRanges + , hebrewRangeCount + , hebrewRanges + , hkhRangeCount + , hkhRanges + ) + +-- | Is @cp@ in the Greek script? RFC 5892 A.4 consults this on +-- the codepoint immediately following @U+0375@ (Greek Lower +-- Numeral Sign). +isGreekCp :: Int -> Bool +isGreekCp = inRanges greekRanges greekRangeCount +{-# INLINE isGreekCp #-} + +-- | Is @cp@ in the Hebrew script? RFC 5892 A.5 \/ A.6 consult +-- this on the codepoint immediately preceding @U+05F3@ (Hebrew +-- Punctuation Geresh) or @U+05F4@ (Hebrew Punctuation +-- Gershayim). +isHebrewCp :: Int -> Bool +isHebrewCp = inRanges hebrewRanges hebrewRangeCount +{-# INLINE isHebrewCp #-} + +-- | Is @cp@ in Hiragana, Katakana, or Han? RFC 5892 A.7 +-- requires that a label containing @U+30FB@ (Katakana Middle +-- Dot) also contain at least one codepoint in this union. +isHkhCp :: Int -> Bool +isHkhCp = inRanges hkhRanges hkhRangeCount +{-# INLINE isHkhCp #-} + diff --git a/internal/Text/IDNA2008/Internal/Script/Data.hs b/internal/Text/IDNA2008/Internal/Script/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Script/Data.hs @@ -0,0 +1,174 @@ +-- | +-- Module : Text.IDNA2008.Internal.Script.Data +-- Description : Compact codepoint range tables for the Unicode +-- Scripts consulted by the CONTEXTO contextual rules. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaScript.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/Scripts.txt +-- Aligned with: Unicode 17.0.0 +-- Greek ranges: 36 +-- Hebrew ranges: 9 +-- HKH (Hiragana | Katakana | Han) ranges: 39 +-- +-- Only the four scripts whose membership is tested by RFC 5892 +-- Appendix A.4-A.7 (Greek, Hebrew, Hiragana \/ Katakana \/ Han) are +-- represented. Each script's @sc@ (Script) ranges are stored as a +-- sorted sequence of @(start, end)@ pairs (inclusive on both ends), +-- packed into a single 'ByteArray' as alternating 'Word32's: +-- +-- > [start0, end0, start1, end1, ...] +-- +-- Lookup is binary search for the largest @start_i \<= cp@ and a +-- bounds check @cp \<= end_i@; see "Text.IDNA2008.Internal.Script". +-- +-- The Hiragana, Katakana, and Han ranges are merged into a single +-- @hkhRanges@ table because RFC 5892 A.7 only ever asks for +-- \"any of these three\", and the merged table costs no extra +-- comparisons at lookup time. Greek and Hebrew remain separate +-- because A.4 only asks for Greek and A.5\/A.6 only for Hebrew. +module Text.IDNA2008.Internal.Script.Data + ( greekRangeCount + , greekRanges + , hebrewRangeCount + , hebrewRanges + , hkhRangeCount + , hkhRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + + +---------------------------------------------------------------------- +-- Greek (Script: Grek) +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'greekRanges'. +greekRangeCount :: Int +greekRangeCount = 36 + +-- | Greek script codepoint ranges, alternating start\/end. +greekRanges :: ByteArray +greekRanges = PBA.byteArrayFromList + [ (0x000370 :: Word32), 0x000373 + , 0x000375, 0x000377 + , 0x00037A, 0x00037D + , 0x00037F, 0x00037F + , 0x000384, 0x000384 + , 0x000386, 0x000386 + , 0x000388, 0x00038A + , 0x00038C, 0x00038C + , 0x00038E, 0x0003A1 + , 0x0003A3, 0x0003E1 + , 0x0003F0, 0x0003FF + , 0x001D26, 0x001D2A + , 0x001D5D, 0x001D61 + , 0x001D66, 0x001D6A + , 0x001DBF, 0x001DBF + , 0x001F00, 0x001F15 + , 0x001F18, 0x001F1D + , 0x001F20, 0x001F45 + , 0x001F48, 0x001F4D + , 0x001F50, 0x001F57 + , 0x001F59, 0x001F59 + , 0x001F5B, 0x001F5B + , 0x001F5D, 0x001F5D + , 0x001F5F, 0x001F7D + , 0x001F80, 0x001FB4 + , 0x001FB6, 0x001FC4 + , 0x001FC6, 0x001FD3 + , 0x001FD6, 0x001FDB + , 0x001FDD, 0x001FEF + , 0x001FF2, 0x001FF4 + , 0x001FF6, 0x001FFE + , 0x002126, 0x002126 + , 0x00AB65, 0x00AB65 + , 0x010140, 0x01018E + , 0x0101A0, 0x0101A0 + , 0x01D200, 0x01D245 + ] + +---------------------------------------------------------------------- +-- Hebrew (Script: Hebr) +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'hebrewRanges'. +hebrewRangeCount :: Int +hebrewRangeCount = 9 + +-- | Hebrew script codepoint ranges, alternating start\/end. +hebrewRanges :: ByteArray +hebrewRanges = PBA.byteArrayFromList + [ (0x000591 :: Word32), 0x0005C7 + , 0x0005D0, 0x0005EA + , 0x0005EF, 0x0005F4 + , 0x00FB1D, 0x00FB36 + , 0x00FB38, 0x00FB3C + , 0x00FB3E, 0x00FB3E + , 0x00FB40, 0x00FB41 + , 0x00FB43, 0x00FB44 + , 0x00FB46, 0x00FB4F + ] + +---------------------------------------------------------------------- +-- Hiragana | Katakana | Han (merged for RFC 5892 A.7) +---------------------------------------------------------------------- + +-- | Number of (start, end) range pairs in 'hkhRanges'. +hkhRangeCount :: Int +hkhRangeCount = 39 + +-- | Codepoints whose @sc@ (Script) is Hiragana, Katakana, +-- or Han, merged into one sorted range list. +hkhRanges :: ByteArray +hkhRanges = PBA.byteArrayFromList + [ (0x002E80 :: Word32), 0x002E99 + , 0x002E9B, 0x002EF3 + , 0x002F00, 0x002FD5 + , 0x003005, 0x003005 + , 0x003007, 0x003007 + , 0x003021, 0x003029 + , 0x003038, 0x00303B + , 0x003041, 0x003096 + , 0x00309D, 0x00309F + , 0x0030A1, 0x0030FA + , 0x0030FD, 0x0030FF + , 0x0031F0, 0x0031FF + , 0x0032D0, 0x0032FE + , 0x003300, 0x003357 + , 0x003400, 0x004DBF + , 0x004E00, 0x009FFF + , 0x00F900, 0x00FA6D + , 0x00FA70, 0x00FAD9 + , 0x00FF66, 0x00FF6F + , 0x00FF71, 0x00FF9D + , 0x016FE2, 0x016FE3 + , 0x016FF0, 0x016FF6 + , 0x01AFF0, 0x01AFF3 + , 0x01AFF5, 0x01AFFB + , 0x01AFFD, 0x01AFFE + , 0x01B000, 0x01B122 + , 0x01B132, 0x01B132 + , 0x01B150, 0x01B152 + , 0x01B155, 0x01B155 + , 0x01B164, 0x01B167 + , 0x01F200, 0x01F200 + , 0x020000, 0x02A6DF + , 0x02A700, 0x02B81D + , 0x02B820, 0x02CEAD + , 0x02CEB0, 0x02EBE0 + , 0x02EBF0, 0x02EE5D + , 0x02F800, 0x02FA1D + , 0x030000, 0x03134A + , 0x031350, 0x033479 + ] diff --git a/internal/Text/IDNA2008/Internal/Tokens.hs b/internal/Text/IDNA2008/Internal/Tokens.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Tokens.hs @@ -0,0 +1,224 @@ +-- | +-- Module : Text.IDNA2008.Internal.Tokens +-- Description : Command-line option syntax for bit-flag types. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Reusable parser and renderer for monoidal bit-flag types +-- ('Text.IDNA2008.Internal.LabelFormSet.LabelFormSet', +-- 'Text.IDNA2008.Internal.Flags.IdnaFlags') exposed on the +-- command line as comma-separated lists of canonical names with +-- @+@\/@-@ prefix syntax for set arithmetic. +-- +-- Each type provides a table of 'TokenEntry' values (one per +-- single-bit flag, with a canonical CLI name and an optional list +-- of memorable aliases) plus an optional list of preset names. +-- The functions in this module turn that data into a parser and a +-- renderer. +-- +-- Lookup is case-insensitive. An exact match against the +-- canonical name, any alias, or any preset wins. If no exact +-- match is found, a unique prefix match against the same union +-- is accepted; ambiguous prefixes are rejected with an error +-- listing the candidates. +{-# LANGUAGE CPP #-} + +module Text.IDNA2008.Internal.Tokens + ( -- * Token table entries + TokenEntry(..) + , Preset + -- * Parser + , parseTokens + -- * String-input helper (ASCII validation) + , asciiByteString + ) where + +import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BS8 +import Data.ByteString (ByteString) +import Data.Char (isSpace, toLower) +#if !MIN_VERSION_base(4,20,0) +import Data.Foldable(foldl') +#endif +import Data.List (intercalate, nubBy, sortOn) + +---------------------------------------------------------------------- +-- Token-table types +---------------------------------------------------------------------- + +-- | One row of a CLI token table: a single-bit flag value plus +-- the canonical CLI name and any number of additional aliases +-- accepted on input. Aliases are case-insensitive; the +-- canonical name (also case-insensitive on input) is the one the +-- renderer emits. +data TokenEntry a = TokenEntry + { teValue :: !a -- ^ The flag this row describes. + , teCanonical :: !ByteString -- ^ Canonical CLI name (lower-case). + , teAliases :: ![ByteString] -- ^ Additional accepted forms. + } + +-- | A preset is a named multi-bit composite (e.g. @host@ for +-- 'Text.IDNA2008.Internal.LabelFormSet.LabelFormSet'). Presets are +-- an input-side convenience and are never emitted by the +-- renderer. +type Preset a = (ByteString, a) + +---------------------------------------------------------------------- +-- Parser +---------------------------------------------------------------------- + +-- | Parse a comma-separated CLI value into a monoidal flag set. +-- +-- Each comma-separated token is optionally prefixed with @\'+\'@ +-- (additive, the default) or @\'-\'@ (subtractive). The token +-- name is resolved against the canonical names, aliases, and +-- preset names; case-insensitively, exact match preferred, +-- unambiguous prefix match accepted as fallback. Prefix +-- matches require at least three characters of input so that +-- short shortcuts that happen to be unambiguous today don't +-- quietly break when a future release adds a collision. +-- +-- The starting state of the running result depends on the first +-- token's leading character: +-- +-- * If the first token has a @\'+\'@ or @\'-\'@ prefix, the +-- running result is seeded with the caller-supplied +-- @implicitBase@. This makes @\"+X\"@ read naturally as +-- \"add X to my default\" and @\"-X\"@ as \"remove X from +-- my default\". +-- * Otherwise the running result starts at 'mempty', so a +-- leading absolute token replaces the default cleanly. +-- +-- Whitespace around commas and around tokens is ignored. An +-- empty input string yields 'mempty'. +parseTokens + :: forall a. (Monoid a) + => [TokenEntry a] -- ^ Single-bit token table. + -> [Preset a] -- ^ Multi-bit presets. + -> a -- ^ Implicit base used when input + -- begins with @+@ or @-@. + -> (a -> a -> a) -- ^ Subtractive op (set \`op\` value). + -> ByteString -- ^ Input. + -> Either String a +parseTokens !table !presets !implicitBase !withoutOp !input = + foldl' step (Right initial) tokens + where + tokens = splitComma input + initial = case tokens of + (t:_) | hasLeadingSign (trim t) -> implicitBase + _ -> mempty + + hasLeadingSign b = case BS8.uncons b of + Just ('+', _) -> True + Just ('-', _) -> True + _ -> False + + -- Flat name -> value map, with names lower-cased. + nameMap :: [(ByteString, a)] + nameMap = nubFstOn $ + concatMap entryNames table + ++ [(BS8.map toLower n, v) | (n, v) <- presets] + + entryNames :: TokenEntry a -> [(ByteString, a)] + entryNames (TokenEntry v c as) = + [(BS8.map toLower n, v) | n <- c : as] + + -- Sorted by length so prefix matching reports candidates in + -- a stable, readable order. + sortedNames :: [(ByteString, a)] + sortedNames = sortOn (BS.length . fst) nameMap + + step :: Either String a -> ByteString -> Either String a + step (Left e) _ = Left e + step (Right s) tok = + let (op, name) = splitOp (trim tok) + in if BS.null name + then Left "empty token in CLI flag list" + else case lookupName name of + Right v -> Right (op s v) + Left e -> Left e + + splitOp :: ByteString -> (a -> a -> a, ByteString) + splitOp b = case BS8.uncons b of + Just ('+', rest) -> ((<>), rest) + Just ('-', rest) -> (withoutOp, rest) + _ -> ((<>), b) + + lookupName :: ByteString -> Either String a + lookupName !name = case lookup nameLc nameMap of + Just v -> Right v + Nothing + | BS.length nameLc < minPrefixLen -> + Left $ "token " ++ show (BS8.unpack name) + ++ " is too short for prefix matching" + ++ " (need at least " + ++ show minPrefixLen + ++ " characters)" + | otherwise -> case prefixMatches of + [(_, v)] -> Right v + [] -> Left $ "unknown token " + ++ show (BS8.unpack name) + cs -> Left $ "ambiguous token " + ++ show (BS8.unpack name) + ++ "; matches: " + ++ intercalate ", " + [BS8.unpack n | (n, _) <- cs] + where + nameLc = BS8.map toLower name + prefixMatches = + [ entry | entry@(n, _) <- sortedNames + , nameLc `BS.isPrefixOf` n ] + + -- | Minimum input length for an inexact (prefix) match. An + -- exact match is always accepted regardless of length; this + -- floor only constrains the prefix-match fallback so that + -- short shortcuts can't be added implicitly today and + -- silently invalidated by a later release that shares the + -- prefix. Three characters is enough for the short prefixes + -- we want today (@asc@, @bid@, @att@, ...) while leaving + -- room for future tokens to share a 1- or 2-letter prefix + -- without breaking existing scripts. + minPrefixLen :: Int + minPrefixLen = 3 + +---------------------------------------------------------------------- +-- String-input helper +---------------------------------------------------------------------- + +-- | Validate that a 'String' contains only ASCII codepoints +-- (@\< '\\x80'@) and convert it to a 'ByteString' suitable for +-- 'parseTokens'. +asciiByteString :: String -> Either String ByteString +asciiByteString s = case break (>= '\x80') s of + (ok, []) -> Right (BS8.pack ok) + (ok, bad:_) -> Left $ "non-ASCII character " + ++ show bad + ++ " at column " + ++ show (length ok + 1) + ++ " of " + ++ show s + +---------------------------------------------------------------------- +-- Internals +---------------------------------------------------------------------- + +-- | Split @input@ on commas, dropping empty fragments at the +-- ends and around adjacent commas. +splitComma :: ByteString -> [ByteString] +splitComma !s = + filter (not . BS.null . trim) + (BS.split (fromIntegral (fromEnum ',')) s) + +-- | Strip ASCII whitespace from both ends. +trim :: ByteString -> ByteString +trim = BS8.dropWhileEnd isSpace . BS8.dropWhile isSpace + +-- | Deduplicate by first component, keeping the first +-- occurrence. Used to discard accidentally-duplicated entries +-- (e.g. an alias that matches the canonical name of a different +-- token in a misconfigured table). +nubFstOn :: Eq k => [(k, v)] -> [(k, v)] +nubFstOn = nubBy (\(a, _) (b, _) -> a == b) diff --git a/internal/Text/IDNA2008/Internal/UTS46.hs b/internal/Text/IDNA2008/Internal/UTS46.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/UTS46.hs @@ -0,0 +1,275 @@ +-- | +-- Module : Text.IDNA2008.Internal.UTS46 +-- Description : Hand-curated allow-list of UTS #46 mappings adopted +-- by the @MAPUTS46@ option, beyond strict IDNA2008. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- A single lookup, 'uts46Lookup', returning the multi-codepoint +-- target of each source codepoint in the @MAPUTS46@ allow-list, or +-- 'Nothing' for codepoints outside the allow-list. Used by the +-- pre-validation MAPUTS46 pass in "Text.IDNA2008.Internal.Parse" +-- when the @MAPUTS46@ option is set. The full list of in\\/out +-- decisions and rationale lives in the code-comment block in the +-- source of this module, immediately above 'uts46Lookup'. +{-# LANGUAGE LambdaCase #-} + +module Text.IDNA2008.Internal.UTS46 + ( uts46Lookup + ) where + +-- --------------------------------------------------------------------------- +-- Rationale -- scope of the MAPUTS46 allow-list +-- --------------------------------------------------------------------------- +-- +-- Strict IDNA2008 has no notion of compatibility folding beyond the four +-- RFC 5895 input mappings (MAPDOTS, MAPCASE, MAPNFC, MAPWIDTH). UTS #46 +-- offers a much larger table of mappings (~6400 entries in Unicode +-- 17.0.0). We adopt only the small subset that passes a joint test: +-- +-- 1. SOURCE. An IME or normal text-entry path might inadvertently +-- produce this codepoint while a user attempts to type a domain- +-- name component. Codepoints requiring a Unicode picker, math +-- editor, or typesetting tool are NOT in scope. +-- +-- 2. TARGET. The mapping result is plausible content for a real +-- domain label — a name, word, or letter sequence in a script +-- that registrars actually accept — rather than an editorial +-- expansion of a symbol. +-- +-- Both halves must pass. Failure of either disqualifies the entry. +-- +-- IN -- Japanese era / corporation IME shortcuts (6 entries) +-- ---------------------------------------------------------- +-- Modern Japanese IMEs (Mozc, Microsoft IME, Google IME) propose these +-- single-codepoint squared forms as one-keystroke alternatives to typing +-- out the spelling. The expanded targets are exactly what an attentive +-- user would have chosen. +-- +-- U+32FF (SQUARE ERA NAME REIWA) -> U+4EE4 U+548C (Reiwa) +-- U+337B (SQUARE ERA NAME HEISEI) -> U+5E73 U+6210 (Heisei) +-- U+337C (SQUARE ERA NAME SYOUWA) -> U+662D U+548C (Showa) +-- U+337D (SQUARE ERA NAME TAISYOU) -> U+5927 U+6B63 (Taisho) +-- U+337E (SQUARE ERA NAME MEIZI) -> U+660E U+6CBB (Meiji) +-- +-- IN -- Circled CJK ideographs at U+3244..U+3247 and U+3280..U+32B0 +-- EXCLUDING U+3297 and U+3299 (51 entries) +-- ---------------------------------------------------------------- +-- Each source codepoint maps to a single CJK ideograph (e.g. U+3280 +-- CIRCLED IDEOGRAPH ONE -> U+4E00, U+329F CIRCLED IDEOGRAPH ATTENTION +-- -> U+6CE8). These are pre-emoji codepoints from the CJK +-- Compatibility block that some Japanese/Chinese IMEs propose as a +-- stylistic option when marking lists or notes; the target ideograph +-- is everyday content. Two codepoints in this range (U+3297 and +-- U+3299) are deliberately omitted on the same grounds as the +-- squared CJK ideograph emoji; see the OUT list below. +-- +-- OUT -- Squared CJK ideograph emoji (U+1F210..U+1F23B, U+1F250..U+1F251) +-- ---------------------------------------------------------------------- +-- The UTS #46 mapping table folds these to single CJK ideographs +-- (e.g. U+1F238 SQUARED CJK UNIFIED IDEOGRAPH-7533 maps to U+7533). +-- We do NOT adopt these mappings. +-- +-- Empirical (May 2026, dataset of 6300 registered emoji-bearing IDNs): +-- the .ws registry has registered xn--q97h.ws (U+1F238 as a label +-- under .ws) as a parking-lot domain. Safari, applying UTS #46 +-- mapping, silently routes lookups of that label to xn--uny.ws +-- (U+7533 as a label under .ws), which is a SEPARATELY registered +-- domain belonging to a different operator (Quarken URL Shortener). +-- The two namespaces are distinct, the user's intended destination +-- and actual destination differ, and the user has no signal that the +-- rewrite occurred. Future re-registrations could turn the kanji- +-- form target into a phishing site without the emoji-form +-- registrant's knowledge or consent. +-- +-- libidn2 in non-transitional mode rejects these codepoints outright +-- ("character forbidden in non-transitional mode"); the mainstream +-- tooling landscape is split, with browsers folding and BIND/dig +-- refusing. Our refusing-to-fold position is conservative but +-- defensible. See the EMOJIOK pattern in +-- "Text.IDNA2008.Internal.Flags" for the parallel decision on the +-- admit-as-is side: these codepoints are excluded from EMOJIOK too, +-- so under no combination of flags will this library route a +-- U+1F238 label to a U+7533 label. +-- +-- OUT -- U+3297 CIRCLED IDEOGRAPH CONGRATULATION and U+3299 CIRCLED +-- IDEOGRAPH SECRET +-- ------------------------------------------------------------- +-- Although these two codepoints sit in the U+3280..U+32B0 BMP block +-- whose other members are in the IN list, they are also @Emoji=Yes@ +-- and folded by UTS #46 -- the same cross-tool ambiguity shape as +-- the squared CJK ideograph emoji. Their UTS #46 fold targets +-- (U+795D and U+79D8 respectively) could attract separate +-- registration by an opportunistic third party at any time; +-- empirically there is no live divergence today (the kanji-form +-- .ws labels resolve to NXDOMAIN), but folding by default would +-- bake in a future hijack vector with no signal to the user. +-- Refused symmetrically with the rest of the mappedEmojiRanges +-- set. +-- +-- OUT -- Squared kana abbreviations (U+3300..U+3357, 88 entries) +-- ------------------------------------------------------------- +-- Each source folds to a multi-character katakana loanword like +-- "calorie", "dollar", or "mansion" -- a complete Japanese word +-- rather than single-label content. The source codepoints exist +-- primarily for vertical-typesetting space-economy in Japanese +-- newspapers, not IME output. +-- +-- OUT -- Squared unit / math abbreviations (U+3382..U+33DF range) +-- --------------------------------------------------------------- +-- Each source folds to mixed Greek+Latin or unit-notation sequences +-- like "us", "mhz", "m/s", or "kohm", several of which contain +-- U+2215 (DIVISION SLASH, itself disallowed in IDNA2008). Not +-- plausible label content. +-- +-- OUT -- Circled Katakana (U+32D0..U+32FE, 50 entries) +-- --------------------------------------------------- +-- Each source folds to a single katakana. Such labels appear in +-- domain registrations rarely enough that the IME-accident +-- probability for the circled form is lower than the false-positive +-- cost of folding. +-- +-- OUT -- Circled Hangul (U+3260..U+327E, 35 entries) +-- -------------------------------------------------- +-- Same rationale as circled Katakana; single-jamo / one-syllable +-- forms are not the kind of content registrars routinely accept as +-- whole labels. +-- +-- OUT -- Arabic positional forms (U+FB50..U+FDFD and U+FE70..U+FEFC, +-- ~706 entries) +-- ---------------------------------------------------------------- +-- Targets are natural Arabic base letters, but the sources are +-- presentation forms not produced by any modern Arabic IME. They +-- appear in user input only from copy-paste out of legacy PDFs that +-- baked shaping into the encoding. IDNA2008 explicitly disallows the +-- source codepoints; we honour that disposition. +-- +-- OUT -- Latin compatibility ligatures (U+FB00..U+FB06, 7 entries) +-- ---------------------------------------------------------------- +-- Sources fold to ASCII letter sequences (fi, fl, ffi, ffl, ff, st). +-- Copy-paste from typeset documents can produce these, but the +-- population producing such documents is not shopping for domain +-- names, and the targets are ASCII (we reject non-ASCII -> ASCII +-- mappings on principle). +-- +-- OUT -- Latin digraphs (U+0132 U+01C4 U+01C7 U+01CA U+01F1 and their +-- title/lower variants, 11 entries) +-- ----------------------------------------------------------------- +-- Even rarer than the ligatures; no auto-conversion path puts them +-- in user input. +-- +-- OUT -- Math alphanumeric / font-variant Latin and Greek (~439 +-- entries) +-- ------------------------------------------------------------ +-- Sources are math-italic, math-bold, fraktur, script, and related +-- typesetting alphabets (e.g. U+210F PLANCK CONSTANT OVER TWO PI -> +-- U+0127, U+1D400 MATHEMATICAL BOLD CAPITAL A -> "a"). Math +-- typesetting only; never an IME or text-entry accident. +-- +-- OUT -- Roman numerals, vulgar fractions, symbol expansions, +-- superscripts, subscripts, degree-Celsius/Fahrenheit -- +-- everything else in the UTS #46 mapping table. +-- ---------------------------------------------------------- +-- Examples: U+2122 TRADE MARK SIGN -> "tm", U+2116 NUMERO SIGN -> +-- "no", U+2121 TELEPHONE SIGN -> "tel", U+2175 SMALL ROMAN NUMERAL +-- SIX -> "vi", U+00BC VULGAR FRACTION ONE QUARTER -> "1" U+2044 "4", +-- U+00B2 SUPERSCRIPT TWO -> "2", U+2103 DEGREE CELSIUS -> U+00B0 "C". +-- Either the target isn't plausible domain content (fraction slash, +-- degree sign), or the source requires deliberate insertion via a +-- Unicode picker. +-- +-- --------------------------------------------------------------------------- +-- Revision discipline +-- --------------------------------------------------------------------------- +-- Any change to the IN\\/OUT classifications above must carry, in the +-- same commit: +-- +-- 1. A clear rationale, written as an extension or amendment of the +-- relevant section of the comment block above — so the standing +-- record of why each entry is where it is stays current and +-- self-contained. +-- +-- 2. Test coverage in one or more of the test programs (the unit +-- tests under tests/, the conformance suite, or the idnaparse \\/ +-- idnabench harnesses) exercising the new or revised behaviour. +-- --------------------------------------------------------------------------- + +-- | Look up @cp@ in the @MAPUTS46@ allow-list. Returns the list of +-- target codepoints to substitute (always non-empty), or 'Nothing' +-- if @cp@ is not in the allow-list. +-- +-- The @'Int' -> 'Maybe' ['Int']@ shape mirrors the way UTS #46 +-- mappings work: a single source codepoint may expand to one or +-- several target codepoints (era-name codepoints expand 1:2, the +-- circled CJK ideographs expand 1:1). +uts46Lookup :: Int -> Maybe [Int] +uts46Lookup = \case + -- Japanese era + 0x32FF -> Just [0x4EE4, 0x548C] -- Reiwa + 0x337B -> Just [0x5E73, 0x6210] -- Heisei + 0x337C -> Just [0x662D, 0x548C] -- Showa + 0x337D -> Just [0x5927, 0x6B63] -- Taisho + 0x337E -> Just [0x660E, 0x6CBB] -- Meiji + + -- Circled CJK ideographs in U+3244..U+3247 + 0x3244 -> Just [0x554F] -- question + 0x3245 -> Just [0x5E7C] -- kindergarten + 0x3246 -> Just [0x6587] -- school + 0x3247 -> Just [0x7B8F] -- koto + + -- Circled CJK ideographs in U+3280..U+32B0 + 0x3280 -> Just [0x4E00] -- one + 0x3281 -> Just [0x4E8C] -- two + 0x3282 -> Just [0x4E09] -- three + 0x3283 -> Just [0x56DB] -- four + 0x3284 -> Just [0x4E94] -- five + 0x3285 -> Just [0x516D] -- six + 0x3286 -> Just [0x4E03] -- seven + 0x3287 -> Just [0x516B] -- eight + 0x3288 -> Just [0x4E5D] -- nine + 0x3289 -> Just [0x5341] -- ten + 0x328A -> Just [0x6708] -- moon + 0x328B -> Just [0x706B] -- fire + 0x328C -> Just [0x6C34] -- water + 0x328D -> Just [0x6728] -- wood + 0x328E -> Just [0x91D1] -- metal + 0x328F -> Just [0x571F] -- earth + 0x3290 -> Just [0x65E5] -- sun + 0x3291 -> Just [0x682A] -- stock + 0x3292 -> Just [0x6709] -- have + 0x3293 -> Just [0x793E] -- society + 0x3294 -> Just [0x540D] -- name + 0x3295 -> Just [0x7279] -- special + 0x3296 -> Just [0x8CA1] -- financial + -- U+3297, U+3299 deliberately excluded (also mappedEmoji); + -- see the OUT-list rationale block above. + 0x3298 -> Just [0x52B4] -- labor + 0x329A -> Just [0x7537] -- male + 0x329B -> Just [0x5973] -- female + 0x329C -> Just [0x9069] -- suitable + 0x329D -> Just [0x512A] -- excellent + 0x329E -> Just [0x5370] -- print + 0x329F -> Just [0x6CE8] -- attention + 0x32A0 -> Just [0x9805] -- item + 0x32A1 -> Just [0x4F11] -- rest + 0x32A2 -> Just [0x5199] -- copy + 0x32A3 -> Just [0x6B63] -- correct + 0x32A4 -> Just [0x4E0A] -- high + 0x32A5 -> Just [0x4E2D] -- centre + 0x32A6 -> Just [0x4E0B] -- low + 0x32A7 -> Just [0x5DE6] -- left + 0x32A8 -> Just [0x53F3] -- right + 0x32A9 -> Just [0x533B] -- medicine + 0x32AA -> Just [0x5B97] -- religion + 0x32AB -> Just [0x5B66] -- study + 0x32AC -> Just [0x76E3] -- supervise + 0x32AD -> Just [0x4F01] -- enterprise + 0x32AE -> Just [0x8CC7] -- resource + 0x32AF -> Just [0x5354] -- alliance + 0x32B0 -> Just [0x591C] -- night + + _ -> Nothing +{-# INLINE uts46Lookup #-} diff --git a/internal/Text/IDNA2008/Internal/Width.hs b/internal/Text/IDNA2008/Internal/Width.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Width.hs @@ -0,0 +1,61 @@ +-- | +-- Module : Text.IDNA2008.Internal.Width +-- Description : Typed wrapper over the fullwidth\/halfwidth +-- decomposition table consumed by the @MAPWIDTH@ +-- mapping (RFC 5895 section 2.2). +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- One lookup, 'widthMapCp', backed by binary search over the sorted +-- parallel arrays in "Text.IDNA2008.Internal.Width.Data". +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE MultiWayIf #-} + +module Text.IDNA2008.Internal.Width + ( widthMapCp + ) where + +import Data.Primitive.ByteArray (indexByteArray) +import Data.Word (Word32) + +import Text.IDNA2008.Internal.Width.Data + ( widthRangeCount + , widthSources + , widthTargets + ) + +-- | Apply the RFC 5895 section 2.2 fullwidth\/halfwidth mapping to +-- a single codepoint. Returns the input unchanged if it has no +-- @\@ or @\@ decomposition. Codepoints absent +-- from the table (which is the vast majority) are unaffected. +widthMapCp :: Int -> Int +widthMapCp !cp + | widthRangeCount == 0 = cp + | cp < srcAt 0 = cp + | cp > srcAt (widthRangeCount - 1) = cp + | otherwise = case bsearch 0 (widthRangeCount - 1) of + Just i -> tgtAt i + Nothing -> cp + where + srcAt :: Int -> Int + srcAt !i = fromIntegral (indexByteArray widthSources i :: Word32) + {-# INLINE srcAt #-} + + tgtAt :: Int -> Int + tgtAt !i = fromIntegral (indexByteArray widthTargets i :: Word32) + {-# INLINE tgtAt #-} + + -- Standard binary search for an exact match. + bsearch :: Int -> Int -> Maybe Int + bsearch !lo !hi + | lo > hi = Nothing + | otherwise = + let !mid = (lo + hi) `quot` 2 + !k = srcAt mid + in if | k == cp -> Just mid + | k < cp -> bsearch (mid + 1) hi + | otherwise -> bsearch lo (mid - 1) +{-# INLINE widthMapCp #-} diff --git a/internal/Text/IDNA2008/Internal/Width/Data.hs b/internal/Text/IDNA2008/Internal/Width/Data.hs new file mode 100644 --- /dev/null +++ b/internal/Text/IDNA2008/Internal/Width/Data.hs @@ -0,0 +1,500 @@ +-- | +-- Module : Text.IDNA2008.Internal.Width.Data +-- Description : Compact codepoint mapping table for the Unicode +-- wide \/ narrow decompositions consumed by the +-- RFC 5895 section 2.2 (fullwidth\/halfwidth) mapping. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaWidth.py \\ +-- +-- +-- Source: https://www.unicode.org/Public/17.0.0/ucd/UnicodeData.txt +-- Aligned with: Unicode 17.0.0 +-- Width-mapping entries: 226 +-- +-- Each entry is a single-codepoint decomposition: a fullwidth or +-- halfwidth codepoint that maps to a single canonical-width target. +-- Stored as two parallel sorted arrays -- 'widthSources' (source +-- codepoints, sorted ascending) and 'widthTargets' (the targets, in +-- the same order). Lookup is binary search over 'widthSources'. +module Text.IDNA2008.Internal.Width.Data + ( widthRangeCount + , widthSources + , widthTargets + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +-- | Number of (source, target) entries. +widthRangeCount :: Int +widthRangeCount = 226 + +-- | Source codepoints, sorted ascending. +widthSources :: ByteArray +widthSources = PBA.byteArrayFromList + [ (0x003000 :: Word32) + , 0x00FF01 + , 0x00FF02 + , 0x00FF03 + , 0x00FF04 + , 0x00FF05 + , 0x00FF06 + , 0x00FF07 + , 0x00FF08 + , 0x00FF09 + , 0x00FF0A + , 0x00FF0B + , 0x00FF0C + , 0x00FF0D + , 0x00FF0E + , 0x00FF0F + , 0x00FF10 + , 0x00FF11 + , 0x00FF12 + , 0x00FF13 + , 0x00FF14 + , 0x00FF15 + , 0x00FF16 + , 0x00FF17 + , 0x00FF18 + , 0x00FF19 + , 0x00FF1A + , 0x00FF1B + , 0x00FF1C + , 0x00FF1D + , 0x00FF1E + , 0x00FF1F + , 0x00FF20 + , 0x00FF21 + , 0x00FF22 + , 0x00FF23 + , 0x00FF24 + , 0x00FF25 + , 0x00FF26 + , 0x00FF27 + , 0x00FF28 + , 0x00FF29 + , 0x00FF2A + , 0x00FF2B + , 0x00FF2C + , 0x00FF2D + , 0x00FF2E + , 0x00FF2F + , 0x00FF30 + , 0x00FF31 + , 0x00FF32 + , 0x00FF33 + , 0x00FF34 + , 0x00FF35 + , 0x00FF36 + , 0x00FF37 + , 0x00FF38 + , 0x00FF39 + , 0x00FF3A + , 0x00FF3B + , 0x00FF3C + , 0x00FF3D + , 0x00FF3E + , 0x00FF3F + , 0x00FF40 + , 0x00FF41 + , 0x00FF42 + , 0x00FF43 + , 0x00FF44 + , 0x00FF45 + , 0x00FF46 + , 0x00FF47 + , 0x00FF48 + , 0x00FF49 + , 0x00FF4A + , 0x00FF4B + , 0x00FF4C + , 0x00FF4D + , 0x00FF4E + , 0x00FF4F + , 0x00FF50 + , 0x00FF51 + , 0x00FF52 + , 0x00FF53 + , 0x00FF54 + , 0x00FF55 + , 0x00FF56 + , 0x00FF57 + , 0x00FF58 + , 0x00FF59 + , 0x00FF5A + , 0x00FF5B + , 0x00FF5C + , 0x00FF5D + , 0x00FF5E + , 0x00FF5F + , 0x00FF60 + , 0x00FF61 + , 0x00FF62 + , 0x00FF63 + , 0x00FF64 + , 0x00FF65 + , 0x00FF66 + , 0x00FF67 + , 0x00FF68 + , 0x00FF69 + , 0x00FF6A + , 0x00FF6B + , 0x00FF6C + , 0x00FF6D + , 0x00FF6E + , 0x00FF6F + , 0x00FF70 + , 0x00FF71 + , 0x00FF72 + , 0x00FF73 + , 0x00FF74 + , 0x00FF75 + , 0x00FF76 + , 0x00FF77 + , 0x00FF78 + , 0x00FF79 + , 0x00FF7A + , 0x00FF7B + , 0x00FF7C + , 0x00FF7D + , 0x00FF7E + , 0x00FF7F + , 0x00FF80 + , 0x00FF81 + , 0x00FF82 + , 0x00FF83 + , 0x00FF84 + , 0x00FF85 + , 0x00FF86 + , 0x00FF87 + , 0x00FF88 + , 0x00FF89 + , 0x00FF8A + , 0x00FF8B + , 0x00FF8C + , 0x00FF8D + , 0x00FF8E + , 0x00FF8F + , 0x00FF90 + , 0x00FF91 + , 0x00FF92 + , 0x00FF93 + , 0x00FF94 + , 0x00FF95 + , 0x00FF96 + , 0x00FF97 + , 0x00FF98 + , 0x00FF99 + , 0x00FF9A + , 0x00FF9B + , 0x00FF9C + , 0x00FF9D + , 0x00FF9E + , 0x00FF9F + , 0x00FFA0 + , 0x00FFA1 + , 0x00FFA2 + , 0x00FFA3 + , 0x00FFA4 + , 0x00FFA5 + , 0x00FFA6 + , 0x00FFA7 + , 0x00FFA8 + , 0x00FFA9 + , 0x00FFAA + , 0x00FFAB + , 0x00FFAC + , 0x00FFAD + , 0x00FFAE + , 0x00FFAF + , 0x00FFB0 + , 0x00FFB1 + , 0x00FFB2 + , 0x00FFB3 + , 0x00FFB4 + , 0x00FFB5 + , 0x00FFB6 + , 0x00FFB7 + , 0x00FFB8 + , 0x00FFB9 + , 0x00FFBA + , 0x00FFBB + , 0x00FFBC + , 0x00FFBD + , 0x00FFBE + , 0x00FFC2 + , 0x00FFC3 + , 0x00FFC4 + , 0x00FFC5 + , 0x00FFC6 + , 0x00FFC7 + , 0x00FFCA + , 0x00FFCB + , 0x00FFCC + , 0x00FFCD + , 0x00FFCE + , 0x00FFCF + , 0x00FFD2 + , 0x00FFD3 + , 0x00FFD4 + , 0x00FFD5 + , 0x00FFD6 + , 0x00FFD7 + , 0x00FFDA + , 0x00FFDB + , 0x00FFDC + , 0x00FFE0 + , 0x00FFE1 + , 0x00FFE2 + , 0x00FFE3 + , 0x00FFE4 + , 0x00FFE5 + , 0x00FFE6 + , 0x00FFE8 + , 0x00FFE9 + , 0x00FFEA + , 0x00FFEB + , 0x00FFEC + , 0x00FFED + , 0x00FFEE + ] + +-- | Target codepoints; parallel to 'widthSources'. +widthTargets :: ByteArray +widthTargets = PBA.byteArrayFromList + [ (0x000020 :: Word32) + , 0x000021 + , 0x000022 + , 0x000023 + , 0x000024 + , 0x000025 + , 0x000026 + , 0x000027 + , 0x000028 + , 0x000029 + , 0x00002A + , 0x00002B + , 0x00002C + , 0x00002D + , 0x00002E + , 0x00002F + , 0x000030 + , 0x000031 + , 0x000032 + , 0x000033 + , 0x000034 + , 0x000035 + , 0x000036 + , 0x000037 + , 0x000038 + , 0x000039 + , 0x00003A + , 0x00003B + , 0x00003C + , 0x00003D + , 0x00003E + , 0x00003F + , 0x000040 + , 0x000041 + , 0x000042 + , 0x000043 + , 0x000044 + , 0x000045 + , 0x000046 + , 0x000047 + , 0x000048 + , 0x000049 + , 0x00004A + , 0x00004B + , 0x00004C + , 0x00004D + , 0x00004E + , 0x00004F + , 0x000050 + , 0x000051 + , 0x000052 + , 0x000053 + , 0x000054 + , 0x000055 + , 0x000056 + , 0x000057 + , 0x000058 + , 0x000059 + , 0x00005A + , 0x00005B + , 0x00005C + , 0x00005D + , 0x00005E + , 0x00005F + , 0x000060 + , 0x000061 + , 0x000062 + , 0x000063 + , 0x000064 + , 0x000065 + , 0x000066 + , 0x000067 + , 0x000068 + , 0x000069 + , 0x00006A + , 0x00006B + , 0x00006C + , 0x00006D + , 0x00006E + , 0x00006F + , 0x000070 + , 0x000071 + , 0x000072 + , 0x000073 + , 0x000074 + , 0x000075 + , 0x000076 + , 0x000077 + , 0x000078 + , 0x000079 + , 0x00007A + , 0x00007B + , 0x00007C + , 0x00007D + , 0x00007E + , 0x002985 + , 0x002986 + , 0x003002 + , 0x00300C + , 0x00300D + , 0x003001 + , 0x0030FB + , 0x0030F2 + , 0x0030A1 + , 0x0030A3 + , 0x0030A5 + , 0x0030A7 + , 0x0030A9 + , 0x0030E3 + , 0x0030E5 + , 0x0030E7 + , 0x0030C3 + , 0x0030FC + , 0x0030A2 + , 0x0030A4 + , 0x0030A6 + , 0x0030A8 + , 0x0030AA + , 0x0030AB + , 0x0030AD + , 0x0030AF + , 0x0030B1 + , 0x0030B3 + , 0x0030B5 + , 0x0030B7 + , 0x0030B9 + , 0x0030BB + , 0x0030BD + , 0x0030BF + , 0x0030C1 + , 0x0030C4 + , 0x0030C6 + , 0x0030C8 + , 0x0030CA + , 0x0030CB + , 0x0030CC + , 0x0030CD + , 0x0030CE + , 0x0030CF + , 0x0030D2 + , 0x0030D5 + , 0x0030D8 + , 0x0030DB + , 0x0030DE + , 0x0030DF + , 0x0030E0 + , 0x0030E1 + , 0x0030E2 + , 0x0030E4 + , 0x0030E6 + , 0x0030E8 + , 0x0030E9 + , 0x0030EA + , 0x0030EB + , 0x0030EC + , 0x0030ED + , 0x0030EF + , 0x0030F3 + , 0x003099 + , 0x00309A + , 0x003164 + , 0x003131 + , 0x003132 + , 0x003133 + , 0x003134 + , 0x003135 + , 0x003136 + , 0x003137 + , 0x003138 + , 0x003139 + , 0x00313A + , 0x00313B + , 0x00313C + , 0x00313D + , 0x00313E + , 0x00313F + , 0x003140 + , 0x003141 + , 0x003142 + , 0x003143 + , 0x003144 + , 0x003145 + , 0x003146 + , 0x003147 + , 0x003148 + , 0x003149 + , 0x00314A + , 0x00314B + , 0x00314C + , 0x00314D + , 0x00314E + , 0x00314F + , 0x003150 + , 0x003151 + , 0x003152 + , 0x003153 + , 0x003154 + , 0x003155 + , 0x003156 + , 0x003157 + , 0x003158 + , 0x003159 + , 0x00315A + , 0x00315B + , 0x00315C + , 0x00315D + , 0x00315E + , 0x00315F + , 0x003160 + , 0x003161 + , 0x003162 + , 0x003163 + , 0x0000A2 + , 0x0000A3 + , 0x0000AC + , 0x0000AF + , 0x0000A6 + , 0x0000A5 + , 0x0020A9 + , 0x002502 + , 0x002190 + , 0x002191 + , 0x002192 + , 0x002193 + , 0x0025A0 + , 0x0025CB + ] diff --git a/internal/tools/genIdnaBidi.py b/internal/tools/genIdnaBidi.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaBidi.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +""" +genIdnaBidi.py -- regenerate Text.IDNA2008.Internal.Bidi.Data +from the Unicode UCD's DerivedBidiClass.txt. + +DerivedBidiClass.txt lists Bidi_Class assignments for every assigned +codepoint plus a small set of @missing fall-throughs that fill in the +class for unassigned codepoints in specific blocks (e.g. Hebrew, Arabic +private-use ranges) plus a global @missing line giving Left_To_Right as +the universal default. + +Usage: + + python3 internal/tools/genIdnaBidi.py \\ + > internal/Text/IDNA2008/Internal/Bidi/Data.hs + +The output module exposes two parallel arrays: + + bidiRangeStarts :: ByteArray of Word32 -- sorted ascending + bidiRangeTags :: ByteArray of Word8 -- one tag per range + +Each range covers codepoints @[bidiRangeStarts[i], bidiRangeStarts[i+1] - 1]@, +with the final range running through @0x10FFFF@. The eleven Bidi classes +relevant to RFC 5893 (L, R, AL, AN, EN, ES, CS, ET, ON, BN, NSM) get +distinct tags 0..10; everything else (B, S, WS, LRE/RLE/PDF, LRO/RLO, +LRI/RLI/FSI/PDI) collapses into a single @Other@ tag (11), which the +parser treats as outside the rule set. +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# Eleven RFC 5893 classes, plus an "Other" bucket. +TAG_L = 0 +TAG_R = 1 +TAG_AL = 2 +TAG_AN = 3 +TAG_EN = 4 +TAG_ES = 5 +TAG_CS = 6 +TAG_ET = 7 +TAG_ON = 8 +TAG_BN = 9 +TAG_NSM = 10 +TAG_OTHER = 11 + +# Both abbreviations (used in regular lines) and full names (used in +# @missing lines) appear in DerivedBidiClass.txt. +CLASS_MAP = { + # The eleven we care about. + "L": TAG_L, "Left_To_Right": TAG_L, + "R": TAG_R, "Right_To_Left": TAG_R, + "AL": TAG_AL, "Arabic_Letter": TAG_AL, + "AN": TAG_AN, "Arabic_Number": TAG_AN, + "EN": TAG_EN, "European_Number": TAG_EN, + "ES": TAG_ES, "European_Separator": TAG_ES, + "CS": TAG_CS, "Common_Separator": TAG_CS, + "ET": TAG_ET, "European_Terminator": TAG_ET, + "ON": TAG_ON, "Other_Neutral": TAG_ON, + "BN": TAG_BN, "Boundary_Neutral": TAG_BN, + "NSM": TAG_NSM, "Nonspacing_Mark": TAG_NSM, + # Everything else collapses to Other. + "B": TAG_OTHER, "Paragraph_Separator": TAG_OTHER, + "S": TAG_OTHER, "Segment_Separator": TAG_OTHER, + "WS": TAG_OTHER, "White_Space": TAG_OTHER, + "LRE": TAG_OTHER, "Left_To_Right_Embedding": TAG_OTHER, + "RLE": TAG_OTHER, "Right_To_Left_Embedding": TAG_OTHER, + "PDF": TAG_OTHER, "Pop_Directional_Format": TAG_OTHER, + "LRO": TAG_OTHER, "Left_To_Right_Override": TAG_OTHER, + "RLO": TAG_OTHER, "Right_To_Left_Override": TAG_OTHER, + "LRI": TAG_OTHER, "Left_To_Right_Isolate": TAG_OTHER, + "RLI": TAG_OTHER, "Right_To_Left_Isolate": TAG_OTHER, + "FSI": TAG_OTHER, "First_Strong_Isolate": TAG_OTHER, + "PDI": TAG_OTHER, "Pop_Directional_Isolate": TAG_OTHER, +} + +TAG_NAMES = { + TAG_L: "L", + TAG_R: "R", + TAG_AL: "AL", + TAG_AN: "AN", + TAG_EN: "EN", + TAG_ES: "ES", + TAG_CS: "CS", + TAG_ET: "ET", + TAG_ON: "ON", + TAG_BN: "BN", + TAG_NSM: "NSM", + TAG_OTHER: "Other", +} + +CP_MAX = 0x10FFFF +ARRAY_LEN = CP_MAX + 1 + +# Patterns for lines we consume. +ASSIGN_RE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z_]+)\b") +MISSING_RE = re.compile( + r"^\s*#\s*@missing:\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z_]+)\b") + +STARTS_PER_LINE = 8 +TAGS_PER_LINE = 32 + + +def lookup_tag(name: str, where: str) -> int: + try: + return CLASS_MAP[name] + except KeyError as exc: + raise SystemExit(f"unknown Bidi class {name!r} in {where}") from exc + + +def parse(path: str) -> tuple[list[int], list[int]]: + """Return (starts, tags) for the resolved class of every + codepoint in @[0, CP_MAX]@. Adjacent codepoints with the same + class are coalesced into a single range. + + UAX #44 section 5.7.4 specifies that @missing fall-throughs + apply only to codepoints that lack an explicit assignment, and + where multiple @missing lines cover the same codepoint, the + later one wins (the file lists the global default first, + block-specific overrides after). We therefore collect both + kinds of line in source order and replay them in two phases: + @missing entries first (later overwriting earlier), then + explicit assignments (overwriting any @missing default). + """ + UNSET = -1 + classes = [UNSET] * ARRAY_LEN + missing: list[tuple[int, int, int]] = [] + assigns: list[tuple[int, int, int]] = [] + + with open(path, encoding="utf-8") as f: + for line_no, raw in enumerate(f, start=1): + line = raw.rstrip("\n") + + mm = MISSING_RE.match(line) + if mm: + lo = int(mm.group(1), 16) + hi = int(mm.group(2), 16) if mm.group(2) else lo + tag = lookup_tag(mm.group(3), + f"{path}:{line_no} (@missing)") + if lo < 0 or hi > CP_MAX or lo > hi: + raise SystemExit( + f"{path}:{line_no}: @missing range " + f"{lo:X}..{hi:X} out of bounds") + missing.append((lo, hi, tag)) + continue + + # Strip comments before testing for an assignment line. + payload = line.split("#", 1)[0].strip() + if not payload: + continue + am = ASSIGN_RE.match(payload) + if not am: + raise SystemExit( + f"{path}:{line_no}: cannot parse line: {raw!r}") + lo = int(am.group(1), 16) + hi = int(am.group(2), 16) if am.group(2) else lo + tag = lookup_tag(am.group(3), + f"{path}:{line_no}") + if lo < 0 or hi > CP_MAX or lo > hi: + raise SystemExit( + f"{path}:{line_no}: range {lo:X}..{hi:X} out of bounds") + assigns.append((lo, hi, tag)) + + # Phase 1: @missing in source order. Later overwrites earlier + # for codepoints both ranges cover. + for lo, hi, tag in missing: + for cp in range(lo, hi + 1): + classes[cp] = tag + + # Phase 2: explicit assignments, overwriting any @missing + # default that landed on the same codepoint. + for lo, hi, tag in assigns: + for cp in range(lo, hi + 1): + classes[cp] = tag + + # Sanity: full coverage required. The global @missing line + # (@missing: 0000..10FFFF; Left_To_Right) guarantees this, so + # an UNSET cp here means the input file is malformed. + for cp in range(ARRAY_LEN): + if classes[cp] == UNSET: + raise SystemExit( + f"codepoint U+{cp:04X} has no Bidi_Class assignment " + f"and no @missing fall-through") + + # Coalesce adjacent equal-class codepoints into ranges. + starts: list[int] = [] + tags: list[int] = [] + last = -1 + for cp in range(ARRAY_LEN): + c = classes[cp] + if c != last: + starts.append(cp) + tags.append(c) + last = c + return starts, tags + + +def emit_starts(name: str, starts: list[int], out) -> None: + n = len(starts) + p = out.write + p(f"{name} :: ByteArray\n") + p(f"{name} = PBA.byteArrayFromList\n") + for i in range(0, n, STARTS_PER_LINE): + chunk = starts[i:i + STARTS_PER_LINE] + if i == 0: + head = f"(0x{chunk[0]:06X} :: Word32)" + tail = ", ".join(f"0x{s:06X}" for s in chunk[1:]) + body = head + ((", " + tail) if tail else "") + p(f" [ {body}\n") + else: + body = ", ".join(f"0x{s:06X}" for s in chunk) + p(f" , {body}\n") + p(" ]\n\n") + + +def emit_tags(name: str, tags: list[int], out) -> None: + n = len(tags) + p = out.write + p(f"{name} :: ByteArray\n") + p(f"{name} = PBA.byteArrayFromList @Word8\n") + for i in range(0, n, TAGS_PER_LINE): + chunk = tags[i:i + TAGS_PER_LINE] + body = ", ".join(str(t) for t in chunk) + if i == 0: + p(f" [ {body}\n") + else: + p(f" , {body}\n") + p(" ]\n") + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Bidi.Data +-- Description : Codegen'd Bidi_Class table for the per-label RFC 5893 +-- check applied in "Text.IDNA2008.Internal.Parse". +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED FROM DerivedBidiClass.txt. Do not edit by +-- hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaBidi.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/extracted/DerivedBidiClass.txt +-- Aligned with: Unicode {version} +-- Range count: {n} +-- +-- The table covers every Unicode codepoint in @[0, 0x10FFFF]@ with no +-- gaps: each entry says \"from this start until the next entry's start +-- (or 0x10FFFF for the last), the resolved Bidi_Class is /T/\". The +-- eleven classes relevant to RFC 5893 -- L, R, AL, AN, EN, ES, CS, +-- ET, ON, BN, NSM -- get distinct tags 0..10; every other class (B, +-- S, WS, the embedding\\/override\\/isolate format characters) +-- collapses to a single @Other@ tag (11), which the parser treats as +-- a Bidi-rule failure for any label in which it appears. +module Text.IDNA2008.Internal.Bidi.Data + ( bidiRangeCount + , bidiRangeStarts + , bidiRangeTags + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + version, path = sys.argv[1], sys.argv[2] + starts, tags = parse(path) + if not starts: + sys.exit(f"{path}: no entries parsed") + + out = sys.stdout + out.write(HEADER.format(version=version, n=len(starts))) + + out.write("-- | Number of (start, class) entries.\n") + out.write("bidiRangeCount :: Int\n") + out.write(f"bidiRangeCount = {len(starts)}\n\n") + + out.write("-- | Range starts; sorted ascending.\n") + emit_starts("bidiRangeStarts", starts, out) + + out.write("-- | Class tags; parallel to 'bidiRangeStarts'.\n") + out.write("--\n") + out.write("-- * 0 -- L -- Left_To_Right\n") + out.write("-- * 1 -- R -- Right_To_Left\n") + out.write("-- * 2 -- AL -- Arabic_Letter\n") + out.write("-- * 3 -- AN -- Arabic_Number\n") + out.write("-- * 4 -- EN -- European_Number\n") + out.write("-- * 5 -- ES -- European_Separator\n") + out.write("-- * 6 -- CS -- Common_Separator\n") + out.write("-- * 7 -- ET -- European_Terminator\n") + out.write("-- * 8 -- ON -- Other_Neutral\n") + out.write("-- * 9 -- BN -- Boundary_Neutral\n") + out.write("-- * 10 -- NSM -- Nonspacing_Mark\n") + out.write("-- * 11 -- Other -- everything else (B, S, WS,\n") + out.write("-- embedding\\/override\\/isolate format\n") + out.write("-- characters); rejected by the\n") + out.write("-- per-label Bidi check.\n") + emit_tags("bidiRangeTags", tags, out) + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaCombining.py b/internal/tools/genIdnaCombining.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaCombining.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +genIdnaCombining.py -- regenerate Text.IDNA2008.Internal.Combining.Data +from the Unicode UCD's UnicodeData.txt. + +A combining mark is a codepoint whose General_Category is one of +Mn (Nonspacing_Mark), Mc (Spacing_Mark), or Me (Enclosing_Mark). +RFC 5891 section 4.2.3.2 forbids a U-label from starting with one; +RFC 5892's disposition table classifies many of those codepoints +(Mn/Mc) as PVALID, so the per-codepoint validator alone doesn't +catch the rule. The table this script emits is the dedicated +lookup the validator consults. + +Usage: + + python3 internal/tools/genIdnaCombining.py \\ + > internal/Text/IDNA2008/Internal/Combining/Data.hs + +The output module exposes two parallel arrays: + + combiningRangeStarts :: ByteArray of Word32 -- sorted ascending + combiningRangeTags :: ByteArray of Word8 -- one tag per range + +Each range covers codepoints @[combiningRangeStarts[i], +combiningRangeStarts[i+1] - 1]@, with the final range running through +@0x10FFFF@. Tag 0 means \"not a combining mark\", tag 1 means +\"combining mark\". Codepoints not explicitly assigned in +UnicodeData.txt -- the gaps in the file -- are treated as +non-combining (tag 0); none of the reserved/unassigned blocks +carry a combining gc by default. +""" +from __future__ import annotations + +import re +import sys + +TAG_NON = 0 +TAG_CM = 1 + +COMBINING_GC = {"Mn", "Mc", "Me"} + +CP_MAX = 0x10FFFF +ARRAY_LEN = CP_MAX + 1 + +STARTS_PER_LINE = 8 +TAGS_PER_LINE = 32 + +# Matches an ordinary UnicodeData.txt assignment line. The two +# fields we read are columns 1 (codepoint, hex) and 3 (gc). +ASSIGN_RE = re.compile( + r"^\s*([0-9A-Fa-f]+)\s*;[^;]*;([A-Za-z]+)\s*;") + +# UnicodeData.txt encodes large CJK / surrogate / PUA blocks as a +# pair of synthetic / rows that share a gc. We +# detect the pair and expand the range. +FIRST_RE = re.compile(r"<.*,\s*First>") +LAST_RE = re.compile(r"<.*,\s*Last>") + + +def parse(path: str) -> tuple[list[int], list[int]]: + """Walk UnicodeData.txt, mark every codepoint whose gc is in + @{Mn, Mc, Me}@, then coalesce equal-tag runs into ranges.""" + classes = [TAG_NON] * ARRAY_LEN + pending_first: tuple[int, str] | None = None + + with open(path, encoding="utf-8") as f: + for line_no, raw in enumerate(f, start=1): + line = raw.rstrip("\n") + if not line: + continue + fields = line.split(";") + if len(fields) < 4: + raise SystemExit( + f"{path}:{line_no}: too few fields: {raw!r}") + cp = int(fields[0], 16) + name = fields[1] + gc = fields[2] + if cp < 0 or cp > CP_MAX: + raise SystemExit( + f"{path}:{line_no}: codepoint U+{cp:04X} out of range") + + if FIRST_RE.search(name): + if pending_first is not None: + raise SystemExit( + f"{path}:{line_no}: nested entry") + pending_first = (cp, gc) + continue + if LAST_RE.search(name): + if pending_first is None: + raise SystemExit( + f"{path}:{line_no}: without ") + lo, lo_gc = pending_first + if lo_gc != gc: + raise SystemExit( + f"{path}:{line_no}: / gc mismatch " + f"({lo_gc!r} vs {gc!r})") + pending_first = None + if gc in COMBINING_GC: + for c in range(lo, cp + 1): + classes[c] = TAG_CM + continue + + if pending_first is not None: + raise SystemExit( + f"{path}:{line_no}: not followed by ") + + if gc in COMBINING_GC: + classes[cp] = TAG_CM + + if pending_first is not None: + raise SystemExit(f"{path}: dangling at end of file") + + # Coalesce equal-tag runs over the full codepoint space. + starts: list[int] = [] + tags: list[int] = [] + last = -1 + for cp in range(ARRAY_LEN): + c = classes[cp] + if c != last: + starts.append(cp) + tags.append(c) + last = c + return starts, tags + + +def emit_starts(name: str, starts: list[int], out) -> None: + n = len(starts) + p = out.write + p(f"{name} :: ByteArray\n") + p(f"{name} = PBA.byteArrayFromList\n") + for i in range(0, n, STARTS_PER_LINE): + chunk = starts[i:i + STARTS_PER_LINE] + if i == 0: + head = f"(0x{chunk[0]:06X} :: Word32)" + tail = ", ".join(f"0x{s:06X}" for s in chunk[1:]) + body = head + ((", " + tail) if tail else "") + p(f" [ {body}\n") + else: + body = ", ".join(f"0x{s:06X}" for s in chunk) + p(f" , {body}\n") + p(" ]\n\n") + + +def emit_tags(name: str, tags: list[int], out) -> None: + n = len(tags) + p = out.write + p(f"{name} :: ByteArray\n") + p(f"{name} = PBA.byteArrayFromList @Word8\n") + for i in range(0, n, TAGS_PER_LINE): + chunk = tags[i:i + TAGS_PER_LINE] + body = ", ".join(str(t) for t in chunk) + if i == 0: + p(f" [ {body}\n") + else: + p(f" , {body}\n") + p(" ]\n") + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Combining.Data +-- Description : Codegen'd combining-mark table (gc in @{{Mn, Mc, Me}}@) +-- for the RFC 5891 section 4.2.3.2 leading-mark check. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED FROM UnicodeData.txt. Do not edit by +-- hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaCombining.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/UnicodeData.txt +-- Aligned with: Unicode {version} +-- Range count: {n} +-- +-- The table covers every Unicode codepoint in @[0, 0x10FFFF]@ with no +-- gaps: each entry says \"from this start until the next entry's start +-- (or 0x10FFFF for the last), the resolved combining-mark status is +-- /T/\". Tag 0 means the codepoint is /not/ a combining mark; tag 1 +-- means it is (gc in @{{Mn, Mc, Me}}@). Codepoints absent from +-- UnicodeData.txt are treated as non-combining. +module Text.IDNA2008.Internal.Combining.Data + ( combiningRangeCount + , combiningRangeStarts + , combiningRangeTags + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + version, path = sys.argv[1], sys.argv[2] + starts, tags = parse(path) + if not starts: + sys.exit(f"{path}: no entries parsed") + + out = sys.stdout + out.write(HEADER.format(version=version, n=len(starts))) + + out.write("-- | Number of (start, tag) entries.\n") + out.write("combiningRangeCount :: Int\n") + out.write(f"combiningRangeCount = {len(starts)}\n\n") + + out.write("-- | Range starts; sorted ascending.\n") + emit_starts("combiningRangeStarts", starts, out) + + out.write("-- | Combining-mark tags; parallel to\n") + out.write("-- 'combiningRangeStarts'.\n") + out.write("--\n") + out.write("-- * 0 -- not a combining mark\n") + out.write("-- * 1 -- combining mark (gc in @{Mn, Mc, Me}@)\n") + emit_tags("combiningRangeTags", tags, out) + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaDerive.py b/internal/tools/genIdnaDerive.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaDerive.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +""" +genIdnaDerive.py -- compute the IDNA2008 disposition table from UCD +inputs by running the RFC 5892 Section 3 derivation locally. + +Output is a CSV with the same shape as IANA's idna-tables-properties.csv +(columns "Codepoint" and "Property"), so the existing genIdnaProperty.py +consumes it unchanged. + + python3 genIdnaDerive.py \\ + \\ + \\ + \\ + \\ + \\ + > idna-tables-properties.csv + +The first argument is recorded in stderr for provenance. + +Verification: at Unicode 12.0.0 the output matches IANA's published +https://www.iana.org/assignments/idna-tables-12.0.0/ +idna-tables-properties.csv byte-for-byte when both files are +reduced to the (Codepoint, Property) projection. Once that round +trip passes, the same derivation produces a valid disposition table +at any Unicode version, and the IANA dependency is gone. + +The "Unstable" check (RFC 5892 section 2.6) consults the target version's +@NFKC_CF@ (NFKC_Casefold) mapping from +DerivedNormalizationProps.txt directly, rather than recomputing +NFKC via Python's stdlib. This avoids a silent misclassification +for codepoints assigned later than the Python interpreter's bundled +Unicode data version. +""" +from __future__ import annotations + +import csv +import re +import sys +from typing import Iterable + +#----------------------------------------------------------------------- +# Static tables lifted verbatim from RFC 5892 +#----------------------------------------------------------------------- + +# Section 2.6: Exceptions. Codepoint -> disposition; tested before +# any computed rule. Includes the seven CONTEXTO codepoints and the +# eight CONTEXTJ-related explicit overrides plus a handful of +# letter-shape overrides (final sigma, sharp s, ...). +EXCEPTIONS = { + # PVALID + 0x00DF: "PVALID", # LATIN SMALL LETTER SHARP S + 0x03C2: "PVALID", # GREEK SMALL LETTER FINAL SIGMA + 0x06FD: "PVALID", # ARABIC SIGN SINDHI AMPERSAND + 0x06FE: "PVALID", # ARABIC SIGN SINDHI POSTPOSITION MEN + 0x0F0B: "PVALID", # TIBETAN MARK INTERSYLLABIC TSHEG + 0x3007: "PVALID", # IDEOGRAPHIC NUMBER ZERO + + # CONTEXTO + 0x00B7: "CONTEXTO", # MIDDLE DOT (A.3) + 0x0375: "CONTEXTO", # GREEK LOWER NUMERAL SIGN (A.4) + 0x05F3: "CONTEXTO", # HEBREW PUNCTUATION GERESH (A.5) + 0x05F4: "CONTEXTO", # HEBREW PUNCTUATION GERSHAYIM (A.6) + 0x30FB: "CONTEXTO", # KATAKANA MIDDLE DOT (A.9) + + # DISALLOWED + 0x0640: "DISALLOWED", # ARABIC TATWEEL + 0x07FA: "DISALLOWED", # NKO LAJANYALAN + 0x302E: "DISALLOWED", # HANGUL SINGLE DOT TONE MARK + 0x302F: "DISALLOWED", # HANGUL DOUBLE DOT TONE MARK + 0x3031: "DISALLOWED", # VERTICAL KANA REPEAT MARK + 0x3032: "DISALLOWED", # VERTICAL KANA REPEAT WITH VOICED SOUND MARK + 0x3033: "DISALLOWED", # VERTICAL KANA REPEAT MARK UPPER HALF + 0x3034: "DISALLOWED", # VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF + 0x3035: "DISALLOWED", # VERTICAL KANA REPEAT MARK LOWER HALF + 0x303B: "DISALLOWED", # VERTICAL IDEOGRAPHIC ITERATION MARK +} + +# Arabic-Indic digits (A.8) and Extended Arabic-Indic digits (A.9) +# are CONTEXTO via section 2.6 too. Codified as ranges for brevity. +for cp in range(0x0660, 0x066A): + EXCEPTIONS[cp] = "CONTEXTO" +for cp in range(0x06F0, 0x06FA): + EXCEPTIONS[cp] = "CONTEXTO" + +# Section 2.7: BackwardCompatible. Currently empty per RFC 5892; +# the slot exists so future codepoints can be pinned to a specific +# disposition that overrides their derived class. +BACKWARD_COMPATIBLE: dict[int, str] = {} + +# Section 2.6 also pins the two Join-Control codepoints to CONTEXTJ. +EXCEPTIONS[0x200C] = "CONTEXTJ" # ZERO WIDTH NON-JOINER +EXCEPTIONS[0x200D] = "CONTEXTJ" # ZERO WIDTH JOINER + +#----------------------------------------------------------------------- +# UCD input parsing +#----------------------------------------------------------------------- + +# UnicodeData.txt is one record per line, semicolon-separated, 15 +# fields. We need fields 2 (General_Category) and 5 (decomp). +def parse_unicode_data(path: str) -> tuple[ + dict[int, str], # codepoint -> General_Category + dict[int, bool], # codepoint -> has compatibility decomp +]: + gc: dict[int, str] = {} + has_compat: dict[int, bool] = {} + range_first: int | None = None + range_first_gc: str = "" + with open(path, encoding="utf-8") as f: + for line in f: + fields = line.rstrip("\n").split(";") + if len(fields) < 6: + continue + cp = int(fields[0], 16) + name = fields[1] + cat = fields[2] + decomp = fields[5] + + # Compatibility decomposition: starts with "<...>" + compat = decomp.startswith("<") and decomp != "" + + if name.endswith(", First>"): + range_first = cp + range_first_gc = cat + continue + if name.endswith(", Last>") and range_first is not None: + for r in range(range_first, cp + 1): + gc[r] = range_first_gc + # Range entries have no decomposition. + has_compat[r] = False + range_first = None + continue + + gc[cp] = cat + has_compat[cp] = compat + return gc, has_compat + + +# DerivedCoreProperties.txt and PropList.txt share a syntax: +# start[..end] ; PropertyName [# comment] +PROP_LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z0-9_]+)" +) + + +def parse_property_file(path: str, wanted: set[str]) -> dict[str, set[int]]: + """Return {property -> {codepoint}} for each property in 'wanted'.""" + out: dict[str, set[int]] = {p: set() for p in wanted} + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = PROP_LINE.match(line) + if not m: + continue + prop = m.group(3) + if prop not in wanted: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + out[prop].update(range(start, end + 1)) + return out + + +def parse_hangul_syllable_type(path: str) -> dict[int, str]: + """codepoint -> Hangul_Syllable_Type (L, V, T, LV, LVT, NA).""" + out: dict[int, str] = {} + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = PROP_LINE.match(line) + if not m: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + tag = m.group(3) + for cp in range(start, end + 1): + out[cp] = tag + return out + + +# DerivedNormalizationProps.txt NFKC_CF lines have one of two shapes: +# +# start[..end] ; NFKC_CF; # mapping is empty +# start[..end] ; NFKC_CF; # mapping is a sequence +# +# A codepoint is "Unstable" iff it has an explicit NFKC_CF entry whose +# mapping target differs from the source (single-cp identity is implicit +# for codepoints with no entry). +NFKC_CF_LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*NFKC_CF\s*;\s*([^#\n]*)" +) + + +def parse_nfkc_cf_unstable(path: str) -> set[int]: + """Return the set of codepoints whose NFKC_CF mapping differs + from the codepoint itself (i.e. NFKC + casefold + NFKC changes + them). These are exactly the IDNA Unstable codepoints.""" + out: set[int] = set() + with open(path, encoding="utf-8") as f: + for raw in f: + m = NFKC_CF_LINE.match(raw) + if not m: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + mapping_str = m.group(3).strip() + for cp in range(start, end + 1): + # An empty mapping ("") means "drop this codepoint", + # which is a change. A single-codepoint mapping + # equal to cp itself would be an identity (rare in + # the file -- usually identity is implicit), so check + # for that explicitly. + if mapping_str == "": + out.add(cp) + else: + target = [int(t, 16) for t in mapping_str.split()] + if target != [cp]: + out.add(cp) + return out + + +#----------------------------------------------------------------------- +# Section 2 categories -- predicates over a single codepoint. +# +# Each is a small function that assumes the parsed UCD tables have +# been bound at module scope (set up in main()). +#----------------------------------------------------------------------- + +GC: dict[int, str] = {} # General_Category +HAS_COMPAT: dict[int, bool] = {} # has compatibility decomposition +DEFAULT_IGNORABLE: set[int] = set() # Default_Ignorable_Code_Point +NONCHARACTER: set[int] = set() # Noncharacter_Code_Point +JOIN_CONTROL: set[int] = set() # Join_Control +WHITE_SPACE: set[int] = set() # White_Space +HANGUL_TYPE: dict[int, str] = {} # Hangul_Syllable_Type +UNSTABLE: set[int] = set() # NFKC_CF != cp + + +def gc(cp: int) -> str: + """General_Category of cp; "Cn" for codepoints not in + UnicodeData.txt (the Cn default applies to all unassigned and + reserved codepoints).""" + return GC.get(cp, "Cn") + + +# Section 2.7: Unassigned. +# +# GC == Cn AND Noncharacter_Code_Point != true +# +# Default_Ignorable codepoints that happen to be Cn are still +# Unassigned per the spec; they get DISALLOWED later via +# IgnorableProperties only if they were assigned (GC != Cn). +def is_unassigned(cp: int) -> bool: + return gc(cp) == "Cn" and cp not in NONCHARACTER + + +# Section 2.9: LDH. Lowercase letters + digits + hyphen ONLY; +# uppercase A-Z is NOT LDH for IDNA -- it falls through to the +# Unstable check (NFKC + casefold collapses A-Z onto a-z) and +# ends up DISALLOWED. +def is_ldh(cp: int) -> bool: + return ( + (0x0061 <= cp <= 0x007A) # a-z + or (0x0030 <= cp <= 0x0039) # 0-9 + or cp == 0x002D # '-' + ) + + +# Section 2.5/2.10: JoinControl. Covered explicitly via Exceptions +# above; this predicate exists for completeness and is used only as +# a fallback if Exceptions has been redacted. +def is_join_control(cp: int) -> bool: + return cp in JOIN_CONTROL + + +# Section 2.6 of RFC 5892 (also termed "Unstable" in some drafts): +# the codepoint changes under NFKC + case-fold + NFKC. Read +# directly from the target version's NFKC_CF (NFKC_Casefold) +# property in DerivedNormalizationProps.txt -- equivalent to the +# (NFKC . casefold . NFKC) composition by definition, and free of +# the version skew that would arise from running Python's stdlib +# normalisation against a Unicode version it doesn't know. +def is_unstable(cp: int) -> bool: + return cp in UNSTABLE + + +# Section 2.11: IgnorableProperties. Default_Ignorable or Noncharacter. +def is_ignorable_properties(cp: int) -> bool: + return cp in DEFAULT_IGNORABLE or cp in NONCHARACTER + + +# Section 2.12: IgnorableBlocks. Codepoints in these specific +# Unicode blocks are blanket-DISALLOWED regardless of GC; the block +# boundaries don't shift across Unicode versions, so the literal +# ranges are stable. Per RFC 5892 section 2.12: +# +# Combining Diacritical Marks for Symbols +# Musical Symbols +# Ancient Greek Musical Notation +IGNORABLE_BLOCKS: list[tuple[int, int]] = [ + (0x20D0, 0x20FF), # Combining Diacritical Marks for Symbols + (0x1D100, 0x1D1FF), # Musical Symbols + (0x1D200, 0x1D24F), # Ancient Greek Musical Notation +] + + +def is_ignorable_block(cp: int) -> bool: + for lo, hi in IGNORABLE_BLOCKS: + if lo <= cp <= hi: + return True + return False + + +# Section 2.13: OldHangulJamo. Hangul jamo that aren't part of the +# closed precomposed-syllable cluster -- these are the historical +# / archaic jamo plus the modern jamo not covered by the LV / LVT +# precomposition. +def is_old_hangul_jamo(cp: int) -> bool: + return HANGUL_TYPE.get(cp, "NA") in ("L", "V", "T") + + +# Section 2.14: HasCompat. +def is_has_compat(cp: int) -> bool: + return HAS_COMPAT.get(cp, False) + + +# Section 2.15: LetterDigits. General_Category in {Ll, Lu, Lo, Lm, +# Mn, Mc, Nd}. +LETTER_DIGITS_GC = {"Ll", "Lu", "Lo", "Lm", "Mn", "Mc", "Nd"} + + +def is_letter_digits(cp: int) -> bool: + return gc(cp) in LETTER_DIGITS_GC + + +#----------------------------------------------------------------------- +# Section 3: derivation cascade +#----------------------------------------------------------------------- + +def derive(cp: int) -> str: + if cp in EXCEPTIONS: + return EXCEPTIONS[cp] + if cp in BACKWARD_COMPATIBLE: + return BACKWARD_COMPATIBLE[cp] + if is_unassigned(cp): + return "UNASSIGNED" + if is_ldh(cp): + return "PVALID" + if is_join_control(cp): + return "CONTEXTJ" + if is_unstable(cp): + return "DISALLOWED" + if is_ignorable_properties(cp): + return "DISALLOWED" + if is_ignorable_block(cp): + return "DISALLOWED" + if is_old_hangul_jamo(cp): + return "DISALLOWED" + if is_has_compat(cp): + return "DISALLOWED" + if is_letter_digits(cp): + return "PVALID" + return "DISALLOWED" + + +#----------------------------------------------------------------------- +# Range-coalesce and CSV emit +#----------------------------------------------------------------------- + +def coalesce_runs(dispositions: list[str]) -> list[tuple[int, int, str]]: + """Given dispositions[cp] for cp in [0, 0x10FFFF], return a list + of (start, end, prop) runs sharing a single disposition.""" + out: list[tuple[int, int, str]] = [] + if not dispositions: + return out + run_start = 0 + run_prop = dispositions[0] + for cp in range(1, len(dispositions)): + if dispositions[cp] != run_prop: + out.append((run_start, cp - 1, run_prop)) + run_start = cp + run_prop = dispositions[cp] + out.append((run_start, len(dispositions) - 1, run_prop)) + return out + + +def fmt_codepoint(start: int, end: int) -> str: + """IANA-style codepoint column: bare hex for singleton, lo-hi + for ranges.""" + if start == end: + return f"{start:04X}" + return f"{start:04X}-{end:04X}" + + +def emit_csv(version: str, runs: list[tuple[int, int, str]], + out=sys.stdout) -> None: + # Header row only; no leading '#' comments -- csv.DictReader + # treats the first line as the header, and a comment line there + # causes downstream consumers to choke. Provenance / version + # info goes to stderr in main() instead. + w = csv.writer(out) + w.writerow(["Codepoint", "Property", "Description"]) + for start, end, prop in runs: + w.writerow([fmt_codepoint(start, end), prop, ""]) + + +#----------------------------------------------------------------------- +# main +#----------------------------------------------------------------------- + +def main() -> None: + if len(sys.argv) != 7: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + + global GC, HAS_COMPAT, DEFAULT_IGNORABLE, NONCHARACTER + global JOIN_CONTROL, WHITE_SPACE, HANGUL_TYPE, UNSTABLE + + GC, HAS_COMPAT = parse_unicode_data(sys.argv[2]) + + derived = parse_property_file(sys.argv[3], {"Default_Ignorable_Code_Point"}) + DEFAULT_IGNORABLE = derived["Default_Ignorable_Code_Point"] + + proplist = parse_property_file(sys.argv[4], { + "Noncharacter_Code_Point", + "Join_Control", + "White_Space", + }) + NONCHARACTER = proplist["Noncharacter_Code_Point"] + JOIN_CONTROL = proplist["Join_Control"] + WHITE_SPACE = proplist["White_Space"] + + HANGUL_TYPE = parse_hangul_syllable_type(sys.argv[5]) + UNSTABLE = parse_nfkc_cf_unstable(sys.argv[6]) + + sys.stderr.write(f"genIdnaDerive: target Unicode {version}\n") + + dispositions = [derive(cp) for cp in range(0, 0x110000)] + runs = coalesce_runs(dispositions) + emit_csv(version, runs) + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaEmoji.py b/internal/tools/genIdnaEmoji.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaEmoji.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.Emoji.Data from the Unicode UCD +emoji-data.txt extracted property file and the UTS #46 +IdnaMappingTable.txt. + +Usage: + python3 internal/tools/genIdnaEmoji.py \\ + + +Reads two inputs: + + 1. The UCD emoji-data.txt property file. Only entries with property + name "Emoji" are kept; the related properties + (Emoji_Presentation, Emoji_Modifier, Emoji_Modifier_Base, + Emoji_Component, Extended_Pictographic) are ignored. A codepoint + is "an emoji" for IDNA-relaxation purposes iff Emoji=Yes. + + 2. The UTS #46 IdnaMappingTable.txt. We extract codepoints with + status @mapped@ and intersect with the Emoji=Yes set above to + produce the "mapped-emoji" subset. These codepoints are + Emoji=Yes but UTS #46 also folds them to a different codepoint + sequence -- typically a CJK ideograph or an ASCII letter -- and + so resolve ambiguously across the ecosystem: browsers (Safari) + apply the fold and reach the target; tools like libidn2 in + non-transitional mode reject them; this library admits them + as-is under EMOJIOK. Because the mapped form and the admit-as-is + form may resolve to separately registered domains under different + operators (see U+1F238 (1F238).ws vs. U+7533 (7533).ws for + a real-world case), EMOJIOK in this library specifically + EXCLUDES this subset: it admits Emoji=Yes codepoints that are + not UTS #46-mapped, and rejects (as DisallowedCodepoint) those + that are. + +Source files: + + Unicode 12.0 emoji: https://www.unicode.org/Public/emoji/12.0/emoji-data.txt + Unicode 13.0+ emoji: https://www.unicode.org/Public//ucd/emoji/emoji-data.txt + Unicode 17.0+ uts46: https://www.unicode.org/Public//idna/IdnaMappingTable.txt + Unicode <=16 uts46: https://www.unicode.org/Public/idna//IdnaMappingTable.txt +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# Property-file line: hex-cp[..hex-cp] ; property-name [# comment] +LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z0-9_]+)" +) + + +def parse_emoji(path: str) -> list[tuple[int, int]]: + """Return [(start, end)] from `path` for entries with property + name == "Emoji".""" + out: list[tuple[int, int]] = [] + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = LINE.match(line) + if not m: + continue + if m.group(3) != "Emoji": + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + out.append((start, end)) + return out + + +def parse_uts46_mapped(path: str) -> set[int]: + """Return the set of codepoints whose UTS #46 status is exactly + `mapped` in `IdnaMappingTable.txt`. Other statuses (valid, + disallowed, ignored, deviation) are not returned.""" + out: set[int] = set() + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = [p.strip() for p in line.split(";")] + if len(parts) < 2 or parts[1] != "mapped": + continue + rng = parts[0] + if ".." in rng: + a, b = rng.split("..") + cps = range(int(a, 16), int(b, 16) + 1) + else: + cps = [int(rng, 16)] + out.update(cps) + return out + + +def mapped_emoji_pairs( + emoji_pairs: list[tuple[int, int]], + mapped_cps: set[int], +) -> list[tuple[int, int]]: + """Intersect the Emoji=Yes range list with the mapped-codepoint + set, returning a coalesced (start, end) list of codepoints in + both.""" + matches: list[int] = sorted( + cp for s, e in emoji_pairs for cp in range(s, e + 1) if cp in mapped_cps + ) + return coalesce_pairs([(cp, cp) for cp in matches]) + + +def coalesce_pairs(pairs: Iterable[tuple[int, int]]) -> list[tuple[int, int]]: + """Sort and merge adjacent (start, end) pairs.""" + pairs = sorted(pairs, key=lambda p: p[0]) + out: list[tuple[int, int]] = [] + for s, e in pairs: + if out and out[-1][1] + 1 >= s: + ps, pe = out[-1] + out[-1] = (ps, max(pe, e)) + else: + out.append((s, e)) + return out + + +def cp_hex(cp: int) -> str: + """Format a codepoint as 0xNNNNNN with at least 6 hex digits.""" + return f"0x{cp:06X}" + + +def emit_word32_list(name: str, items: list[int]) -> str: + """Render @name :: ByteArray ; name = PBA.byteArrayFromList [...]@ + as a formatted Haskell expression with one element per line. The + first element carries the @:: Word32@ annotation.""" + lines = [f"{name} :: ByteArray"] + if not items: + # Empty-list literals need an explicit element type to keep + # 'PBA.byteArrayFromList' unambiguous. + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word32])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + rendered = [] + for i, v in enumerate(items): + if i == 0: + rendered.append(f"({cp_hex(v)} :: Word32)") + else: + rendered.append(cp_hex(v)) + body = "\n , ".join(rendered) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Emoji.Data +-- Description : Compact codepoint range tables for the Unicode +-- @Emoji@ property and its UTS #46-mapped subset. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaEmoji.py \\\\ +-- +-- +-- Source (Unicode 12.0): https://www.unicode.org/Public/emoji/12.0/emoji-data.txt +-- Source (Unicode 13.0+): https://www.unicode.org/Public/{version}/ucd/emoji/emoji-data.txt +-- Source (Unicode 17.0+, idna): https://www.unicode.org/Public/{version}/idna/IdnaMappingTable.txt +-- Aligned with: Unicode {version} +-- Emoji ranges: {n_emoji} +-- Mapped-emoji ranges: {n_mapped} +-- +-- Two range tables, both with the same @Word32@ start\\/end layout +-- (sorted by start, abutting ranges coalesced): +-- +-- * 'emojiRanges' -- codepoints with Unicode property @Emoji=Yes@. +-- +-- * 'mappedEmojiRanges' -- the subset of 'emojiRanges' whose +-- UTS #46 status is @mapped@; cross-tool ambiguous. +-- +-- Both tables feed the @EMOJIOK@ relaxation in +-- "Text.IDNA2008.Internal.Parse"; see the @EMOJIOK@ pattern in +-- "Text.IDNA2008.Internal.Flags" for the policy rationale. +module Text.IDNA2008.Internal.Emoji.Data + ( emojiRangeCount + , emojiRanges + , mappedEmojiRangeCount + , mappedEmojiRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +""" + + +def main() -> None: + if len(sys.argv) != 4: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + emoji_path = sys.argv[2] + uts46_path = sys.argv[3] + + emoji_pairs = coalesce_pairs(parse_emoji(emoji_path)) + mapped_cps = parse_uts46_mapped(uts46_path) + mapped_pairs = mapped_emoji_pairs(emoji_pairs, mapped_cps) + + flat: list[int] = [] + for s, e in emoji_pairs: + flat.append(s); flat.append(e) + mapped_flat: list[int] = [] + for s, e in mapped_pairs: + mapped_flat.append(s); mapped_flat.append(e) + + out = sys.stdout + out.write(HEADER.format(version=version, n_emoji=len(emoji_pairs), + n_mapped=len(mapped_pairs))) + + out.write("-- | Number of (start, end) range pairs in 'emojiRanges'.\n") + out.write("emojiRangeCount :: Int\n") + out.write(f"emojiRangeCount = {len(emoji_pairs)}\n\n") + + out.write("-- | Codepoints with Unicode @Emoji=Yes@, alternating\n") + out.write("-- start\\/end.\n") + out.write(emit_word32_list("emojiRanges", flat)) + out.write("\n\n") + + out.write("-- | Number of (start, end) range pairs in\n") + out.write("-- 'mappedEmojiRanges'.\n") + out.write("mappedEmojiRangeCount :: Int\n") + out.write(f"mappedEmojiRangeCount = {len(mapped_pairs)}\n\n") + + out.write("-- | Subset of 'emojiRanges' whose UTS #46 status is\n") + out.write("-- @mapped@, alternating start\\/end. Excluded from\n") + out.write("-- the @EMOJIOK@ relaxation; see module haddock for\n") + out.write("-- the rationale.\n") + out.write(emit_word32_list("mappedEmojiRanges", mapped_flat)) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaJoining.py b/internal/tools/genIdnaJoining.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaJoining.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.Joining.Data from the Unicode UCD +extracted property files. + +Usage: + python3 internal/tools/genIdnaJoining.py \\ + \\ + + +Reads the two UCD-derived property files and writes a complete Haskell +module to stdout. Only the four Joining_Type values consulted by RFC +5892 Appendix A.1 (L, R, D, T) are kept; U (Non_Joining) and C +(Join_Causing) entries are dropped: the table-miss path in +"Text.IDNA2008.Internal.Joining" matches their "stop" semantics in +the rule, so storing them buys nothing. + +Only codepoints with Canonical_Combining_Class = 9 (Virama) are kept +from the combining-class file. + +Source files are the Unicode 12 (or later) UCD extracts: + + https://www.unicode.org/Public/12.0.0/ucd/extracted/DerivedJoiningType.txt + https://www.unicode.org/Public/12.0.0/ucd/extracted/DerivedCombiningClass.txt +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# Joining_Type tags must agree with jtTagL/jtTagR/jtTagD/jtTagT in +# Text.IDNA2008.Internal.Joining.Data. +JT_TAG = {"L": 1, "R": 2, "D": 3, "T": 4} + +# Property-file line: hex-cp[..hex-cp] ; property-name [# comment] +LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z0-9_]+)" +) + + +def parse_lines(path: str, wanted: set[str]) -> list[tuple[int, int, str]]: + """Return [(start, end, prop)] from `path`, keeping only entries + whose property name is in `wanted`. Lines are parsed individually, + not coalesced.""" + out: list[tuple[int, int, str]] = [] + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = LINE.match(line) + if not m: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + prop = m.group(3) + if prop in wanted: + out.append((start, end, prop)) + return out + + +def coalesce(triples: Iterable[tuple[int, int, str]]) -> list[tuple[int, int, str]]: + """Sort by start and merge ranges that abut and share a property.""" + triples = sorted(triples, key=lambda t: t[0]) + out: list[tuple[int, int, str]] = [] + for s, e, p in triples: + if out and out[-1][2] == p and out[-1][1] + 1 == s: + ps, _, _ = out[-1] + out[-1] = (ps, e, p) + else: + out.append((s, e, p)) + return out + + +def coalesce_pairs(pairs: Iterable[tuple[int, int]]) -> list[tuple[int, int]]: + """Sort and merge adjacent (start, end) pairs.""" + pairs = sorted(pairs, key=lambda p: p[0]) + out: list[tuple[int, int]] = [] + for s, e in pairs: + if out and out[-1][1] + 1 == s: + ps, _ = out[-1] + out[-1] = (ps, e) + else: + out.append((s, e)) + return out + + +def cp_hex(cp: int) -> str: + """Format a codepoint as 0xNNNNNN with at least 6 hex digits.""" + return f"0x{cp:06X}" + + +def emit_word32_list( + name: str, + typ: str, + items: list[int], + *, + line_comment: list[str] | None = None, +) -> str: + """Render @name :: typ ; name = PBA.byteArrayFromList [...]@ as a + formatted Haskell expression with one element per line. The first + element carries the @:: Word32@ annotation.""" + lines = [f"{name} :: {typ}", f"{name} = PBA.byteArrayFromList"] + if not items: + lines.append(" [ ]") + return "\n".join(lines) + rendered = [] + for i, v in enumerate(items): + if i == 0: + cell = f"({cp_hex(v)} :: Word32)" + else: + cell = cp_hex(v) + comment = "" + if line_comment is not None: + comment = f" -- {line_comment[i]}" + rendered.append(f"{cell}{comment}") + body = "\n , ".join(rendered) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +def emit_word8_list(name: str, items: list[int]) -> str: + """Same shape as 'emit_word32_list' but for Word8 tag arrays.""" + lines = [f"{name} :: ByteArray", f"{name} = PBA.byteArrayFromList"] + if not items: + lines.append(" [ ]") + return "\n".join(lines) + rendered = [] + for i, v in enumerate(items): + if i == 0: + cell = f"({v} :: Word8)" + else: + cell = str(v) + rendered.append(cell) + body = "\n , ".join(rendered) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Joining.Data +-- Description : Compact codepoint range tables for the Unicode +-- Joining_Type and Virama (Canonical_Combining_Class +-- = 9) properties consulted by the CONTEXTJ +-- contextual rules. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaJoining.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/extracted/ +-- Aligned with: Unicode {version} +-- Joining_Type ranges: {n_jt} +-- Virama ranges: {n_v} +-- +-- The four Joining_Type values relevant to RFC 5892 A.1 are encoded +-- as a 'Word8' tag per range: +-- +-- * 1 = L (Left_Joining) +-- * 2 = R (Right_Joining) +-- * 3 = D (Dual_Joining) +-- * 4 = T (Transparent) +-- +-- Joining_Type=C (Join_Causing, including ZWJ\\/ZWNJ themselves and +-- Arabic Tatweel @U+0640@) and Joining_Type=U (Non_Joining) are not +-- encoded: at any position a codepoint with one of those types is a +-- \\"stop\\" for the regex. +module Text.IDNA2008.Internal.Joining.Data + ( -- * Joining_Type + jtRangeCount + , jtRangeStarts + , jtRangeEnds + , jtRangeTags + + -- * Joining_Type tag values + , jtTagL + , jtTagR + , jtTagD + , jtTagT + + -- * Virama (CCC = 9) + , viramaRangeCount + , viramaRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- Joining_Type +---------------------------------------------------------------------- + +-- | Tag value for Joining_Type=L (Left_Joining). +jtTagL :: Word8 +jtTagL = 1 + +-- | Tag value for Joining_Type=R (Right_Joining). +jtTagR :: Word8 +jtTagR = 2 + +-- | Tag value for Joining_Type=D (Dual_Joining). +jtTagD :: Word8 +jtTagD = 3 + +-- | Tag value for Joining_Type=T (Transparent). +jtTagT :: Word8 +jtTagT = 4 + +""" + + +VIRAMA_HEADING = """\ + +---------------------------------------------------------------------- +-- Virama (Canonical_Combining_Class = 9) +---------------------------------------------------------------------- + +""" + + +def main() -> None: + if len(sys.argv) != 4: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + jt_path = sys.argv[2] + ccc_path = sys.argv[3] + + # ----- Joining_Type -------------------------------------------------- + jt_long = { + "Left_Joining": "L", + "Right_Joining": "R", + "Dual_Joining": "D", + "Transparent": "T", + } + jt_short = set(jt_long.values()) + + # Some UCD distributions use long names ("Right_Joining"), others use + # the short codes ("R"). Accept either. + raw = parse_lines(jt_path, set(jt_long.keys()) | jt_short) + normalised = [] + for s, e, p in raw: + if p in jt_long: + p = jt_long[p] + normalised.append((s, e, p)) + + jt = coalesce(normalised) + starts = [s for (s, _e, _p) in jt] + ends = [e for (_s, e, _p) in jt] + tags = [JT_TAG[p] for (_s, _e, p) in jt] + + # ----- Virama (CCC = 9) --------------------------------------------- + raw_ccc = parse_lines(ccc_path, {"9"}) + v_pairs = coalesce_pairs([(s, e) for (s, e, _p) in raw_ccc]) + + virama_flat = [] + for s, e in v_pairs: + virama_flat.append(s) + virama_flat.append(e) + + # ----- Emit ---------------------------------------------------------- + out = sys.stdout + out.write(HEADER.format(version=version, n_jt=len(jt), n_v=len(v_pairs))) + + out.write(f"-- | Number of joining-type ranges.\n") + out.write(f"jtRangeCount :: Int\n") + out.write(f"jtRangeCount = {len(jt)}\n\n") + + out.write("-- | Inclusive range starts, sorted ascending.\n") + out.write(emit_word32_list("jtRangeStarts", "ByteArray", starts)) + out.write("\n\n") + + out.write("-- | Inclusive range ends, parallel to 'jtRangeStarts'.\n") + out.write(emit_word32_list("jtRangeEnds", "ByteArray", ends)) + out.write("\n\n") + + out.write("-- | Joining_Type tag for each range, parallel to\n") + out.write("-- 'jtRangeStarts' and 'jtRangeEnds'.\n") + out.write(emit_word8_list("jtRangeTags", tags)) + out.write("\n") + + out.write(VIRAMA_HEADING) + out.write(f"-- | Number of (start, end) range pairs in 'viramaRanges'.\n") + out.write(f"viramaRangeCount :: Int\n") + out.write(f"viramaRangeCount = {len(v_pairs)}\n\n") + + out.write("-- | Codepoints with Canonical_Combining_Class = 9\n") + out.write("-- (Virama), alternating start\\/end.\n") + out.write(emit_word32_list("viramaRanges", "ByteArray", virama_flat)) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaNFC.py b/internal/tools/genIdnaNFC.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaNFC.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.NFC.Data from the Unicode UCD +file DerivedNormalizationProps.txt. + +Usage: + python3 internal/tools/genIdnaNFC.py \\ + + +Reads NFC_Quick_Check entries (the lines tagged @NFC_QC@ with +values @N@ or @M@) and writes a complete Haskell module to +stdout with two parallel range tables: codepoints whose +@NFC_QC = No@ and those whose @NFC_QC = Maybe@. Codepoints +not covered by either are implicitly @Yes@. + +The source file lives at: + + https://www.unicode.org/Public//ucd/DerivedNormalizationProps.txt +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# Property-file line shape: +# start[..end] ; NFC_QC; [# comment] +# We only care about NFC_QC; everything else is skipped. +LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*NFC_QC\s*;\s*([NMY])\b" +) + + +def parse_lines(path: str, wanted_values: set[str]) -> list[tuple[int, int, str]]: + """Return [(start, end, value)] for every NFC_QC line whose + third column is in `wanted_values`. Lines are kept as given + (no coalescing yet).""" + out: list[tuple[int, int, str]] = [] + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = LINE.match(line) + if not m: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + value = m.group(3) + if value in wanted_values: + out.append((start, end, value)) + return out + + +def coalesce_pairs(pairs: Iterable[tuple[int, int]]) -> list[tuple[int, int]]: + """Sort and merge adjacent (start, end) ranges.""" + pairs = sorted(pairs, key=lambda p: p[0]) + out: list[tuple[int, int]] = [] + for s, e in pairs: + if out and out[-1][1] + 1 >= s: + ps, pe = out[-1] + out[-1] = (ps, max(pe, e)) + else: + out.append((s, e)) + return out + + +def cp_hex(cp: int) -> str: + return f"0x{cp:06X}" + + +def emit_ranges( + base: str, + qc_label: str, + ranges: list[tuple[int, int]], +) -> str: + """Emit @RangeCount@ and @Ranges@ bindings for + the given coalesced (start, end) list, with comments + referring to the QC label (\"No\" or \"Maybe\").""" + name = f"{base}Ranges" + count_name = f"{base}RangeCount" + n = len(ranges) + lines = [] + lines.append(f"-- | Number of (start, end) range pairs in '{name}'.") + lines.append(f"{count_name} :: Int") + lines.append(f"{count_name} = {n}") + lines.append("") + lines.append( + f"-- | Codepoint ranges with @NFC_Quick_Check = {qc_label}@," + ) + lines.append("-- alternating start\\/end @Word32@ pairs.") + lines.append(f"{name} :: ByteArray") + if n == 0: + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word32])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + flat: list[str] = [] + for i, (s, e) in enumerate(ranges): + if i == 0: + flat.append(f"({cp_hex(s)} :: Word32)") + else: + flat.append(cp_hex(s)) + flat.append(cp_hex(e)) + body = "\n , ".join(flat) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.NFC.Data +-- Description : Codepoint range tables for the Unicode +-- @NFC_Quick_Check@ property values @No@ and +-- @Maybe@. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaNFC.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/DerivedNormalizationProps.txt +-- Aligned with: Unicode {version} +-- NFC_QC=No ranges: {n_no} +-- NFC_QC=Maybe ranges: {n_maybe} +-- +-- Codepoints absent from both tables are implicitly @Yes@ +-- (already in NFC). +module Text.IDNA2008.Internal.NFC.Data + ( nfcNoRangeCount + , nfcNoRanges + , nfcMaybeRangeCount + , nfcMaybeRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- NFC_Quick_Check = No +---------------------------------------------------------------------- + +""" + + +SECTION_BREAK = """ + +---------------------------------------------------------------------- +-- NFC_Quick_Check = Maybe +---------------------------------------------------------------------- + +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + props_path = sys.argv[2] + + raw = parse_lines(props_path, {"N", "M"}) + no_pairs = coalesce_pairs([(s, e) for (s, e, v) in raw if v == "N"]) + maybe_pairs = coalesce_pairs([(s, e) for (s, e, v) in raw if v == "M"]) + + out = sys.stdout + out.write( + HEADER.format( + version=version, + n_no=len(no_pairs), + n_maybe=len(maybe_pairs), + ) + ) + out.write(emit_ranges("nfcNo", "No", no_pairs)) + out.write("\n") + out.write(SECTION_BREAK) + out.write(emit_ranges("nfcMaybe", "Maybe", maybe_pairs)) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaNFCTables.py b/internal/tools/genIdnaNFCTables.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaNFCTables.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.NFC.Tables.Data from the Unicode +UCD files UnicodeData.txt and CompositionExclusions.txt. + +Usage: + python3 internal/tools/genIdnaNFCTables.py \\ + + +Reads canonical decomposition mappings (field 5 of UnicodeData.txt +without a @<...>@ tag), Canonical_Combining_Class (field 3), and the +list of codepoints excluded from composition. Emits a complete +Haskell module to stdout with three tables: + + * Canonical decomposition: source codepoint -> (first, second) + where second == 0 denotes a singleton decomposition. + + * Primary composition: (first, second) -> result, restricted to + decompositions that are not singletons, where the first + constituent is a starter (CCC = 0), and where the source is + not in the Composition_Exclusion set. + + * Canonical_Combining_Class: codepoint -> CCC value, encoded as + sorted ranges of equal-CCC runs. + +Hangul syllables are intentionally excluded from all three tables; +they are handled by closed-form math in the consumer module. +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# UnicodeData.txt is one record per line, semicolon-separated, 15 +# fields. We only care about a few of them; just split() and index. +DECOMP_TAG = re.compile(r"^<[^>]+>\s*") + +# Hangul ranges (decomposed via algorithmic math, not table). +HANGUL_S_BASE = 0xAC00 +HANGUL_S_COUNT = 11172 # 0xAC00 .. 0xD7A3 +HANGUL_L_BASE = 0x1100 +HANGUL_L_COUNT = 19 +HANGUL_V_BASE = 0x1161 +HANGUL_V_COUNT = 21 +HANGUL_T_BASE = 0x11A7 +HANGUL_T_COUNT = 28 +HANGUL_END = HANGUL_S_BASE + HANGUL_S_COUNT - 1 + + +def is_hangul_syllable(cp: int) -> bool: + return HANGUL_S_BASE <= cp <= HANGUL_END + + +def parse_unicode_data( + path: str, +) -> tuple[ + dict[int, list[int]], # source -> canonical decomposition cps + dict[int, int], # codepoint -> CCC (only entries with CCC > 0) +]: + decomps: dict[int, list[int]] = {} + cccs: dict[int, int] = {} + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.rstrip("\n") + if not line: + continue + fields = line.split(";") + if len(fields) < 15: + continue + cp = int(fields[0], 16) + if is_hangul_syllable(cp): + # Hangul handled algorithmically; skip the table. + continue + ccc_str = fields[3].strip() + if ccc_str: + ccc = int(ccc_str) + if ccc != 0: + cccs[cp] = ccc + decomp_field = fields[5].strip() + if not decomp_field: + continue + # Strip any "" prefix; canonical entries + # have no tag. + if DECOMP_TAG.match(decomp_field): + continue + parts = decomp_field.split() + decomp = [int(p, 16) for p in parts] + if decomp: + decomps[cp] = decomp + return decomps, cccs + + +def parse_composition_exclusions(path: str) -> set[int]: + excluded: set[int] = set() + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0].strip() + if not line: + continue + # Some lines are ranges "start..end"; most are single cps. + if ".." in line: + lo, hi = line.split("..", 1) + lo_cp = int(lo.strip(), 16) + hi_cp = int(hi.strip(), 16) + for cp in range(lo_cp, hi_cp + 1): + excluded.add(cp) + else: + excluded.add(int(line, 16)) + return excluded + + +def coalesce_ccc(cccs: dict[int, int]) -> list[tuple[int, int, int]]: + """Group consecutive codepoints with equal CCC into ranges, + sorted by start. Returns [(start, end, ccc)].""" + items = sorted(cccs.items()) + out: list[tuple[int, int, int]] = [] + for cp, v in items: + if out and out[-1][2] == v and out[-1][1] + 1 == cp: + ps, _, pv = out[-1] + out[-1] = (ps, cp, pv) + else: + out.append((cp, cp, v)) + return out + + +def build_composition( + decomps: dict[int, list[int]], + excluded: set[int], + cccs: dict[int, int], +) -> list[tuple[int, int, int]]: + """Reverse the decomposition map into primary composition entries. + A composition (first, second) -> result is included iff: + + * The decomposition has exactly two codepoints. + * The result is not in the Composition_Exclusion set. + * The first constituent is a starter (CCC == 0), so cccs + does not list it. + """ + out: list[tuple[int, int, int]] = [] + for src, decomp in decomps.items(): + if len(decomp) != 2: + continue + if src in excluded: + continue + first, second = decomp + if cccs.get(first, 0) != 0: + # Non-starter primary; UAX #15 excludes these from + # primary composition. + continue + out.append((first, second, src)) + out.sort(key=lambda t: (t[0], t[1])) + return out + + +def cp_hex(cp: int) -> str: + return f"0x{cp:06X}" + + +def emit_word32_array(name: str, items: Iterable[int]) -> str: + items = list(items) + n = len(items) + lines: list[str] = [] + lines.append(f"{name} :: ByteArray") + if n == 0: + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word32])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + pieces: list[str] = [] + for i, v in enumerate(items): + pieces.append(f"({cp_hex(v)} :: Word32)" if i == 0 else cp_hex(v)) + body = "\n , ".join(pieces) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +def emit_word8_array(name: str, items: Iterable[int]) -> str: + items = list(items) + n = len(items) + lines: list[str] = [] + lines.append(f"{name} :: ByteArray") + if n == 0: + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word8])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + pieces: list[str] = [] + for i, v in enumerate(items): + pieces.append(f"({v} :: Word8)" if i == 0 else str(v)) + body = "\n , ".join(pieces) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.NFC.Tables.Data +-- Description : Canonical decomposition, primary composition, and +-- Canonical_Combining_Class tables used by the full +-- NFC normalize-and-compare validator. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaNFCTables.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/ +-- Aligned with: Unicode {version} +-- Decomposition entries: {n_decomp} +-- Composition entries: {n_comp} +-- CCC ranges: {n_ccc} +-- +-- Hangul syllables are excluded from all three tables; the +-- consumer module handles them via closed-form arithmetic. +module Text.IDNA2008.Internal.NFC.Tables.Data + ( -- * Canonical decomposition + decompCount + , decompKeys + , decompFirst + , decompSecond + + -- * Primary composition + , compCount + , compFirst + , compSecond + , compResult + + -- * Canonical_Combining_Class + , cccCount + , cccStarts + , cccEnds + , cccValues + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word8, Word32) +import qualified Data.Primitive.ByteArray as PBA + +---------------------------------------------------------------------- +-- Canonical decomposition +---------------------------------------------------------------------- + +decompCount :: Int +decompCount = {n_decomp} + +""" + +COMP_HEADING = """ + +---------------------------------------------------------------------- +-- Primary composition +---------------------------------------------------------------------- + +compCount :: Int +compCount = {n_comp} + +""" + +CCC_HEADING = """ + +---------------------------------------------------------------------- +-- Canonical_Combining_Class (CCC) +---------------------------------------------------------------------- + +cccCount :: Int +cccCount = {n_ccc} + +""" + + +def main() -> None: + if len(sys.argv) != 4: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + udata_path = sys.argv[2] + excl_path = sys.argv[3] + + decomps, cccs = parse_unicode_data(udata_path) + excluded = parse_composition_exclusions(excl_path) + comps = build_composition(decomps, excluded, cccs) + ccc_ranges = coalesce_ccc(cccs) + + # Sort decomp by source codepoint. + decomp_items = sorted(decomps.items()) + decomp_keys = [k for (k, _) in decomp_items] + decomp_first = [v[0] for (_, v) in decomp_items] + decomp_second = [ + v[1] if len(v) >= 2 else 0 + for (_, v) in decomp_items + ] + + out = sys.stdout + out.write( + HEADER.format( + version=version, + n_decomp=len(decomp_keys), + n_comp=len(comps), + n_ccc=len(ccc_ranges), + ) + ) + out.write(emit_word32_array("decompKeys", decomp_keys)) + out.write("\n\n") + out.write(emit_word32_array("decompFirst", decomp_first)) + out.write("\n\n") + out.write(emit_word32_array("decompSecond", decomp_second)) + out.write("\n") + + out.write(COMP_HEADING.format(n_comp=len(comps))) + out.write(emit_word32_array("compFirst", [c[0] for c in comps])) + out.write("\n\n") + out.write(emit_word32_array("compSecond", [c[1] for c in comps])) + out.write("\n\n") + out.write(emit_word32_array("compResult", [c[2] for c in comps])) + out.write("\n") + + out.write(CCC_HEADING.format(n_ccc=len(ccc_ranges))) + out.write(emit_word32_array("cccStarts", [r[0] for r in ccc_ranges])) + out.write("\n\n") + out.write(emit_word32_array("cccEnds", [r[1] for r in ccc_ranges])) + out.write("\n\n") + out.write(emit_word8_array ("cccValues", [r[2] for r in ccc_ranges])) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaProperty.py b/internal/tools/genIdnaProperty.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaProperty.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +genIdnaProperty.py -- regenerate Text.IDNA2008.Internal.Property.Data +from an IANA IDNA properties CSV. + +The IANA registry publishes per-Unicode-version derived-property tables +at: + + https://www.iana.org/assignments/idna-tables-properties/ + +Concrete CSV download URLs follow the pattern: + + https://www.iana.org/assignments/idna-tables-X.Y.Z/idna-tables-properties.csv + +For example (Unicode 12.0.0, the most recent IANA-curated version as of +2024-04-26): + + https://www.iana.org/assignments/idna-tables-12.0.0/idna-tables-properties.csv + +Usage: + + python3 internal/tools/genIdnaProperty.py \\ + > internal/Text/IDNA2008/Internal/Property/Data.hs + +The first argument is recorded in the generated module's header for +provenance. The second is the local path to the IANA CSV. + +Each row of the CSV gives a codepoint or codepoint range and one of +five disposition values: + + PVALID, CONTEXTJ, CONTEXTO, DISALLOWED, UNASSIGNED + +The output module exposes two parallel arrays: + + rangeStarts :: ByteArray of Word32 -- sorted ascending + rangeTags :: ByteArray of Word8 -- one tag per range + +with disposition tags 0 = PVALID, 1 = CONTEXTJ, 2 = CONTEXTO, +3 = DISALLOWED, 4 = UNASSIGNED. Each range covers codepoints +[rangeStarts[i], rangeStarts[i+1] - 1] (or 0x10FFFF for the last +range). +""" +import csv +import sys + +DISP_MAP = { + "PVALID": 0, + "CONTEXTJ": 1, + "CONTEXTO": 2, + "DISALLOWED": 3, + "UNASSIGNED": 4, +} + +STARTS_PER_LINE = 8 +TAGS_PER_LINE = 32 + + +def parse_csv(path): + starts = [] + tags = [] + with open(path, newline="") as f: + reader = csv.DictReader(f) + for row in reader: + cp = row["Codepoint"] + prop = row["Property"] + start = int(cp.split("-")[0], 16) + try: + tag = DISP_MAP[prop] + except KeyError: + raise SystemExit(f"unknown disposition {prop!r} in {path}") + starts.append(start) + tags.append(tag) + return starts, tags + + +def emit_module(version, starts, tags, out=sys.stdout): + n = len(starts) + p = out.write + + p("-- |\n") + p("-- Module : Text.IDNA2008.Internal.Property.Data\n") + p("-- Description : Codegen'd IDNA2008 codepoint disposition table\n") + p("-- Copyright : (c) Viktor Dukhovni, 2026\n") + p("-- License : BSD-3-Clause\n") + p("--\n") + p("-- Maintainer : ietf-dane@dukhovni.org\n") + p("-- Stability : unstable\n") + p("--\n") + p("-- AUTOMATICALLY GENERATED FROM idna-tables-properties.csv.\n") + p("-- Do not edit by hand. Regenerate with:\n") + p("--\n") + p("-- python3 internal/tools/genIdnaProperty.py \n") + p("--\n") + p("-- Source: https://www.iana.org/assignments/idna-tables-properties/\n") + p(f"-- Aligned with: Unicode {version}\n") + p(f"-- Range count: {n}\n") + p("--\n") + p("-- Each disposition tag is a small Word8:\n") + p("--\n") + p("-- * 0 -- PVALID\n") + p("-- * 1 -- CONTEXTJ\n") + p("-- * 2 -- CONTEXTO\n") + p("-- * 3 -- DISALLOWED\n") + p("-- * 4 -- UNASSIGNED\n") + p("module Text.IDNA2008.Internal.Property.Data\n") + p(" ( rangeCount\n") + p(" , rangeStarts\n") + p(" , rangeTags\n") + p(" ) where\n\n") + + p("import Data.Array.Byte (ByteArray)\n") + p("import Data.Word (Word8, Word32)\n") + p("import qualified Data.Primitive.ByteArray as PBA\n\n") + + p("rangeCount :: Int\n") + p(f"rangeCount = {n}\n\n") + + p("rangeStarts :: ByteArray\n") + p("rangeStarts = PBA.byteArrayFromList\n") + for i in range(0, n, STARTS_PER_LINE): + chunk = starts[i:i + STARTS_PER_LINE] + if i == 0: + head = f"(0x{chunk[0]:06X} :: Word32)" + tail = ", ".join(f"0x{s:06X}" for s in chunk[1:]) + body = head + ((", " + tail) if tail else "") + p(f" [ {body}\n") + else: + body = ", ".join(f"0x{s:06X}" for s in chunk) + p(f" , {body}\n") + p(" ]\n\n") + + p("rangeTags :: ByteArray\n") + p("rangeTags = PBA.byteArrayFromList @Word8\n") + for i in range(0, n, TAGS_PER_LINE): + chunk = tags[i:i + TAGS_PER_LINE] + body = ", ".join(str(t) for t in chunk) + if i == 0: + p(f" [ {body}\n") + else: + p(f" , {body}\n") + p(" ]\n") + + +def main(): + if len(sys.argv) != 3: + sys.exit( + "usage: genIdnaProperty.py \n" + "\n" + "Fetch the CSV from\n" + " https://www.iana.org/assignments/idna-tables-X.Y.Z/idna-tables-properties.csv\n" + "and pass the local path as the second argument." + ) + version, path = sys.argv[1], sys.argv[2] + starts, tags = parse_csv(path) + if not starts: + sys.exit(f"{path}: no entries parsed") + emit_module(version, starts, tags) + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaScript.py b/internal/tools/genIdnaScript.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaScript.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.Script.Data from the Unicode UCD +Scripts.txt extracted property file. + +Usage: + python3 internal/tools/genIdnaScript.py \\ + + +Reads Scripts.txt and emits a complete Haskell module to stdout with +three tables: + + * 'greekRanges' -- codepoints with Script = Greek + * 'hebrewRanges' -- codepoints with Script = Hebrew + * 'hkhRanges' -- the union Hiragana \\| Katakana \\| Han + +Each table is encoded as alternating @start, end@ Word32 entries +(both endpoints inclusive), sorted by start with abutting ranges +coalesced. RFC 5892 Appendix A.4-A.7 are the consumers; the +Hiragana/Katakana/Han trio is merged because A.7 only ever asks +for "any of these three". + +Source: https://www.unicode.org/Public//ucd/Scripts.txt +""" +from __future__ import annotations + +import re +import sys +from typing import Iterable + +# Property-file line: hex-cp[..hex-cp] ; property-name [# comment] +LINE = re.compile( + r"^\s*([0-9A-Fa-f]+)(?:\.\.([0-9A-Fa-f]+))?\s*;\s*([A-Za-z0-9_]+)" +) + + +def parse_lines(path: str, wanted: set[str]) -> dict[str, list[tuple[int, int]]]: + """Return {script -> [(start, end)]} for every entry whose + script name is in 'wanted'.""" + out: dict[str, list[tuple[int, int]]] = {s: [] for s in wanted} + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.split("#", 1)[0] + m = LINE.match(line) + if not m: + continue + script = m.group(3) + if script not in wanted: + continue + start = int(m.group(1), 16) + end = int(m.group(2), 16) if m.group(2) else start + out[script].append((start, end)) + return out + + +def coalesce(pairs: Iterable[tuple[int, int]]) -> list[tuple[int, int]]: + """Sort and merge adjacent (or overlapping) (start, end) pairs.""" + pairs = sorted(pairs, key=lambda p: p[0]) + out: list[tuple[int, int]] = [] + for s, e in pairs: + if out and out[-1][1] + 1 >= s: + ps, pe = out[-1] + out[-1] = (ps, max(pe, e)) + else: + out.append((s, e)) + return out + + +def cp_hex(cp: int) -> str: + """Format a codepoint as 0xNNNNNN with at least 6 hex digits.""" + return f"0x{cp:06X}" + + +def emit_word32_pairs(name: str, pairs: list[tuple[int, int]]) -> str: + """Render @name :: ByteArray ; name = PBA.byteArrayFromList [...]@ + with one (start, end) row per source line.""" + lines = [f"{name} :: ByteArray"] + if not pairs: + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word32])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + rendered: list[str] = [] + for i, (s, e) in enumerate(pairs): + if i == 0: + cell = f"({cp_hex(s)} :: Word32), {cp_hex(e)}" + else: + cell = f"{cp_hex(s)}, {cp_hex(e)}" + rendered.append(cell) + body = "\n , ".join(rendered) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Script.Data +-- Description : Compact codepoint range tables for the Unicode +-- Scripts consulted by the CONTEXTO contextual rules. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaScript.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/Scripts.txt +-- Aligned with: Unicode {version} +-- Greek ranges: {n_grek} +-- Hebrew ranges: {n_hebr} +-- HKH (Hiragana | Katakana | Han) ranges: {n_hkh} +-- +-- Only the four scripts whose membership is tested by RFC 5892 +-- Appendix A.4-A.7 (Greek, Hebrew, Hiragana \\/ Katakana \\/ Han) are +-- represented. Each script's @sc@ (Script) ranges are stored as a +-- sorted sequence of @(start, end)@ pairs (inclusive on both ends), +-- packed into a single 'ByteArray' as alternating 'Word32's: +-- +-- > [start0, end0, start1, end1, ...] +-- +-- Lookup is binary search for the largest @start_i \\<= cp@ and a +-- bounds check @cp \\<= end_i@; see "Text.IDNA2008.Internal.Script". +-- +-- The Hiragana, Katakana, and Han ranges are merged into a single +-- @hkhRanges@ table because RFC 5892 A.7 only ever asks for +-- \\"any of these three\\", and the merged table costs no extra +-- comparisons at lookup time. Greek and Hebrew remain separate +-- because A.4 only asks for Greek and A.5\\/A.6 only for Hebrew. +module Text.IDNA2008.Internal.Script.Data + ( greekRangeCount + , greekRanges + , hebrewRangeCount + , hebrewRanges + , hkhRangeCount + , hkhRanges + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +""" + + +SECTION_HEADING = """\ + +---------------------------------------------------------------------- +-- {title} +---------------------------------------------------------------------- + +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + scripts_path = sys.argv[2] + + wanted = {"Greek", "Hebrew", "Hiragana", "Katakana", "Han"} + raw = parse_lines(scripts_path, wanted) + + greek = coalesce(raw["Greek"]) + hebrew = coalesce(raw["Hebrew"]) + hkh = coalesce(raw["Hiragana"] + raw["Katakana"] + raw["Han"]) + + out = sys.stdout + out.write(HEADER.format( + version=version, + n_grek=len(greek), + n_hebr=len(hebrew), + n_hkh=len(hkh), + )) + + out.write(SECTION_HEADING.format(title="Greek (Script: Grek)")) + out.write("-- | Number of (start, end) range pairs in 'greekRanges'.\n") + out.write("greekRangeCount :: Int\n") + out.write(f"greekRangeCount = {len(greek)}\n\n") + out.write("-- | Greek script codepoint ranges, alternating start\\/end.\n") + out.write(emit_word32_pairs("greekRanges", greek)) + out.write("\n") + + out.write(SECTION_HEADING.format(title="Hebrew (Script: Hebr)")) + out.write("-- | Number of (start, end) range pairs in 'hebrewRanges'.\n") + out.write("hebrewRangeCount :: Int\n") + out.write(f"hebrewRangeCount = {len(hebrew)}\n\n") + out.write("-- | Hebrew script codepoint ranges, alternating start\\/end.\n") + out.write(emit_word32_pairs("hebrewRanges", hebrew)) + out.write("\n") + + out.write(SECTION_HEADING.format( + title="Hiragana | Katakana | Han (merged for RFC 5892 A.7)")) + out.write("-- | Number of (start, end) range pairs in 'hkhRanges'.\n") + out.write("hkhRangeCount :: Int\n") + out.write(f"hkhRangeCount = {len(hkh)}\n\n") + out.write("-- | Codepoints whose @sc@ (Script) is Hiragana, Katakana,\n") + out.write("-- or Han, merged into one sorted range list.\n") + out.write(emit_word32_pairs("hkhRanges", hkh)) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaSimpleLower.py b/internal/tools/genIdnaSimpleLower.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaSimpleLower.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +genIdnaSimpleLower.py -- regenerate Text.IDNA2008.Internal.Case.Data +from the Unicode UCD's UnicodeData.txt. + +The output module exposes a single function + + toLower :: Char -> Char + +implementing the Unicode @Simple_Lowercase_Mapping@ property (column +13 of @UnicodeData.txt@), pinned to the UCD version supplied as the +first argument. Codepoints with no defined lowercase mapping fall +through to identity. + +The function is realised as a 'LambdaCase' lookup whose shape mirrors +@GHC.Internal.Unicode.Char.UnicodeData.SimpleLowerCaseMapping@ from +the GHC 9.14.1 @base@ library; GHC compiles a @\\case@ over @Char@ +into a balanced decision tree on the codepoint integer, so the +runtime cost is @O(log n)@ with very small constants and no +indirection through 'ByteArray' / FFI machinery. + +Usage: + + python3 internal/tools/genIdnaSimpleLower.py \\ + > internal/Text/IDNA2008/Internal/Case/Data.hs + +UnicodeData.txt columns of interest: + + 0 Codepoint (hex, no prefix) + 1 Character name + 2 General_Category + ... + 12 Simple_Uppercase_Mapping + 13 Simple_Lowercase_Mapping <-- consumed here + 14 Simple_Titlecase_Mapping + +Codepoint ranges denoted by @<..., First>@ / @<..., Last>@ pairs in +UnicodeData.txt (CJK ideographs, Hangul syllables, etc.) never carry +case mappings, so we don't attempt to expand them. +""" +from __future__ import annotations + +import sys + + +def parse(path: str) -> list[tuple[int, int]]: + """Return [(source_cp, target_cp)] for every row with a non-empty + Simple_Lowercase_Mapping field, sorted ascending by source.""" + out: list[tuple[int, int]] = [] + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.rstrip("\n") + if not line: + continue + cols = line.split(";") + if len(cols) < 14: + continue + slm = cols[13].strip() + if not slm: + continue + src = int(cols[0], 16) + tgt = int(slm, 16) + out.append((src, tgt)) + out.sort(key=lambda p: p[0]) + return out + + +HEADER = """\ +-- DO NOT EDIT: This file is automatically generated by +-- internal/tools/genIdnaSimpleLower.py, with data from: +-- https://www.unicode.org/Public/{version}/ucd/UnicodeData.txt +-- +-- Pinned to Unicode {version}; bump @x-unicode-version@ in +-- @idna2008.cabal@ and re-run @internal/tools/update@ to advance. + +{{-# LANGUAGE NoImplicitPrelude, LambdaCase #-}} +{{-# OPTIONS_HADDOCK hide #-}} + +----------------------------------------------------------------------------- +-- | +-- Module : Text.IDNA2008.Internal.Case.Data +-- Description : Unicode @Simple_Lowercase_Mapping@ table for the +-- UCD version pinned by the @idna2008@ library. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaSimpleLower.py \\\\ +-- +-- +-- Aligned with: Unicode {version} +-- Mapping entries: {count} +----------------------------------------------------------------------------- + +module Text.IDNA2008.Internal.Case.Data +(toLower) +where + +import Prelude (Char) + +{{-# NOINLINE toLower #-}} +toLower :: Char -> Char +toLower = \\case +""" + + +FOOTER = """\ + c -> c +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + + version = sys.argv[1] + ucd_path = sys.argv[2] + + pairs = parse(ucd_path) + + out = sys.stdout + out.write(HEADER.format(version=version, count=len(pairs))) + for src, tgt in pairs: + out.write(f" '\\x{src:x}' -> '\\x{tgt:x}'\n") + out.write(FOOTER) + + +if __name__ == "__main__": + main() diff --git a/internal/tools/genIdnaWidth.py b/internal/tools/genIdnaWidth.py new file mode 100644 --- /dev/null +++ b/internal/tools/genIdnaWidth.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Generate Text.IDNA2008.Internal.Width.Data from UnicodeData.txt. + +Usage: + python3 internal/tools/genIdnaWidth.py \\ + + +Reads field 5 (Decomposition_Mapping) for each codepoint and keeps +those tagged or , emitting two parallel sorted Word32 +arrays of (source, target) codepoints for the RFC 5895 section 2.2 +fullwidth/halfwidth mapping. + +Source: https://www.unicode.org/Public//ucd/UnicodeData.txt +""" +from __future__ import annotations + +import re +import sys + +WIDE_NARROW_RE = re.compile(r"^<(wide|narrow)>\s+([0-9A-Fa-f]+)\s*$") + + +def parse(path: str) -> list[tuple[int, int]]: + """Return [(source, target)] for every codepoint with a single- + codepoint or decomposition.""" + out: list[tuple[int, int]] = [] + with open(path, encoding="utf-8") as f: + for line in f: + fields = line.rstrip("\n").split(";") + if len(fields) < 6: + continue + cp = int(fields[0], 16) + decomp = fields[5] + m = WIDE_NARROW_RE.match(decomp) + if not m: + continue + target = int(m.group(2), 16) + out.append((cp, target)) + return sorted(out) + + +def cp_hex(cp: int) -> str: + return f"0x{cp:06X}" + + +def emit_word32_list(name: str, items: list[int]) -> str: + lines = [f"{name} :: ByteArray"] + if not items: + lines.append(f"{name} = PBA.byteArrayFromList ([] :: [Word32])") + return "\n".join(lines) + lines.append(f"{name} = PBA.byteArrayFromList") + rendered = [] + for i, v in enumerate(items): + cell = f"({cp_hex(v)} :: Word32)" if i == 0 else cp_hex(v) + rendered.append(cell) + body = "\n , ".join(rendered) + lines.append(f" [ {body}") + lines.append(" ]") + return "\n".join(lines) + + +HEADER = """\ +-- | +-- Module : Text.IDNA2008.Internal.Width.Data +-- Description : Compact codepoint mapping table for the Unicode +-- wide \\/ narrow decompositions consumed by the +-- RFC 5895 section 2.2 (fullwidth\\/halfwidth) mapping. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- AUTOMATICALLY GENERATED. Do not edit by hand. Regenerate with: +-- +-- python3 internal/tools/genIdnaWidth.py \\\\ +-- +-- +-- Source: https://www.unicode.org/Public/{version}/ucd/UnicodeData.txt +-- Aligned with: Unicode {version} +-- Width-mapping entries: {n} +-- +-- Each entry is a single-codepoint decomposition: a fullwidth or +-- halfwidth codepoint that maps to a single canonical-width target. +-- Stored as two parallel sorted arrays -- 'widthSources' (source +-- codepoints, sorted ascending) and 'widthTargets' (the targets, in +-- the same order). Lookup is binary search over 'widthSources'. +module Text.IDNA2008.Internal.Width.Data + ( widthRangeCount + , widthSources + , widthTargets + ) where + +import Data.Array.Byte (ByteArray) +import Data.Word (Word32) +import qualified Data.Primitive.ByteArray as PBA + +""" + + +def main() -> None: + if len(sys.argv) != 3: + sys.stderr.write(__doc__) + sys.exit(1) + version = sys.argv[1] + pairs = parse(sys.argv[2]) + sources = [s for s, _ in pairs] + targets = [t for _, t in pairs] + + out = sys.stdout + out.write(HEADER.format(version=version, n=len(pairs))) + + out.write("-- | Number of (source, target) entries.\n") + out.write("widthRangeCount :: Int\n") + out.write(f"widthRangeCount = {len(pairs)}\n\n") + + out.write("-- | Source codepoints, sorted ascending.\n") + out.write(emit_word32_list("widthSources", sources)) + out.write("\n\n") + + out.write("-- | Target codepoints; parallel to 'widthSources'.\n") + out.write(emit_word32_list("widthTargets", targets)) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/internal/tools/update b/internal/tools/update new file mode 100644 --- /dev/null +++ b/internal/tools/update @@ -0,0 +1,261 @@ +#!/bin/sh +# Regenerate every Unicode-derived asset in the package from the +# upstream Unicode sources for the requested version: +# +# * The library's table modules under +# internal/Text/IDNA2008/Internal/*/Data.hs (Property, NFC, +# Joining, Script, Width, Case, Bidi, Combining, Emoji). +# * The UTS #46 reference data the `unicode-conformance` test +# suite consults, bundled XZ-compressed under tests/data/. +# +# Usage: +# internal/tools/update [] +# +# When no version argument is supplied the script reads the default +# from the @x-unicode-version@ custom field in @idna2008.cabal@, +# which is the single source of truth for the Unicode version the +# library tracks. Bump the field there and re-run this script to +# move to a new Unicode release; everything Unicode-derived stays +# in lockstep. +# +# All output paths are anchored at this script's directory, so the +# script can be run from anywhere. +# +# Override the Python interpreter via the PYTHON environment variable +# (e.g. PYTHON=python3.12 ./update 17.0.0). +# +# The IDNA disposition table is derived locally per RFC 5892 section 3 by +# genIdnaDerive.py from UCD inputs; we no longer fetch from the IANA +# IDNA-tables-properties registry, which lags Unicode releases (it +# tops out at 12.0.0 as of mid-2025). The local derivation has been +# verified to reproduce IANA's published 12.0.0 CSV byte-for-byte. + +set -eu + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +CABAL_FILE="${SCRIPT_DIR}/../../idna2008.cabal" + +read_unicode_version() { + awk '/^x-unicode-version:/ { print $2; exit }' "$CABAL_FILE" +} + +DEFAULT_VERSION=$(read_unicode_version) +if [ -z "$DEFAULT_VERSION" ]; then + echo "update: could not read x-unicode-version from $CABAL_FILE" >&2 + exit 1 +fi + +VERSION="${1:-$DEFAULT_VERSION}" +PYTHON="${PYTHON:-python3}" + +#----------------------------------------------------------------------- +# Source URL templates -- edit here if Unicode reorganises its +# filesystem layout. '%s' is the Unicode version string passed in +# as the first argument (e.g. "17.0.0"). +#----------------------------------------------------------------------- + +# Unicode UCD bundle: most files live under /Public//ucd/. +UCD_BASE="https://www.unicode.org/Public/%s/ucd" + +UNICODE_DATA_URL="${UCD_BASE}/UnicodeData.txt" +COMPOSITION_EXCLUSIONS_URL="${UCD_BASE}/CompositionExclusions.txt" +DERIVED_CORE_PROPERTIES_URL="${UCD_BASE}/DerivedCoreProperties.txt" +DERIVED_NORM_PROPS_URL="${UCD_BASE}/DerivedNormalizationProps.txt" +PROP_LIST_URL="${UCD_BASE}/PropList.txt" +HANGUL_SYLLABLE_TYPE_URL="${UCD_BASE}/HangulSyllableType.txt" +SCRIPTS_URL="${UCD_BASE}/Scripts.txt" +DERIVED_JOINING_TYPE_URL="${UCD_BASE}/extracted/DerivedJoiningType.txt" +DERIVED_COMBINING_CLASS_URL="${UCD_BASE}/extracted/DerivedCombiningClass.txt" +DERIVED_BIDI_CLASS_URL="${UCD_BASE}/extracted/DerivedBidiClass.txt" + +# Emoji file: in 13.0+ it's part of the UCD bundle (Public//ucd/emoji/); +# in 12.x and earlier it lived in a separate Public/emoji// tree. +EMOJI_UCD_URL="${UCD_BASE}/emoji/emoji-data.txt" +EMOJI_LEGACY_URL="https://www.unicode.org/Public/emoji/%s/emoji-data.txt" + +# UTS #46 reference data used by the `unicode-conformance` test suite. +# The Unicode Consortium reorganised the IDNA data-file layout at +# Unicode 17.0.0 -- for >=17 the files live alongside the main UCD +# at /Public//idna/, for <=16 at /Public/idna//, and the +# /Public/idna/latest/ symlink continues to track the most-recently +# published version regardless of layout. +UTS46_NEW_BASE="https://www.unicode.org/Public/%s/idna" +UTS46_OLD_BASE="https://www.unicode.org/Public/idna/%s" + +#----------------------------------------------------------------------- +# Compute concrete URLs and output paths. +#----------------------------------------------------------------------- + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +IDNA_DIR="${SCRIPT_DIR}/../Text/IDNA2008/Internal" + +unidata_url=$(printf "$UNICODE_DATA_URL" "$VERSION") +compex_url=$(printf "$COMPOSITION_EXCLUSIONS_URL" "$VERSION") +dcp_url=$(printf "$DERIVED_CORE_PROPERTIES_URL" "$VERSION") +nfcprops_url=$(printf "$DERIVED_NORM_PROPS_URL" "$VERSION") +plist_url=$(printf "$PROP_LIST_URL" "$VERSION") +hst_url=$(printf "$HANGUL_SYLLABLE_TYPE_URL" "$VERSION") +scripts_url=$(printf "$SCRIPTS_URL" "$VERSION") +jt_url=$(printf "$DERIVED_JOINING_TYPE_URL" "$VERSION") +ccc_url=$(printf "$DERIVED_COMBINING_CLASS_URL" "$VERSION") +bidi_url=$(printf "$DERIVED_BIDI_CLASS_URL" "$VERSION") + +case "$VERSION" in + 12.*|11.*|10.*|9.*|8.*|7.*|6.*) + # Pre-13: emoji files in the legacy Public/emoji// tree. + short_ver=$(echo "$VERSION" | cut -d. -f1,2) + emoji_url=$(printf "$EMOJI_LEGACY_URL" "$short_ver") + ;; + *) + emoji_url=$(printf "$EMOJI_UCD_URL" "$VERSION") + ;; +esac + +# Decide which UTS #46 layout to use based on the version's major +# component. Versions >= 17 use the new under-UCD layout; older +# versions use /Public/idna//. An explicit "latest" value +# resolves through the symlink. +uts46_major=${VERSION%%.*} +case "$VERSION" in + latest) + uts46_test_url="https://www.unicode.org/Public/idna/latest/IdnaTestV2.txt" + uts46_map_url="https://www.unicode.org/Public/idna/latest/IdnaMappingTable.txt" + ;; + *) + case "$uts46_major" in + ''|*[!0-9]*) + uts46_base=$(printf "$UTS46_OLD_BASE" "$VERSION") + ;; + *) + if [ "$uts46_major" -ge 17 ]; then + uts46_base=$(printf "$UTS46_NEW_BASE" "$VERSION") + else + uts46_base=$(printf "$UTS46_OLD_BASE" "$VERSION") + fi + ;; + esac + uts46_test_url="${uts46_base}/IdnaTestV2.txt" + uts46_map_url="${uts46_base}/IdnaMappingTable.txt" + ;; +esac + +TESTS_DATA_DIR="${SCRIPT_DIR}/../../tests/data" + +#----------------------------------------------------------------------- +# Fetch all sources to a scratch dir, then run each generator. +#----------------------------------------------------------------------- + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +fetch() { + # $1 = URL, $2 = local filename inside $WORK + echo "fetching: $1" + curl --fail --silent --show-error --location \ + --output "$WORK/$2" "$1" +} + +fetch "$unidata_url" UnicodeData.txt +fetch "$compex_url" CompositionExclusions.txt +fetch "$dcp_url" DerivedCoreProperties.txt +fetch "$nfcprops_url" DerivedNormalizationProps.txt +fetch "$plist_url" PropList.txt +fetch "$hst_url" HangulSyllableType.txt +fetch "$scripts_url" Scripts.txt +fetch "$jt_url" DerivedJoiningType.txt +fetch "$ccc_url" DerivedCombiningClass.txt +fetch "$bidi_url" DerivedBidiClass.txt +fetch "$emoji_url" emoji-data.txt +fetch "$uts46_test_url" IdnaTestV2.txt +fetch "$uts46_map_url" IdnaMappingTable.txt + +# Derive the IDNA disposition CSV locally per RFC 5892, then feed +# it into the existing CSV-to-Haskell generator. +echo "deriving: idna-tables-properties.csv (RFC 5892 section 3)" +"$PYTHON" "${SCRIPT_DIR}/genIdnaDerive.py" "$VERSION" \ + "$WORK/UnicodeData.txt" \ + "$WORK/DerivedCoreProperties.txt" \ + "$WORK/PropList.txt" \ + "$WORK/HangulSyllableType.txt" \ + "$WORK/DerivedNormalizationProps.txt" \ + > "$WORK/idna-tables-properties.csv" + +run() { + # $1 = output path; remaining args = command + args; stdout is redirected. + out="$1" + shift + echo "writing: $out" + "$@" > "$out" +} + +run "${IDNA_DIR}/Property/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaProperty.py" \ + "$VERSION" "$WORK/idna-tables-properties.csv" + +run "${IDNA_DIR}/NFC/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaNFC.py" \ + "$VERSION" "$WORK/DerivedNormalizationProps.txt" + +run "${IDNA_DIR}/NFC/Tables/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaNFCTables.py" \ + "$VERSION" "$WORK/UnicodeData.txt" "$WORK/CompositionExclusions.txt" + +run "${IDNA_DIR}/Joining/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaJoining.py" \ + "$VERSION" "$WORK/DerivedJoiningType.txt" "$WORK/DerivedCombiningClass.txt" + +run "${IDNA_DIR}/Script/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaScript.py" \ + "$VERSION" "$WORK/Scripts.txt" + +run "${IDNA_DIR}/Width/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaWidth.py" \ + "$VERSION" "$WORK/UnicodeData.txt" + +run "${IDNA_DIR}/Case/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaSimpleLower.py" \ + "$VERSION" "$WORK/UnicodeData.txt" + +run "${IDNA_DIR}/Bidi/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaBidi.py" \ + "$VERSION" "$WORK/DerivedBidiClass.txt" + +run "${IDNA_DIR}/Combining/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaCombining.py" \ + "$VERSION" "$WORK/UnicodeData.txt" + +run "${IDNA_DIR}/Emoji/Data.hs" \ + "$PYTHON" "${SCRIPT_DIR}/genIdnaEmoji.py" \ + "$VERSION" "$WORK/emoji-data.txt" "$WORK/IdnaMappingTable.txt" + +#----------------------------------------------------------------------- +# Bundle the UTS #46 reference data for the `unicode-conformance` test +# suite. The files are committed gzip-compressed under tests/data/ +# and unpacked at test time by Codec.Compression.GZip, so 'cabal +# test' runs offline without a .cache/ directory or network +# dependency. Gzip (rather than xz) keeps the build deps to the +# Haskell @zlib@ package, which ships everywhere and doesn't need a +# pkg-config dance to find a system C library. +#----------------------------------------------------------------------- + +mkdir -p "$TESTS_DATA_DIR" + +bundle() { + # $1 = source filename in $WORK; $2 = destination basename in + # $TESTS_DATA_DIR (the .gz extension is appended). The @-n@ + # flag suppresses the original filename and mtime from the gzip + # header so that re-running @update@ against the same source + # bytes produces a byte-identical output -- no spurious diffs + # in the committed tree after a no-op regeneration. + src="$WORK/$1" + dst="${TESTS_DATA_DIR}/$2.gz" + echo "bundling: $dst" + gzip -9 -n -c "$src" > "$dst" +} + +bundle IdnaTestV2.txt IdnaTestV2.txt +bundle IdnaMappingTable.txt IdnaMappingTable.txt + +echo +echo "All auto-generated IDNA tables are now aligned with Unicode $VERSION." +echo "UTS #46 conformance data refreshed under ${TESTS_DATA_DIR}/." diff --git a/src/Text/IDNA2008.hs b/src/Text/IDNA2008.hs new file mode 100644 --- /dev/null +++ b/src/Text/IDNA2008.hs @@ -0,0 +1,369 @@ +-- | +-- Module : Text.IDNA2008 +-- Description : Strict IDNA2008 parser and renderer. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Given a domain name as the user typed it — possibly mixing +-- Latin, Greek, Hebrew, Arabic, CJK, or any other Unicode script +-- — this library checks that every label is well-formed, encodes +-- any non-ASCII labels into their ACE-prefixed Punycode +-- form for the wire, classifies each label, and (optionally) +-- renders the parsed name back into display form. +-- +-- == Quick start +-- +-- > {-# LANGUAGE OverloadedStrings #-} +-- > module Main(main) where +-- > import qualified Data.Text.IO as T +-- > import Text.IDNA2008 +-- > +-- > main :: IO () +-- > main = do +-- > -- Strict-IDN parse + render two ways. 'mkDomain' uses +-- > -- 'idnLabelForms' and 'defaultIdnaFlags'. +-- > case mkDomain "www.αβγ.gr" of +-- > Right dom -> do +-- > T.putStrLn $ domainToUnicode dom +-- > T.putStrLn $ domainToAscii dom +-- > Left err -> print err +-- > +-- > -- The full parser returns the wire-form 'Domain' /and/ the +-- > -- per-label classification. 'allLabelForms' admits every +-- > -- label class a DNS zone file might carry plus 'ULABEL', +-- > -- so '*' and '_tcp' coexist with the Unicode label. +-- > case parseDomain allLabelForms "*._tcp.αβγ.gr" of +-- > Right (_dom, info) -> print (getLabelForms info) +-- > Left err -> print err +-- +-- Which produces: +-- +-- > ghci> main +-- > www.αβγ.gr +-- > www.xn--mxacd.gr +-- > [WILDLABEL,ATTRLEAF,ULABEL,LDH] +-- +-- == Strictness +-- +-- This library implements /strict/ IDNA2008 (RFCs 5891-5894 +-- including RFC 5893 Bidi rules and RFC 3492 Punycode). Optional +-- RFC5895 mappings are available when parsing text inputs. A +-- basic set of CJK mappings from UTS #46 can also be enabled, and +-- implies the full set of RFC5895 mappings. +-- +-- == Terminology: \"encode\" and \"decode\" +-- +-- The library uses /encode/ and /decode/ in the IDNA-spec +-- sense, centred on the Punycode transformation between a +-- U-label and its ACE-prefixed (A-label) encoded form: +-- +-- * 'parseDomainOpts' (presentation -> wire) is the /encoding/ +-- side: non-ASCII labels that arrive in U-label form are +-- Punycode-encoded to their ACE-prefixed forms used "on the wire" +-- in DNS queries. +-- +-- * 'unparseDomainOpts' (wire -> presentation) is the /decoding/ +-- side: ACE-prefixed labels are Punycode-decoded back to their +-- U-label forms for display. +-- +-- This terminology is natural for transformations between +-- U-labels and A-labels. ASCII LDH labels pass through both +-- directions byte-for-byte. The /encoding/ of 'OCTET' labels +-- (non-LDH bytes admitted as raw octets) to wire form requires +-- decoding backslash-escaped characters or @\\DDD@ decimal +-- triples from their presentation form, and /decoding/ of these +-- labels from wire form to presentation form requires encoding +-- many non-LDH octets as escaped characters or decimal triples. +module Text.IDNA2008 + ( -- * Domain names + Domain(..) + , toLabels + , isValidWireForm + , wireBytes + , wireBytesShort + + -- * Parsing presentation form + -- $parsing + + -- ** Returning wire form plus per-label classification + , parseDomain + , parseDomainOpts + , parseDomainUtf8 + , parseDomainShort + + -- ** Domain-only convenience wrappers + -- $convenience + , mkDomain + , mkDomainStr + , mkDomainUtf8 + , mkDomainShort + + -- ** Compile-time literals + -- $literals + , dnLit + + -- * Rendering wire form back to presentation form + , domainToAscii + , domainToUnicode + , labelToAscii + , labelToUnicode + , unparseDomainOpts + , unparseLabelOpts + + -- * Per-label classification result + -- $labelinfo + , LabelInfo + , getLabelForms + , getLabelFormCount + , getLabelForm + , allLabelFormsIn + + -- * Label form singletons + -- $labelform + , LabelForm + ( LDH + , RLDH + , FAKEA + , ALABEL + , ULABEL + , ATTRLEAF + , OCTET + , WILDLABEL + , LAXULABEL + , NoLabel + ) + + -- * Label form sets + -- $labelformset + , LabelFormSet + , (<+>) + , (<->) + , labelFormToSet + , memberLabelFormSet + , withoutLabelFormSet + , allLabelForms + , idnLabelForms + , hostnameLabelForms + + -- * Parser flags + -- $flags + , IdnaFlags + ( ALABELCHECK + , NFCCHECK + , EMOJIOK + , MAPDOTS + , MAPNFC + , MAPCASE + , MAPWIDTH + , BIDICHECK + , ASCIIFALLBACK + , MAPUTS46 + ) + , allIdnaMappings + , defaultIdnaFlags + , meetsIdnaFlags + , withoutIdnaFlags + + -- * Errors + , IdnaError(..) + , LabelReason(..) + , AceReason(..) + , BidiRuleViolation(..) + + -- * CLI option parsers + -- $cli + , parseIdnaFlags + , parseIdnaFlagsStr + , parseLabelFormSet + , parseLabelFormSetStr + ) where + +import Text.IDNA2008.Internal.Error + ( AceReason(..) + , BidiRuleViolation(..) + , IdnaError(..) + , LabelReason(..) + ) +import Text.IDNA2008.Internal.Flags + ( IdnaFlags + ( ALABELCHECK + , NFCCHECK + , EMOJIOK + , MAPDOTS + , MAPNFC + , MAPCASE + , MAPWIDTH + , BIDICHECK + , ASCIIFALLBACK + , MAPUTS46 + ) + , allIdnaMappings + , defaultIdnaFlags + , meetsIdnaFlags + , parseIdnaFlags + , parseIdnaFlagsStr + , withoutIdnaFlags + ) +import Text.IDNA2008.Internal.LabelForm + ( LabelForm + ( LDH + , RLDH + , FAKEA + , ALABEL + , ULABEL + , ATTRLEAF + , OCTET + , WILDLABEL + , LAXULABEL + , NoLabel + ) + ) +import Text.IDNA2008.Internal.LabelFormSet + ( LabelFormSet + , (<+>) + , (<->) + , labelFormToSet + , allLabelForms + , hostnameLabelForms + , idnLabelForms + , memberLabelFormSet + , parseLabelFormSet + , parseLabelFormSetStr + , withoutLabelFormSet + ) +import Text.IDNA2008.Internal.LabelInfo + ( LabelInfo + , getLabelForm + , getLabelFormCount + , getLabelForms + , allLabelFormsIn + ) +import Text.IDNA2008.Internal.Parse + ( Domain(..) + , dnLit + , domainToAscii + , domainToUnicode + , labelToAscii + , labelToUnicode + , unparseDomainOpts + , unparseLabelOpts + , mkDomain + , mkDomainShort + , mkDomainStr + , mkDomainUtf8 + , parseDomain + , parseDomainOpts + , parseDomainShort + , parseDomainUtf8 + , isValidWireForm + , toLabels + , wireBytes + , wireBytesShort + ) + +-- $parsing +-- The four parsing entry points share a shape: pass the +-- 'LabelFormSet' the application accepts (typically +-- 'idnLabelForms' or 'hostnameLabelForms') plus the input, +-- get back either a parse error or the parsed 'Domain' paired +-- with the per-label classification ('LabelInfo'). They differ +-- only in input type, chosen to match what the caller already +-- has on hand: +-- +-- * 'parseDomain' takes 'Data.Text.Text'. Since +-- 'Data.Text.Text' enforces well-formed UTF-8 at +-- construction, the parser views the underlying byte array +-- directly with no extra copy or re-validation. +-- * 'parseDomainOpts' is the same as 'parseDomain' but takes +-- an explicit 'IdnaFlags' bitmask (the other three use +-- 'defaultIdnaFlags'). +-- * 'parseDomainUtf8' takes a UTF-8-encoded +-- 'Data.ByteString.ByteString' — the convenient entry +-- point when the caller already has raw bytes (a network +-- read, a file, a protocol decoder) and would otherwise +-- pay a 'Data.Text.Encoding.decodeUtf8' just to reach +-- 'parseDomain'. One copy is unavoidable because +-- 'Data.ByteString.ByteString' is pinned and the parser +-- wants an unpinned buffer. +-- * 'parseDomainShort' is the zero-copy variant for callers +-- who already have a 'Data.ByteString.Short.ShortByteString'. +-- +-- The 'Data.ByteString.ByteString' and +-- 'Data.ByteString.Short.ShortByteString' variants are /not/ +-- assumed to be well-formed UTF-8; the parser performs a +-- strict RFC 3629 decode and reports @'ErrInvalidUtf8'@ on any +-- ill-formed sequence. + +-- $convenience +-- The 'mkDomain' convenience functions use defaults for both the +-- allowed label forms and the validation flags. They return just +-- the wire-form 'Domain' without the list of label classifications. + +-- $literals +-- Compile-time literal domains via Template Haskell, by way of +-- the 'dnLit' splice primitive. An invalid name becomes a +-- compile-time error rather than a runtime failure. + +-- $labelinfo +-- 'LabelInfo' is the parser's per-label classification result. +-- The representation is opaque; inspect via the accessors. +-- +-- * 'getLabelForms' returns the list in label order. +-- * 'getLabelFormCount' returns the label count. +-- * 'getLabelForm' indexes the list starting at @0@ for the first +-- label. +-- * 'allLabelFormsIn' tests whether every label belongs to a +-- given 'LabelFormSet'. +-- Vacuously returns 'True' for the root domain's 'LabelInfo'. +-- Applications that want to reject the root domain can check the +-- value of 'getLabelFormCount', or check whether the label list +-- returned by 'Text.IDNA2008.toLabels' is 'null'. + +-- $labelform +-- Each label falls into one of nine classifications: +-- +-- * 'LDH' — a valid label of letters, digits, and hyphens. +-- * 'RLDH' — a legacy reserved label with @--@ at positions 3-4. +-- * 'FAKEA' — an ACE-prefixed label that isn't a valid A-label. +-- * 'ALABEL' — an ACE-prefixed label that encodes a valid IDN label. +-- * 'ULABEL' — a non-ASCII label that can be part of a valid IDN. +-- * 'ATTRLEAF' — an underscore-prefixed label (e.g. @_25._tcp@). +-- * 'OCTET' — a label with characters outside the LDH alphabet. +-- * 'WILDLABEL' — the DNS wildcard label @*@. +-- * 'LAXULABEL' — a U-label that fails strict IDN validation; +-- not admitted unless the caller explicitly opts in. +-- +-- A tenth sentinel ('NoLabel') is returned by 'getLabelForm' for +-- out-of-range indices (every index for the root domain which has +-- zero labels). The parser never produces 'NoLabel' for a real +-- label, and it is not a member of any 'LabelFormSet'. + +-- $labelformset +-- 'LabelFormSet' is the parser's @allowed@ argument: a set of +-- 'LabelForm' values the parser is willing to admit. + +-- $flags +-- 'IdnaFlags' is a bitmask of parser options. +-- +-- The default ('defaultIdnaFlags') is 'ALABELCHECK' '<>' +-- 'NFCCHECK' '<>' 'BIDICHECK': enforce strict A-label round-trip, +-- require NFC, and apply the cross-label Bidi rules. Additional +-- non-default flags include: +-- +-- * 'MAPCASE' - Fold input to lower case. +-- * 'MAPDOTS' - Accept additional Unicode label separators. +-- * 'MAPWIDTH' - Map wide/narrow codepoints. +-- * 'MAPNFC' - Normalise input to composed form (NFC). +-- * 'MAPUTS46' - A small set of UTS #46 mappings plus all the above. +-- * 'EMOJIOK' - Accept most emoji in labels. +-- * 'ASCIIFALLBACK' - Decode to A-labels when a domain violates BIDI rules + +-- $cli +-- Convenience functions for command-line option parsers. +-- +-- Parse comma-separated lists with @+@\/@-@ prefixes and accept +-- either an exact match or any unambiguous three-or-more letter +-- prefix. diff --git a/src/Text/IDNA2008/Wire.hs b/src/Text/IDNA2008/Wire.hs new file mode 100644 --- /dev/null +++ b/src/Text/IDNA2008/Wire.hs @@ -0,0 +1,66 @@ +-- | +-- Module : Text.IDNA2008.Wire +-- Description : Renderers operating on DNS wire-form bytes, for +-- callers that already hold validated wire bytes. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- == Trust contract +-- +-- Every function in this module accepts a 'ShortByteString' that +-- the caller asserts is a well-formed DNS wire-form domain name: +-- +-- * Total length in @[1, 255]@. +-- * Each label preceded by a one-byte length in @[1, 63]@. +-- * Terminated by the zero-length root label. +-- * No DNS compression pointers, no reserved length-byte values. +-- +-- The input is checked to consist of length-byte-prefixed +-- wire-form labels of up to 63 bytes each, not exceeding 255 bytes +-- total, terminated by a final 0-length label. +-- +-- == When to use this module +-- +-- The intended consumer is a downstream library or application +-- that has /already/ validated the wire bytes via its own type +-- system or parser — for instance, a DNS protocol library that +-- carries its own wire-form 'Data.ByteString.Short.ShortByteString' +-- newtype and wants to render names through IDNA2008 without +-- re-validating on every call. +module Text.IDNA2008.Wire + ( toUnicode + , toAscii + , unparseOpts + ) where + +import Data.ByteString.Short (ShortByteString) +import Data.Text (Text) + +import qualified Text.IDNA2008.Internal.Parse as P +import Text.IDNA2008.Internal.Error (IdnaError) +import Text.IDNA2008.Internal.Flags (IdnaFlags) +import Text.IDNA2008.Internal.LabelFormSet (LabelFormSet) +import Text.IDNA2008.Internal.LabelInfo (LabelInfo) + +-- | Render DNS wire-form bytes as Unicode presentation 'Text'. +-- Behaviour is identical to that of 'P.domainToUnicode'. +toUnicode :: ShortByteString -> Text +toUnicode = P.domainToUnicode . P.Domain + +-- | Render DNS wire-form bytes as ASCII presentation 'Text'. +-- Behaviour is identical to that of 'P.domainToAscii'. +toAscii :: ShortByteString -> Text +toAscii = P.domainToAscii . P.Domain + +-- | Render DNS wire-form bytes under caller-chosen 'LabelFormSet' +-- and 'IdnaFlags', paired with the resulting 'LabelInfo'. +-- Behaviour is identical to that of 'P.unparseDomainOpts'. +unparseOpts + :: LabelFormSet + -> IdnaFlags + -> ShortByteString + -> Either IdnaError (Text, LabelInfo) +unparseOpts forms flags = P.unparseDomainOpts forms flags . P.Domain diff --git a/tests/README.md b/tests/README.md new file mode 100644 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,303 @@ +# Conformance test vectors + +This directory holds the conformance test vectors for the +`idna2008` library and the Haskell-side test driver that +consumes them. + +The vectors are deliberately stored in a language-agnostic JSON +file (`vectors.json`) so future ports of the library to other +languages -- Rust, C++, Go, etc. -- can read the same file and +validate that they produce identical results. + +## Files + +| File | Contents | +| --- | --- | +| `vectors.json` | The test vectors. Source of truth. | +| `conformance.hs` | The Haskell test driver. Built as a `tasty` test-suite by `cabal test`. | +| `README.md` | This document. | + +## Running + +``` +cabal test conformance +``` + +## Vector schema + +A vector file is a JSON object: + +```json +{ + "version": 1, + "description": "...", + "tests": [ ... ] +} +``` + +`version` is an integer; the current schema is `1`. Each entry +in `tests` is an object describing one parser run: + +```json +{ + "name": "short-unique-id", + "input": "the domain name as the user types it", + "classes": "host", + "flags": "default", + "expect": { ... } +} +``` + +### `input` + +The presentation-form domain name. A JSON string, so the +encoding is UTF-8. Most non-ASCII content appears inline as raw +UTF-8 bytes for readability. Three categories of content, +however, **must** be written using JSON `\uXXXX` escapes: + + * Combining marks and other non-NFC sequences. Many editors + and terminals silently NFC-normalise on save or on display, + which would corrupt vectors whose whole point is to + exercise non-NFC input. + * Right-to-left codepoints (Hebrew, Arabic, etc.). Editor and + terminal Bidi reordering means the on-disk byte order is + not what a reader sees, which makes inline RTL content + unsafe to edit by hand. Escaping it forces the source to + read in logical (on-disk) order, the same order the parser + sees. + * Format characters (Unicode category `Cf`, e.g. ZWNJ + U+200C and ZWJ U+200D) and control characters (`Cc`). + These render invisibly or as substitution glyphs, so an + inline occurrence is impossible to verify by eye. + +The test driver sees the same Unicode characters either way. + +### `classes` + +A CLI-style token list selecting which `LabelForm` values the +parser should accept. Optional; defaults to `"host"` +(letter-digit-hyphen plus reserved-LDH, A-labels, U-labels, and +fake A-labels -- the permissive set for hostname-shaped input). + +Token names: `ldh`, `rldh`, `fakea`, `alabel`, `ulabel`, +`attrleaf`, `octet`, `wildlabel`, `laxulabel`. Presets: `idn` +(LDH + ALABEL + ULABEL), `host` (the IDN preset plus RLDH and +FAKEA), `all` (every clean-path form -- the eight forms above +/excluding/ `laxulabel`). Admitting `laxulabel` is always an +explicit opt-in (e.g. `"all,+laxulabel"`). + +Comma-separated. Tokens may be prefixed with `+` (additive, +the default) or `-` (subtractive). See `parseLabelFormSet` in +`Text.IDNA2008.Internal.LabelFormSet` for the full grammar. + +### `flags` + +A CLI-style token list selecting parser flags. Optional; +defaults to `"default"`, which is `ALABELCHECK` + `NFCCHECK` ++ `BIDICHECK`. + +Token names: `alabel-check`, `nfc-check`, `emoji-ok`, +`map-dots`, `map-nfc`, `map-case`, `map-width`, `bidi-check`, +`ascii-fallback`, `map-uts46`. Preset: `map` (the four +RFC 5895 mapping flags combined; does /not/ include +`map-uts46`). + +Same syntax as `classes`. See `parseIdnaFlags` in +`Text.IDNA2008.Internal.Flags`. + +#### Editorial flags + +`emoji-ok` (admits a subset of `Emoji=Yes` codepoints, excluding +the UTS #46 mapped-emoji subset) and `map-uts46` (applies the +hand-curated 57-entry UTS #46 mapping subset documented in +`Text.IDNA2008.Internal.UTS46`) are this library's editorial +extensions beyond strict IDNA2008 / RFC 5895. Vectors exercising +them define what /this library/ does and ports of `idna2008` are +expected to match byte-for-byte; they do /not/ define what every +IDNA implementation should do, and other libraries (libidn2, +ICU, etc.) deliberately diverge on this surface. + +### `expect` + +Exactly one of `ok` (the parse should succeed and produce a +specific result) or `err` (the parse should fail with a +specific error). + +#### `expect.ok` + +```json +{ + "ok": { + "wireHex": "037777770765 ... 00", + "classes": ["WILDLABEL", "LDH", "LDH"] + } +} +``` + +`wireHex` is the expected wire-form bytes as a hexadecimal +string. Whitespace inside the hex is allowed and ignored. +The wire form is the standard DNS uncompressed encoding: each +label preceded by a one-byte length, terminated by a +zero-length root label. + +`classes` is the per-label classification: a list with one +entry per label in the parsed name, in the order the labels +appear in the input. Each entry is one of `LDH`, `RLDH`, +`FAKEA`, `ALABEL`, `ULABEL`, `ATTRLEAF`, `OCTET`, `WILDLABEL`, +or (only when the test's `classes` admission set explicitly +opts in) `LAXULABEL`. An empty array means the input was +the root domain (zero labels). + +For example, `*.example.com` parses to three labels in order +`*`, `example`, `com`, so `classes` is +`["WILDLABEL", "LDH", "LDH"]`. + +##### Rendering-direction checks (optional) + +Three further optional fields inside `expect.ok` exercise the +ToUnicode side of the library, run only after the parse succeeds +and the wire form and per-label classes have matched: + +```json +{ + "ok": { + "wireHex": "...", + "classes": [...], + + "displayForm": "münchen.example", + "displayFormLax": "💩.example", + "displayFormAscii": "xn--mnchen-3ya.example", + "displayFormOpts": { + "flags": "bidi-check", + "ok": "münchen.example" + } + } +} +``` + +`displayForm` is compared against the result of `domainToUnicode` +applied to the parsed `Domain`. Strict: an ACE label that +doesn't round-trip is rendered verbatim (as its `xn--` form); +non-LDH content in an OCTET label is escaped per zone-file +conventions (`\\C` for syntactic specials, `\\DDD` for other +non-printable / non-ASCII bytes). + +`displayFormLax` is compared against a test-local +`domainToUnicodeLax` that the test driver builds on top of +`unparseDomainOpts` with a permissive `LabelFormSet` (every +clean-path form plus `LAXULABEL`). This admits labels whose +Punycode body decodes to codepoints that strict IDNA2008 would +reject (`FAKEA` ACE bodies surface as their decoded codepoints +rather than their `xn--` form) while preserving the cross-label +Bidi check and the rest of the strict-form treatment. + +`displayFormAscii` is compared against `domainToAscii`, which +keeps every label in ASCII form: `xn--` labels stay literal +(no decoding back to U-label), and OCTET / WILDLABEL / ATTRLEAF +bytes get the same `\\C`-or-`\\DDD` escape treatment as in the +other two paths. Useful for callers who want the on-the-wire +text representation (zone files, DNS-bound output). + +`displayFormOpts` invokes `unparseDomainOpts` with a caller- +supplied flag set and (optionally) a caller-supplied +`LabelFormSet`. The shape is: + +```json +"displayFormOpts": { + "flags": "bidi-check", + "classes": "host,+laxulabel", + "ok": "..." +} +``` + +`flags` uses the same command-line option syntax as the top-level +`flags` field. The most useful ones at render time are: + + * `BIDICHECK` enables the cross-label RFC 5893 check. This + is on by default (`defaultIdnaFlags` is the seed when no + leading sign is given). + * `ASCIIFALLBACK` renders the whole domain in its all-ASCII + A-label form when the Bidi check would otherwise fail, + instead of producing an error. + * `EMOJIOK` admits emoji codepoints when checking the + decoded form of an `xn--` label. + * `NFCCHECK` requires the decoded form to be in NFC. + +`classes` is optional; absent, the renderer reuses the +admission set from the parse step. Including `LAXULABEL` +(`"...,+laxulabel"`) admits U-labels that fail strict IDN +validation and, when any such label is actually present, +suppresses the cross-label Bidi check on the rendered output +(since `LAXULABEL` labels can carry codepoints outside the +Bidi rule's purview). + +The `displayFormOpts` object must specify exactly one of `ok` +(the call should return `Right`) or `err` (the call should +return `Left`). The `err` shape is the same as `expect.err` (see +below). + +Multiple rendering fields can coexist; each is checked +independently. Absent fields are no-ops. + +The root domain renders as the literal `"."` under all +three renderers. + +#### `expect.err` + +```json +{ + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "LabelBidi", + "rule": "BidiRule1FirstNotLRAL" + } +} +``` + +`kind` is required; everything else is optional. Only fields +present in the vector are checked, so a vector that says +`"kind": "ErrEmptyLabel"` and nothing else will accept any +`ErrEmptyLabel`-shaped error regardless of label index. + +| Field | Applies to | Type | +| --- | --- | --- | +| `kind` | every error | one of `ErrEmptyLabel`, `ErrLabelTooLong`, `ErrNameTooLong`, `ErrBadEscape`, `ErrInvalidUtf8`, `ErrCodepointTooLarge`, `ErrUnpresentableLabel`, `ErrFormNotAllowed`, `ErrLabelInvalid`, `ErrAceInvalid`, `ErrPunycodeOverflow`, `ErrCrossLabelBidi` | +| `labelIndex` | most errors | integer; `-1` is reported as absent | +| `reason` | `ErrLabelInvalid`, `ErrAceInvalid` | one of `DisallowedCodepoint`, `ContextRule`, `NotNFC`, `LabelBidi`, `HyphenViolation`, `LeadingCombiningMark`, `BadPunycode`, `DecodedInvalid`, `RoundTripMismatch` | +| `innerReason` | `ErrAceInvalid (DecodedInvalid ...)` | a `LabelReason` tag | +| `rule` | `ErrLabelInvalid (LabelBidi ...)`, `ErrCrossLabelBidi` | `BidiRule1FirstNotLRAL` ... `BidiRule6LTRBadEnd` | +| `codepoint` | `ErrCodepointTooLarge`, `ErrLabelInvalid (DisallowedCodepoint ...)`, `ErrLabelInvalid (ContextRule ...)` | integer codepoint | +| `length` | `ErrLabelTooLong`, `ErrNameTooLong` | integer byte length | + +## Adding vectors + +Every new vector should pin down a specific behaviour: a known +positive case, a regression for a bug fix, or a known negative +case for a particular spec rule. Vector names are free-form +identifiers; pick something specific +(`bidi-rule-3-trailing-mark-rejected` beats `test42`). + +When adding positive vectors with non-trivial wire forms (IDN +encodings via Punycode), it's tempting to compute the expected +hex by running the library itself. That's fine /once/ -- for a +fresh build of the same library -- but turns the vector into a +self-test rather than a conformance test. Where possible, +cross-check the wire form against a second implementation +(another IDNA library, the IANA test vectors, RFC 5891 / 5893 +worked examples). + +## Cross-language ports + +When porting the library to another language, point the new +implementation's test runner at this same `vectors.json`. The +schema is intentionally simple enough to parse with whatever +JSON facility the language has, and the comparisons are +bytewise (wire form, per-label class names, error tags) so +there's no ambiguity in what counts as "matching". + +The per-label `classes` array assumes the implementation +exposes its label classification as a vector (or array, or +list) indexed in label order. A port whose API only returns a +union bitmask should derive the per-label vector during +parsing rather than throwing the per-label information away. diff --git a/tests/conformance.hs b/tests/conformance.hs new file mode 100644 --- /dev/null +++ b/tests/conformance.hs @@ -0,0 +1,954 @@ +-- | +-- Conformance test harness for the @idna2008@ library. Reads +-- a JSON-encoded list of test vectors from @tests/vectors.json@ +-- and runs each through the public parser, comparing the actual +-- result to the expected one. +-- +-- Exit code: zero on full pass, non-zero on any failure. +-- +-- The same JSON file is intended for reuse by ports of the +-- library to other languages; the schema is documented in +-- @tests/README.md@. +{-# LANGUAGE OverloadedStrings #-} + +module Main (main) where + +import qualified Data.Aeson as A +import qualified Data.ByteString as BS +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Test.Tasty as Tasty +import Control.Monad (when) +import Data.Aeson (FromJSON(..), (.:), (.:?), withObject) +import Data.Bits ((.&.), shiftR) +import Data.ByteString (ByteString) +import Data.Char (chr, ord) +import Data.Foldable (foldMap') +import Data.Text (Text) +import Data.Word (Word8) +import System.Exit (exitFailure) +import Test.Tasty.HUnit (assertFailure, testCase) + +import Text.IDNA2008 + ( BidiRuleViolation(..) + , Domain + , IdnaError(..) + , IdnaFlags(..) + , LabelReason(..) + , AceReason(..) + , LabelForm(..) + , LabelFormSet + , defaultIdnaFlags + , domainToAscii + , domainToUnicode + , getLabelForms + , hostnameLabelForms + , isValidWireForm + , labelFormToSet + , labelToAscii + , labelToUnicode + , mkDomain + , mkDomainShort + , mkDomainStr + , mkDomainUtf8 + , parseDomain + , parseDomainOpts + , parseDomainShort + , parseDomainUtf8 + , parseIdnaFlags + , parseLabelFormSet + , unparseDomainOpts + , unparseLabelOpts + , wireBytes + ) + +import qualified Data.ByteString.Short as SBS +import Data.ByteString.Short (ShortByteString) + +---------------------------------------------------------------------- +-- Test-local equivalents of the (now-retired) lax renderers. +-- +-- These exist purely for the conformance test framework, which +-- pins per-vector @displayFormLax@ expectations against a +-- permissive renderer that decodes @\"xn--\"@ labels regardless +-- of whether they pass strict IDN validation. The library API +-- expresses the same behaviour by including 'LAXULABEL' (and +-- 'FAKEA' as the Punycode-failure fallback) in the +-- 'LabelFormSet'. +---------------------------------------------------------------------- + +domainToUnicodeLax :: Domain -> Text +domainToUnicodeLax dom = case unparseDomainOpts laxForms defaultIdnaFlags dom of + Right (t, _) -> t + Left _ -> error "domainToUnicodeLax: unreachable" + +labelToUnicodeLax :: ShortByteString -> Text +labelToUnicodeLax sbs = case unparseLabelOpts laxForms defaultIdnaFlags sbs of + Right (t, _) -> t + Left _ -> error "labelToUnicodeLax: unreachable" + +laxForms :: LabelFormSet +laxForms = foldMap' labelFormToSet + [LDH, RLDH, ULABEL, LAXULABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + +---------------------------------------------------------------------- +-- JSON schema (mirrors tests/README.md) +---------------------------------------------------------------------- + +data VectorFile = VectorFile + { vfVersion :: !Int + , vfTests :: ![Vector] + } deriving (Show) + +instance FromJSON VectorFile where + parseJSON = withObject "VectorFile" \o -> VectorFile + <$> o .: "version" + <*> o .: "tests" + +data Vector = Vector + { vName :: !Text + , vInput :: !Text + , vClasses :: !(Maybe Text) -- ^ default: \"hostname\" + , vFlags :: !(Maybe Text) -- ^ default: \"default\" + , vExpect :: !Expected + } deriving (Show) + +instance FromJSON Vector where + parseJSON = withObject "Vector" \o -> Vector + <$> o .: "name" + <*> o .: "input" + <*> o .:? "classes" + <*> o .:? "flags" + <*> o .: "expect" + +data Expected + = ExpectOk !OkBody + | ExpectErr !ErrBody + deriving (Show) + +instance FromJSON Expected where + parseJSON = withObject "Expected" \o -> do + mOk <- o .:? "ok" + mErr <- o .:? "err" + case (mOk, mErr) of + (Just ok, Nothing) -> pure (ExpectOk ok) + (Nothing, Just er) -> pure (ExpectErr er) + (Just _, Just _ ) -> fail "vector specifies both ok and err" + (Nothing, Nothing) -> fail "vector specifies neither ok nor err" + +data OkBody = OkBody + { okWireHex :: !Text + , okClasses :: ![Text] -- ^ per-label, in label order + , okDisplayForm :: !(Maybe Text) -- ^ 'domainToUnicode' result, if pinned + , okDisplayFormLax :: !(Maybe Text) -- ^ 'domainToUnicodeLax' result, if pinned + , okDisplayFormAscii :: !(Maybe Text) -- ^ 'domainToAscii' result, if pinned + , okDisplayFormOpt :: !(Maybe ExpectOpt) -- ^ 'unparseDomainOpts' run + outcome + } deriving (Show) + +instance FromJSON OkBody where + parseJSON = withObject "OkBody" \o -> OkBody + <$> o .: "wireHex" + <*> o .: "classes" + <*> o .:? "displayForm" + <*> o .:? "displayFormLax" + <*> o .:? "displayFormAscii" + <*> o .:? "displayFormOpts" + +-- | Specification for an 'unparseDomainOpts' check. The 'flags' +-- field selects the 'IdnaFlags' bitmask to pass; 'BIDICHECK' and +-- 'ASCIIFALLBACK' have an effect at this stage (parse-time flags +-- are baked into the 'Domain' already). The optional 'classes' +-- field overrides the 'LabelFormSet': if absent, the parser-side +-- 'classes' is reused as-is; if present with a leading @\'+\'@ or +-- @\'-\'@ token, it adjusts the parser-side set; otherwise it +-- replaces it. Exactly one of 'ok' or 'err' must be set. +data ExpectOpt = ExpectOpt + { eoFlags :: !Text + , eoClasses :: !(Maybe Text) + , eoOk :: !(Maybe Text) + , eoErr :: !(Maybe ErrBody) + } deriving (Show) + +instance FromJSON ExpectOpt where + parseJSON = withObject "ExpectOpt" \o -> ExpectOpt + <$> o .: "flags" + <*> o .:? "classes" + <*> o .:? "ok" + <*> o .:? "err" + +-- | Flat schema for error-shape vectors. Any field may be +-- omitted; only fields present in the vector are checked. +data ErrBody = ErrBody + { erKind :: !Text -- ^ required: ErrEmptyLabel, ErrCrossLabelBidi, ... + , erLabelIndex :: !(Maybe Int) + , erRule :: !(Maybe Text) -- ^ BidiRuleViolation tag + , erReason :: !(Maybe Text) -- ^ LabelReason or AceReason tag + , erInnerReason :: !(Maybe Text) -- ^ inner LabelReason inside DecodedInvalid + , erCodepoint :: !(Maybe Int) + , erLength :: !(Maybe Int) + } deriving (Show) + +instance FromJSON ErrBody where + parseJSON = withObject "ErrBody" \o -> ErrBody + <$> o .: "kind" + <*> o .:? "labelIndex" + <*> o .:? "rule" + <*> o .:? "reason" + <*> o .:? "innerReason" + <*> o .:? "codepoint" + <*> o .:? "length" + +---------------------------------------------------------------------- +-- Hex helpers (avoid base16-bytestring dep) +---------------------------------------------------------------------- + +encodeHex :: ByteString -> Text +encodeHex = T.pack . concatMap byteHex . BS.unpack + where + byteHex !b = [hex ((b `shiftR` 4) .&. 0xf), hex (b .&. 0xf)] + hex !n + | n < 10 = chr (ord '0' + fromIntegral n) + | otherwise = chr (ord 'a' + fromIntegral n - 10) + +decodeHex :: Text -> Either String ByteString +decodeHex t0 = go (T.unpack (T.toLower (T.filter (/= ' ') t0))) [] + where + go [] acc = Right (BS.pack (reverse acc)) + go [_] _ = Left "odd-length hex" + go (a:b:rest) acc = do + hi <- hexDigit a + lo <- hexDigit b + go rest (fromIntegral (hi * 16 + lo) : acc) + hexDigit c + | c >= '0' && c <= '9' = Right (ord c - ord '0') + | c >= 'a' && c <= 'f' = Right (ord c - ord 'a' + 10) + | otherwise = Left ("bad hex digit: " ++ [c]) + +---------------------------------------------------------------------- +-- LabelForm -> token-name conversion +---------------------------------------------------------------------- + +-- | Singleton name for each 'LabelForm'. Exhaustive over the +-- COMPLETE pragma in 'Text.IDNA2008.Internal.LabelForm'. +-- | Render any tag-style value by the leading constructor name +-- in its 'Show' output. Works for sum-type constructors with +-- or without payloads because GHC's default 'Show' format +-- always emits the constructor name first, separated from any +-- arguments by a space. +ctorName :: Show a => a -> Text +ctorName = T.pack . takeWhile (/= ' ') . show + +formName :: LabelForm -> Text +formName = ctorName + +---------------------------------------------------------------------- +-- Run a single vector +---------------------------------------------------------------------- + +runVector :: Vector -> IO () +runVector v = do + classes <- resolveClasses (vClasses v) + flags <- resolveFlags (vFlags v) + let !result = parseDomainOpts classes flags (vInput v) + case (vExpect v, result) of + (ExpectOk eok, Right (dom, info)) -> do + expectedWire <- case decodeHex (okWireHex eok) of + Right b -> pure b + Left e -> assertFailure ("vector wireHex: " ++ e) + >> pure BS.empty + let !actualWire = wireBytes dom + when (actualWire /= expectedWire) $ + assertFailure $ unlines + [ "wire mismatch:" + , " expected: " ++ T.unpack (encodeHex expectedWire) + , " actual: " ++ T.unpack (encodeHex actualWire) + ] + let !actualClassNames = map formName (getLabelForms info) + !expectedClassNames = okClasses eok + when (actualClassNames /= expectedClassNames) $ + assertFailure $ unlines + [ "per-label form mismatch:" + , " expected: " ++ show expectedClassNames + , " actual: " ++ show actualClassNames + ] + -- Optional rendering-side checks. Each runs only when the + -- corresponding field is present in the vector. + checkDisplayForm "displayForm" domainToUnicode dom + (okDisplayForm eok) + checkDisplayForm "displayFormLax" domainToUnicodeLax dom + (okDisplayFormLax eok) + checkDisplayForm "displayFormAscii" domainToAscii dom + (okDisplayFormAscii eok) + checkDisplayFormOpt classes dom (okDisplayFormOpt eok) + (ExpectOk _, Left err) -> + assertFailure ("expected ok, got error: " ++ show err) + (ExpectErr ee, Left err) -> + checkErr ee err + (ExpectErr ee, Right r) -> + assertFailure + ("expected error " ++ T.unpack (erKind ee) + ++ ", got ok: " ++ show r) + +resolveClasses :: Maybe Text -> IO LabelFormSet +resolveClasses Nothing = pure hostnameLabelForms +resolveClasses (Just t) = + case parseLabelFormSet hostnameLabelForms (T.encodeUtf8 t) of + Right c -> pure c + Left e -> mempty <$ assertFailure ("classes: " ++ e) + +resolveFlags :: Maybe Text -> IO IdnaFlags +resolveFlags Nothing = pure defaultIdnaFlags +resolveFlags (Just t) = + case parseIdnaFlags defaultIdnaFlags (T.encodeUtf8 t) of + Right f -> pure f + Left e -> do _ <- assertFailure ("flags: " ++ e); pure defaultIdnaFlags + +---------------------------------------------------------------------- +-- Rendering-side comparisons +---------------------------------------------------------------------- + +-- | Helper for the two infallible renderers ('domainToUnicode', +-- 'domainToUnicodeLax'). Runs the renderer if the vector pins +-- a value; otherwise no-op. +checkDisplayForm + :: String -- ^ field name, for diagnostics + -> (Domain -> Text) -- ^ renderer + -> Domain -- ^ parsed domain + -> Maybe Text -- ^ expected, if pinned + -> IO () +checkDisplayForm _ _ _ Nothing = pure () +checkDisplayForm field run dom (Just want) = + let !got = run dom + in when (got /= want) $ + assertFailure $ unlines + [ field ++ " mismatch:" + , " expected: " ++ show want + , " actual: " ++ show got + ] + +-- | Run 'unparseDomainOpts' with the vector-supplied flags and +-- 'LabelFormSet' and compare the outcome (ok or err) to what the +-- vector pins. No-op if the field is absent. The parser-side +-- 'LabelFormSet' is threaded in as the default for an optional +-- @classes@ override in the @displayFormOpts@ block: a leading +-- @\'+\'@ or @\'-\'@ in that override adjusts the inherited set, +-- otherwise the override replaces it cleanly. +checkDisplayFormOpt :: LabelFormSet -> Domain -> Maybe ExpectOpt -> IO () +checkDisplayFormOpt _ _ Nothing = pure () +checkDisplayFormOpt parserForms dom (Just o) = do + flags <- resolveOptFlags (eoFlags o) + forms <- resolveOptClasses parserForms (eoClasses o) + let !got = fmap fst (unparseDomainOpts forms flags dom) + case (eoOk o, eoErr o, got) of + (Just want, Nothing, Right actual) -> + when (actual /= want) $ + assertFailure $ unlines + [ "displayFormOpts mismatch:" + , " expected: " ++ show want + , " actual: " ++ show actual + ] + (Just want, Nothing, Left err) -> + assertFailure $ + "displayFormOpts: expected ok " ++ show want + ++ ", got error: " ++ show err + (Nothing, Just ee, Left err) -> + checkErr ee err + (Nothing, Just ee, Right actual) -> + assertFailure $ + "displayFormOpts: expected error " ++ T.unpack (erKind ee) + ++ ", got ok: " ++ show actual + (Nothing, Nothing, _) -> + assertFailure "displayFormOpts: vector specifies neither ok nor err" + (Just _, Just _, _) -> + assertFailure "displayFormOpts: vector specifies both ok and err" + where + -- displayFormOpts flags share the same command-line option + -- syntax as the top-level 'flags' field; 'BIDICHECK' and + -- 'ASCIIFALLBACK' control cross-label Bidi behaviour at this stage. + resolveOptFlags :: Text -> IO IdnaFlags + resolveOptFlags t = + case parseIdnaFlags defaultIdnaFlags (T.encodeUtf8 t) of + Right f -> pure f + Left e -> do _ <- assertFailure ("displayFormOpts.flags: " ++ e) + pure defaultIdnaFlags + + -- Optional classes override: absent → use the parser-side + -- 'LabelFormSet' unchanged; present → parse with the + -- parser-side set as the default, so a leading @\'+\'@ or + -- @\'-\'@ token adjusts the inherited set and a bare token + -- list replaces it. + resolveOptClasses :: LabelFormSet -> Maybe Text -> IO LabelFormSet + resolveOptClasses base Nothing = pure base + resolveOptClasses base (Just t) = + case parseLabelFormSet base (T.encodeUtf8 t) of + Right f -> pure f + Left e -> do _ <- assertFailure ("displayFormOpts.classes: " ++ e) + pure base + +---------------------------------------------------------------------- +-- Error matching: inspect actual error, compare to vector's +-- per-field expectations (each optional) +---------------------------------------------------------------------- + +checkErr :: ErrBody -> IdnaError -> IO () +checkErr ee err = do + let !actualKind = errKind err + when (erKind ee /= actualKind) $ + mismatch "kind" (erKind ee) actualKind + checkOpt "labelIndex" erLabelIndex (errLabelIndex err) (T.pack . show) + checkOpt "rule" erRule (errBidiRule err) id + checkOpt "reason" erReason (errReason err) id + checkOpt "innerReason" erInnerReason (errInnerReason err) id + checkOpt "codepoint" erCodepoint (errCodepoint err) (T.pack . show) + checkOpt "length" erLength (errLength err) (T.pack . show) + where + checkOpt + :: (Eq a) + => Text + -> (ErrBody -> Maybe a) + -> Maybe a + -> (a -> Text) + -> IO () + checkOpt field getWant got render = + case getWant ee of + Nothing -> pure () + Just w -> case got of + Just g | g == w -> pure () + | otherwise -> mismatch field (render w) (render g) + Nothing -> mismatch field (render w) "" + + mismatch field want got = + assertFailure $ T.unpack $ T.concat + [ "error ", field, " mismatch: expected " + , want, ", got ", got + , " (full error: ", T.pack (show err), ")" + ] + +errKind :: IdnaError -> Text +errKind = ctorName + +errLabelIndex :: IdnaError -> Maybe Int +errLabelIndex e = case e of + ErrEmptyLabel loc -> fromLoc loc + ErrLabelTooLong loc _ -> fromLoc loc + ErrNameTooLong _ -> Nothing + ErrBadEscape loc _ -> fromLoc loc + ErrInvalidUtf8 loc _ -> fromLoc loc + ErrCodepointTooLarge loc _ -> fromLoc loc + ErrUnpresentableLabel loc -> fromLoc loc + ErrFormNotAllowed loc _ -> fromLoc loc + ErrLabelInvalid loc _ -> fromLoc loc + ErrAceInvalid loc _ -> fromLoc loc + ErrPunycodeOverflow loc -> fromLoc loc + ErrCrossLabelBidi i _ -> Just i + where + fromLoc loc + | loc < 0 = Nothing + | otherwise = Just loc + +errBidiRule :: IdnaError -> Maybe Text +errBidiRule (ErrCrossLabelBidi _ r) = Just (bidiRuleName r) +errBidiRule (ErrLabelInvalid _ (LabelBidi r)) = Just (bidiRuleName r) +errBidiRule _ = Nothing + +errReason :: IdnaError -> Maybe Text +errReason (ErrLabelInvalid _ r) = Just (labelReasonName r) +errReason (ErrAceInvalid _ r) = Just (aceReasonName r) +errReason _ = Nothing + +errInnerReason :: IdnaError -> Maybe Text +errInnerReason (ErrAceInvalid _ (DecodedInvalid r)) = Just (labelReasonName r) +errInnerReason _ = Nothing + +errCodepoint :: IdnaError -> Maybe Int +errCodepoint (ErrLabelInvalid _ r) = labelReasonCp r +errCodepoint (ErrAceInvalid _ (DecodedInvalid r)) = labelReasonCp r +errCodepoint (ErrCodepointTooLarge _ cp) = Just cp +errCodepoint _ = Nothing + +labelReasonCp :: LabelReason -> Maybe Int +labelReasonCp (DisallowedCodepoint cp) = Just cp +labelReasonCp (ContextRule cp) = Just cp +labelReasonCp (LeadingCombiningMark cp) = Just cp +labelReasonCp _ = Nothing + +errLength :: IdnaError -> Maybe Int +errLength (ErrLabelTooLong _ n) = Just n +errLength (ErrNameTooLong n) = Just n +errLength _ = Nothing + +bidiRuleName :: BidiRuleViolation -> Text +bidiRuleName = ctorName + +labelReasonName :: LabelReason -> Text +labelReasonName = ctorName + +aceReasonName :: AceReason -> Text +aceReasonName = ctorName + +---------------------------------------------------------------------- +-- Driver +---------------------------------------------------------------------- + +main :: IO () +main = do + raw <- BS.readFile "tests/vectors.json" + case A.eitherDecodeStrict raw of + Left err -> do + putStrLn ("vectors.json: parse error: " ++ err) + exitFailure + Right vf -> do + when (vfVersion vf /= 1) $ do + putStrLn ("vectors.json: unsupported version " + ++ show (vfVersion vf)) + exitFailure + Tasty.defaultMain $ Tasty.testGroup "idna2008 tests" + [ Tasty.testGroup "conformance" + [ testCase (T.unpack (vName v)) (runVector v) + | v <- vfTests vf + ] + , labelTests + , domainWireTests + , parserEntryPointTests + , invalidUtf8Tests + , mkDomainFamilyTests + , tokenVocabularyTests + ] + +---------------------------------------------------------------------- +-- Per-label rendering tests. +-- +-- Domain-level semantics (parse-then-render of full names) live +-- in the JSON-driven 'conformance' group above. These hand-built +-- label-level cases exercise 'labelToAscii', 'labelToUnicode', +-- and 'labelToUnicodeLax' in isolation, with the wire-form +-- 'ShortByteString' constructed directly (no parser in the +-- loop). +---------------------------------------------------------------------- + +labelTests :: Tasty.TestTree +labelTests = Tasty.testGroup "labels" + [ Tasty.testGroup "labelToAscii" + [ testCase "LDH passthrough" $ + assertLabel labelToAscii (ascii "example") "example" + , testCase "ATTRLEAF passthrough" $ + assertLabel labelToAscii (ascii "_dmarc") "_dmarc" + , testCase "WILDLABEL passthrough" $ + assertLabel labelToAscii (ascii "*") "*" + , testCase "ACE label stays literal" $ + assertLabel labelToAscii + (ascii "xn--mnchen-3ya") "xn--mnchen-3ya" + , testCase "every special escaped" $ + assertLabel labelToAscii + (ascii "\"$().;@\\") + (T.pack "\\\"\\$\\(\\)\\.\\;\\@\\\\") + , testCase "control bytes -> DDD" $ + assertLabel labelToAscii + (SBS.pack [0x00, 0x09, 0x20, 0x7F]) + (T.pack "\\000\\009\\032\\127") + , testCase "high bytes -> DDD" $ + assertLabel labelToAscii + (SBS.pack [0x80, 0xC2, 0xFF]) + (T.pack "\\128\\194\\255") + ] + , Tasty.testGroup "labelToUnicode" + [ testCase "LDH passthrough" $ + assertLabel labelToUnicode (ascii "example") "example" + , testCase "real A-label decodes" $ + assertLabel labelToUnicode + (ascii "xn--mnchen-3ya") (T.pack "m\252nchen") + , testCase "FAKEA emoji stays in ACE" $ + assertLabel labelToUnicode + (ascii "xn--ls8h") "xn--ls8h" + , testCase "FAKEA control stays in ACE" $ + assertLabel labelToUnicode + (ascii "xn--a") "xn--a" + , testCase "specials escaped same as ASCII" $ + assertLabel labelToUnicode + (ascii "\"$().;@\\") + (T.pack "\\\"\\$\\(\\)\\.\\;\\@\\\\") + ] + , Tasty.testGroup "labelToUnicodeLax" + [ testCase "LDH passthrough" $ + assertLabel labelToUnicodeLax (ascii "example") "example" + , testCase "real A-label decodes" $ + assertLabel labelToUnicodeLax + (ascii "xn--mnchen-3ya") (T.pack "m\252nchen") + , testCase "FAKEA emoji decoded" $ + assertLabel labelToUnicodeLax + (ascii "xn--ls8h") (T.singleton '\x1F4A9') + , testCase "FAKEA control decoded" $ + assertLabel labelToUnicodeLax + (ascii "xn--a") (T.singleton '\x80') + -- The body @\"foo$bar--e6a\"@ is structurally a valid + -- Punycode encoding -- it would Punycode-decode to + -- @\"foo$bar-\252\"@ (the basic prefix @\"foo$bar-\"@ + -- plus a U+00FC insertion). But Punycode only emits + -- LDH bytes; this body contains @\'$\'@, so the bytes + -- on the wire could not have come from any IDN + -- encoder. The classifier therefore routes to 'OCTET' + -- on byte shape alone, and the renderer keeps the + -- literal form with the @\'$\'@ zone-file-escaped -- + -- regardless of any 'LAXULABEL' admission. + , testCase "OCTET xn-- with non-LDH ASCII inside Punycode body" $ + assertLabel labelToUnicodeLax + (ascii "xn--foo$bar--e6a") + (T.pack "xn--foo\\$bar--e6a") + , testCase "undecodable body falls back" $ + assertLabel labelToUnicodeLax + (ascii "xn---hgi") "xn---hgi" + ] + , Tasty.testGroup "unparseLabelOpts" + -- The new fallible per-label primitive. Lets the caller + -- choose between Unicode-rendering ('ULABEL' / 'LAXULABEL') + -- and literal @\"xn--\"@ ('ALABEL' / 'FAKEA') via the + -- 'LabelFormSet'. These tests cover the strict / permissive + -- distinction at the label level. + [ testCase "LDH passthrough" $ + assertUnparseLabel laxForms defaultIdnaFlags + (ascii "example") + (Right ("example", LDH)) + , testCase "clean A-label decodes to ULABEL" $ + assertUnparseLabel laxForms defaultIdnaFlags + (ascii "xn--mnchen-3ya") + (Right (T.pack "m\252nchen", ULABEL)) + , testCase "FAKEA emoji literal under non-lax set" $ + assertUnparseLabel noLaxForms defaultIdnaFlags + (ascii "xn--ls8h") + (Right ("xn--ls8h", FAKEA)) + , testCase "FAKEA emoji Unicode under lax set" $ + assertUnparseLabel laxForms defaultIdnaFlags + (ascii "xn--ls8h") + (Right (T.singleton '\x1F4A9', LAXULABEL)) + , testCase "bad-punycode literal under non-lax set" $ + assertUnparseLabel noLaxForms defaultIdnaFlags + (ascii "xn---hgi") + (Right ("xn---hgi", FAKEA)) + , testCase "strict set rejects FAKEA" $ + assertUnparseLabel ulabelOnlyForms defaultIdnaFlags + (ascii "xn--ls8h") + (Left (ErrFormNotAllowed 0 FAKEA)) + , testCase "EMOJIOK admits emoji as clean ULABEL" $ + assertUnparseLabel laxForms (defaultIdnaFlags <> EMOJIOK) + (ascii "xn--ls8h") + (Right (T.singleton '\x1F4A9', ULABEL)) + , testCase "uppercase ASCII body tolerated" $ + assertUnparseLabel laxForms defaultIdnaFlags + (ascii "xn--Mnchen-3ya") + (Right (T.pack "m\252nchen", ULABEL)) + ] + ] + where + -- Permissive 'LabelFormSet': admits both literal and lax-decoded + -- xn-- variants (parallel to the test-local 'laxForms' defined + -- at the top of the file). + -- + -- The 'noLaxForms' set drops 'LAXULABEL', so a strict-failing + -- xn-- label stays literal as 'FAKEA' instead of decoding lax. + -- + -- The 'ulabelOnlyForms' set drops both 'LAXULABEL' and 'FAKEA', + -- so a strict-failing xn-- label rejects with 'ErrFormNotAllowed'. + noLaxForms = foldMap' labelFormToSet + [LDH, RLDH, ULABEL, ALABEL, FAKEA, ATTRLEAF, WILDLABEL, OCTET] + ulabelOnlyForms = foldMap' labelFormToSet + [LDH, RLDH, ULABEL, ATTRLEAF, WILDLABEL, OCTET] + +-- | Wire-form 'ShortByteString' from a plain ASCII 'String'. +ascii :: String -> ShortByteString +ascii = SBS.pack . map (toEnum . fromEnum) + +-- | Assert that a per-label renderer produces the expected 'Text'. +assertLabel + :: (ShortByteString -> Text) + -> ShortByteString + -> Text + -> IO () +assertLabel f input expected = + let got = f input + in if got == expected + then pure () + else assertFailure $ + "expected " ++ show expected ++ "\n got " ++ show got + +-- | Assert that 'unparseLabelOpts' produces the expected +-- @'Either' 'IdnaError' ('Text', 'LabelForm')@ on the given +-- wire-form 'ShortByteString' under the supplied 'LabelFormSet' +-- and 'IdnaFlags'. +assertUnparseLabel + :: LabelFormSet + -> IdnaFlags + -> ShortByteString + -> Either IdnaError (Text, LabelForm) + -> IO () +assertUnparseLabel forms flags input expected = + let got = unparseLabelOpts forms flags input + in if got == expected + then pure () + else assertFailure $ + "expected " ++ show expected ++ "\n got " ++ show got + + +---------------------------------------------------------------------- +-- Smart-constructor / wire-form validation tests. +-- +-- 'isValidWireForm' is the predicate the 'Domain' bidirectional +-- pattern synonym uses to gate its writer side. These cases +-- cover the canonical accept/reject boundary: minimum / maximum +-- lengths, label-length-byte limits, compression-pointer bytes, +-- truncated input, and stray bytes past the root terminator. +---------------------------------------------------------------------- + +domainWireTests :: Tasty.TestTree +domainWireTests = Tasty.testGroup "isValidWireForm" + [ testCase "root domain" $ + assertValid True (SBS.pack [0x00]) + , testCase "single label" $ + assertValid True (lp "abc" <> term) + , testCase "two labels" $ + assertValid True (lp "abc" <> lp "com" <> term) + , testCase "label of length 63" $ + assertValid True (lp (replicate 63 'x') <> term) + , testCase "wire length exactly 255" $ + -- 127 one-char labels (2 bytes each) + 1 terminator = 255 + assertValid True (mconcat (replicate 127 (lp "a")) <> term) + + , testCase "empty input" $ + assertValid False SBS.empty + , testCase "missing terminator" $ + assertValid False (lp "abc") + , testCase "truncated content" $ + assertValid False (SBS.pack [0x03, 0x61, 0x62]) + , testCase "premature root with trailing junk" $ + assertValid False (SBS.pack [0x00, 0x00]) + , testCase "stray byte past the root" $ + assertValid False (lp "abc" <> term <> SBS.pack [0x78]) + , testCase "length byte 64 (reserved range)" $ + assertValid False (SBS.pack [0x40, 0x66, 0x6F, 0x6F, 0x00]) + , testCase "compression pointer byte 0xC0" $ + assertValid False (SBS.pack [0xC0, 0x00]) + , testCase "length byte 0xFF" $ + assertValid False (SBS.pack [0xFF, 0x00]) + , testCase "wire length exceeds 255" $ + -- 128 one-char labels + terminator = 257 + assertValid False (mconcat (replicate 128 (lp "a")) <> term) + ] + where + -- Length-prefixed label from an ASCII 'String'. + lp :: String -> ShortByteString + lp s = SBS.pack (fromIntegral (length s) : map (toEnum . fromEnum) s) + + -- Root terminator. + term :: ShortByteString + term = SBS.pack [0x00] + + assertValid :: Bool -> ShortByteString -> IO () + assertValid expected input = + let got = isValidWireForm input + in if got == expected + then pure () + else assertFailure $ + "expected " ++ show expected + ++ " got " ++ show got + ++ " for input " ++ show input + +---------------------------------------------------------------------- +-- Parser entry-point parity. +-- +-- All four parser entry points ('parseDomain', 'parseDomainOpts', +-- 'parseDomainUtf8', 'parseDomainShort') call through to the same +-- 'parseDomainView' core, so given the same input they should +-- produce identical results. The JSON-driven 'conformance' group +-- only exercises 'parseDomainOpts'; these tests confirm the three +-- wrappers behave correctly too. +---------------------------------------------------------------------- + +parserEntryPointTests :: Tasty.TestTree +parserEntryPointTests = Tasty.testGroup "parser entry-point parity" + [ testCase "valid name agrees across entry points" $ + parserParity "www.example.com" + , testCase "non-ASCII name agrees across entry points" $ + parserParity "münchen.example" + , testCase "invalid name agrees across entry points" $ + parserParity "foo..bar" + , testCase "too-long label agrees across entry points" $ + parserParity (T.replicate 64 "a" <> ".example") + ] + where + parserParity :: T.Text -> IO () + parserParity input = do + let inputBs = T.encodeUtf8 input + inputSbs = SBS.toShort inputBs + r1 = fmap (wireBytes . fst) (parseDomain hostnameLabelForms input) + r2 = fmap (wireBytes . fst) (parseDomainOpts hostnameLabelForms defaultIdnaFlags input) + r3 = fmap (wireBytes . fst) (parseDomainUtf8 hostnameLabelForms defaultIdnaFlags inputBs) + r4 = fmap (wireBytes . fst) (parseDomainShort hostnameLabelForms defaultIdnaFlags inputSbs) + when (not (r1 == r2 && r1 == r3 && r1 == r4)) $ + assertFailure $ unlines + [ "parser entry points disagreed for " ++ show input + , " parseDomain: " ++ show r1 + , " parseDomainOpts: " ++ show r2 + , " parseDomainUtf8: " ++ show r3 + , " parseDomainShort: " ++ show r4 + ] + +---------------------------------------------------------------------- +-- Invalid UTF-8 byte sequences. +-- +-- These inputs carry bare bytes that no JSON string can express +-- (JSON encodes everything as UTF-8 text), so they're driven from +-- Haskell into 'parseDomainUtf8'. Each input targets one branch of +-- the internal UTF-8 decoder ('decodeUtf8At' in +-- 'Text.IDNA2008.Internal.Parse'); all observable outcomes are +-- 'ErrInvalidUtf8'. +---------------------------------------------------------------------- + +invalidUtf8Tests :: Tasty.TestTree +invalidUtf8Tests = Tasty.testGroup "invalid UTF-8 rejected" + [ rejectUtf8 "bare continuation byte" [0x80] + , rejectUtf8 "invalid start byte 0xC0" [0xC0, 0x80] + , rejectUtf8 "invalid start byte 0xC1" [0xC1, 0x80] + , rejectUtf8 "invalid start byte 0xF5" [0xF5, 0x80, 0x80, 0x80] + , rejectUtf8 "invalid start byte 0xFF" [0xFF] + , rejectUtf8 "two-byte truncated" [0xC2] + , rejectUtf8 "two-byte bad continuation" [0xC2, 0x00] + , rejectUtf8 "three-byte truncated" [0xE0, 0xA0] + , rejectUtf8 "three-byte bad continuation" [0xE0, 0x80, 0x00] + , rejectUtf8 "three-byte overlong" [0xE0, 0x80, 0x80] + , rejectUtf8 "three-byte surrogate" [0xED, 0xA0, 0x80] + , rejectUtf8 "four-byte truncated" [0xF0, 0x90, 0x80] + , rejectUtf8 "four-byte bad continuation" [0xF0, 0x90, 0x80, 0x00] + , rejectUtf8 "four-byte overlong" [0xF0, 0x80, 0x80, 0x80] + , rejectUtf8 "four-byte beyond U+10FFFF" [0xF4, 0x90, 0x80, 0x80] + -- After a backslash, a non-numeric non-ASCII byte is fed to + -- 'decodeUtf8At'. Invalid bytes there propagate as + -- ErrInvalidUtf8 -- distinct internal path from the + -- main-loop UTF-8 decode but the same observable outcome. + , rejectUtf8 "backslash + bare continuation" [0x66, 0x6F, 0x6F, 0x5C, 0x80] + , rejectUtf8 "backslash + truncated 2-byte" [0x66, 0x6F, 0x6F, 0x5C, 0xC2] + ] + where + rejectUtf8 :: String -> [Word8] -> Tasty.TestTree + rejectUtf8 name bytes = testCase name do + case parseDomainUtf8 hostnameLabelForms defaultIdnaFlags (BS.pack bytes) of + Left (ErrInvalidUtf8 {}) -> pure () + other -> assertFailure $ + "expected ErrInvalidUtf8 for " ++ show bytes + ++ ", got: " ++ show other + +---------------------------------------------------------------------- +-- 'Maybe'-returning convenience wrappers. +-- +-- The 'mkDomain' family is a thin shim around the @parseDomain*@ +-- entry points that discards 'LabelInfo' and collapses any parse +-- error to 'Nothing'. Spot-check that each shape works for one +-- valid and one invalid input. +---------------------------------------------------------------------- + +mkDomainFamilyTests :: Tasty.TestTree +mkDomainFamilyTests = Tasty.testGroup "mkDomain helpers" + [ testCase "mkDomain accepts valid name" $ + assertRight "mkDomain" (mkDomain "www.example.com") + , testCase "mkDomain rejects invalid name" $ + assertLeft "mkDomain" (mkDomain "..bad..") + , testCase "mkDomainStr accepts valid name" $ + assertRight "mkDomainStr" (mkDomainStr "www.example.com") + , testCase "mkDomainStr rejects invalid name" $ + assertLeft "mkDomainStr"(mkDomainStr "..bad..") + , testCase "mkDomainUtf8 accepts valid name" $ + assertRight "mkDomainUtf8" + (mkDomainUtf8 (T.encodeUtf8 ("www.example.com" :: T.Text))) + , testCase "mkDomainUtf8 rejects invalid name" $ + assertLeft "mkDomainUtf8" + (mkDomainUtf8 (T.encodeUtf8 ("..bad.." :: T.Text))) + , testCase "mkDomainUtf8 rejects bad UTF-8" $ + assertLeft "mkDomainUtf8" + (mkDomainUtf8 (BS.pack [0xC0, 0x80])) + , testCase "mkDomainShort accepts valid name" $ + assertRight "mkDomainShort" + (mkDomainShort (SBS.toShort (T.encodeUtf8 ("www.example.com" :: T.Text)))) + , testCase "mkDomainShort rejects invalid name" $ + assertLeft "mkDomainShort" + (mkDomainShort (SBS.toShort (T.encodeUtf8 ("..bad.." :: T.Text)))) + ] + where + assertRight :: String -> Either a b -> IO () + assertRight _ (Right _) = pure () + assertRight name (Left _) = + assertFailure (name ++ ": expected Right ..., got Left ...") + + assertLeft :: Show a => String -> Either a b -> IO () + assertLeft _ (Left _) = pure () + assertLeft name (Right _) = + assertFailure (name ++ ": expected Left ..., got Right ...") + +---------------------------------------------------------------------- +-- Command-line option parsers. +-- +-- 'parseLabelFormSet' and 'parseIdnaFlags' translate +-- comma-separated token strings into 'LabelFormSet' / 'IdnaFlags' +-- values. The conformance suite passes happy-path strings via the +-- @classes@ and @flags@ vector fields; these tests cover the +-- negative paths (unknown tokens, ambiguous prefixes) and a few +-- alias-resolution and arithmetic combinations that the JSON +-- vectors don't otherwise exercise. +---------------------------------------------------------------------- + +tokenVocabularyTests :: Tasty.TestTree +tokenVocabularyTests = Tasty.testGroup "command-line option parsers" + [ Tasty.testGroup "parseLabelFormSet" + [ testCase "preset 'host' parses" $ + assertOkLfs "host" + , testCase "preset 'idn' parses" $ + assertOkLfs "idn" + , testCase "preset 'all' parses" $ + assertOkLfs "all" + , testCase "preset 'strict' parses" $ + assertOkLfs "strict" + , testCase "'+attrleaf' extends default" $ + assertOkLfs "+attrleaf" + , testCase "'-fakea' shrinks default" $ + assertOkLfs "-fakea" + , testCase "unknown token rejected" $ + assertFailLfs "bogus" + ] + , Tasty.testGroup "parseIdnaFlags" + [ testCase "preset 'default' parses" $ + assertOkFlags "default" + , testCase "preset 'map' parses" $ + assertOkFlags "map" + , testCase "'+emoji-ok' extends default" $ + assertOkFlags "+emoji-ok" + , testCase "'-bidi-check' shrinks default" $ + assertOkFlags "-bidi-check" + , testCase "alias 'xncheck' resolves" $ + assertOkFlags "xncheck" + , testCase "alias 'cmap' resolves" $ + assertOkFlags "cmap" + , testCase "unknown token rejected" $ + assertFailFlags "bogus-flag" + , testCase "ascii-fallback alone parses" $ + assertOkFlags "ascii-fallback" + ] + ] + where + assertOkLfs :: BS.ByteString -> IO () + assertOkLfs s = case parseLabelFormSet hostnameLabelForms s of + Right _ -> pure () + Left e -> assertFailure ("expected Right for " ++ show s + ++ ", got Left: " ++ e) + + assertFailLfs :: BS.ByteString -> IO () + assertFailLfs s = case parseLabelFormSet hostnameLabelForms s of + Left _ -> pure () + Right _ -> assertFailure ("expected Left for " ++ show s) + + assertOkFlags :: BS.ByteString -> IO () + assertOkFlags s = case parseIdnaFlags defaultIdnaFlags s of + Right _ -> pure () + Left e -> assertFailure ("expected Right for " ++ show s + ++ ", got Left: " ++ e) + + assertFailFlags :: BS.ByteString -> IO () + assertFailFlags s = case parseIdnaFlags defaultIdnaFlags s of + Left _ -> pure () + Right _ -> assertFailure ("expected Left for " ++ show s) diff --git a/tests/data/IdnaMappingTable.txt.gz b/tests/data/IdnaMappingTable.txt.gz new file mode 100644 Binary files /dev/null and b/tests/data/IdnaMappingTable.txt.gz differ diff --git a/tests/data/IdnaTestV2.txt.gz b/tests/data/IdnaTestV2.txt.gz new file mode 100644 Binary files /dev/null and b/tests/data/IdnaTestV2.txt.gz differ diff --git a/tests/data/idna-test-v2-expected-diffs.txt b/tests/data/idna-test-v2-expected-diffs.txt new file mode 100644 --- /dev/null +++ b/tests/data/idna-test-v2-expected-diffs.txt @@ -0,0 +1,31 @@ +# Expected disagreements between strict IDNA2008 (this library) and UTS #46 +# Nontransitional, as exercised by the Unicode IdnaTestV2.txt vectors via +# the `unicode-conformance` Cabal-flagged test-suite. +# +# Format: one entry per non-blank, non-comment line. Fields are separated +# by TAB characters. The first field is the disposition; the second is +# the source string from IdnaTestV2.txt (with `\uXXXX` escapes preserved +# rather than decoded so the file is greppable); the third (optional) +# field is a short free-form reason. +# +# +# +# Dispositions: +# +# skip -- the vector is not run; the library is not expected to +# agree with the UTS #46 column for this input, and the +# disagreement isn't worth a finer-grained classification +# (e.g. UTS #46 applies a mapping we deliberately don't, +# producing an output we have no equivalent for). +# +# xfail -- the vector is run and is expected to disagree with the +# UTS #46 column. If the library happens to agree, that's +# an "unexpected pass" worth investigating (probably means +# a recent fix or a tightening on the UTS #46 side). +# +# Entries are added as systematic UTS #46-vs-IDNA2008 differences +# surface from running the harness. Each should cite either the +# §5892 disposition table, the UTS #46 deviation paragraph, or a +# specific RFC 5891/5893 rule. + +# (no entries yet -- harness reports raw agreement on every vector) diff --git a/tests/punycode.hs b/tests/punycode.hs new file mode 100644 --- /dev/null +++ b/tests/punycode.hs @@ -0,0 +1,246 @@ +{-# LANGUAGE OverloadedStrings #-} + +-- | +-- Module : Main +-- Description : RFC 3492 section 7.1 Punycode conformance vectors, +-- tested as raw codec round-trips through +-- 'Text.IDNA2008.Internal.Punycode'. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- These vectors test the Punycode codec on its own terms, without +-- the IDNA validation layer wrapped around it. They reproduce the +-- worked examples in RFC 3492 section 7.1 (\"Sample Strings\") for +-- Arabic (Egyptian), Chinese (simplified and traditional), Czech, +-- Hebrew, Hindi, Japanese, Korean, Russian, Spanish, Vietnamese, +-- plus the additional ad-hoc cases the RFC lists for short input, +-- punctuation, and mixed-case round-tripping. +-- +-- Note on case preservation: Punycode preserves the case of the +-- basic-ASCII prefix in the encoded form. Vectors @D@ (Czech), +-- @J@ (Spanish), @K@ (Vietnamese), @L@, @M@, @N@, @P@ feed +-- mixed-case U-labels and expect mixed-case Punycode bodies on +-- output. The IDNA layer's @MAPCASE@ option (which lower-cases the +-- input before encoding) is not exercised here. +module Main (main) where + +import Control.Monad.ST (runST) +import Data.Char (chr, ord) +import Data.Primitive.ByteArray + ( ByteArray + , byteArrayFromList + , freezeByteArray + , indexByteArray + , newByteArray + ) +import Data.Primitive.PrimArray + ( PrimArray + , newPrimArray + , primArrayFromList + , readPrimArray + ) +import Data.Word (Word8) +import qualified Data.ByteString as BS + +import Test.Tasty (TestTree, defaultMain, testGroup) +import Test.Tasty.HUnit + ( Assertion + , assertEqual + , assertFailure + , testCase + ) + +import Text.IDNA2008.Internal.Punycode + ( PunycodeErr (..) + , punycodeDecode + , punycodeEncode + ) + +---------------------------------------------------------------------- +-- Ergonomic wrappers around the buffer-based Punycode API. +---------------------------------------------------------------------- + +-- | Encode a 'String' of Unicode codepoints to a Punycode ASCII +-- 'BS.ByteString' (without the @\"xn--\"@ prefix). +encode :: String -> Either PunycodeErr BS.ByteString +encode cps = runST do + let !inBuf = primArrayFromList (map ord cps) :: PrimArray Int + !n = length cps + !cap = n * 8 + 64 -- ample for any well-formed input + outBuf <- newByteArray cap + res <- punycodeEncode inBuf n outBuf 0 cap + case res of + Left e -> pure (Left e) + Right written -> do + frozen <- freezeByteArray outBuf 0 written + pure (Right (BS.pack [ indexByteArray frozen i :: Word8 + | i <- [0 .. written - 1] ])) + +-- | Decode a Punycode ASCII 'BS.ByteString' (without the @\"xn--\"@ +-- prefix) to a 'String' of Unicode codepoints. +decode :: BS.ByteString -> Either PunycodeErr String +decode bs = runST do + let !inBuf = byteArrayFromList (BS.unpack bs) :: ByteArray + !n = BS.length bs + !cap = max 16 (n * 4) + outBuf <- newPrimArray cap + res <- punycodeDecode inBuf 0 n outBuf 0 cap + case res of + Left e -> pure (Left e) + Right written -> do + let collect !i !acc + | i < 0 = pure (Right acc) + | otherwise = do + cp <- readPrimArray outBuf i + collect (i - 1) (chr cp : acc) + collect (written - 1) [] + +---------------------------------------------------------------------- +-- RFC 3492 section 7.1 vectors. +-- +-- For each: (mnemonic, U-label as a Haskell String, expected Punycode +-- body without the "xn--" prefix). Verified byte-for-byte against +-- the RFC text and against Python's bundled @encodings.punycode@ +-- codec. +---------------------------------------------------------------------- + +type Vector = (String, String, BS.ByteString) + +rfc3492Vectors :: [Vector] +rfc3492Vectors = + [ ("(A) Arabic (Egyptian)" + , "\x0644\x064A\x0647\x0645\x0627\x0628\x062A\x0643\x0644\ + \\x0645\x0648\x0634\x0639\x0631\x0628\x064A\x061F" + , "egbpdaj6bu4bxfgehfvwxn") + , ("(B) Chinese (simplified)" + , "\x4ED6\x4EEC\x4E3A\x4EC0\x4E48\x4E0D\x8BF4\x4E2D\x6587" + , "ihqwcrb4cv8a8dqg056pqjye") + , ("(C) Chinese (traditional)" + , "\x4ED6\x5011\x7232\x4EC0\x9EBD\x4E0D\x8AAA\x4E2D\x6587" + , "ihqwctvzc91f659drss3x8bo0yb") + , ("(D) Czech" + , "Pro\x010Dprost\x011Bnemluv\x00ED\x010D\&esky" + , "Proprostnemluvesky-uyb24dma41a") + , ("(E) Hebrew" + , "\x05DC\x05DE\x05D4\x05D4\x05DD\x05E4\x05E9\x05D5\x05D8\ + \\x05DC\x05D0\x05DE\x05D3\x05D1\x05E8\x05D9\x05DD\x05E2\ + \\x05D1\x05E8\x05D9\x05EA" + , "4dbcagdahymbxekheh6e0a7fei0b") + , ("(F) Hindi (Devanagari)" + , "\x092F\x0939\x0932\x094B\x0917\x0939\x093F\x0928\x094D\ + \\x0926\x0940\x0915\x094D\x092F\x094B\x0902\x0928\x0939\ + \\x0940\x0902\x092C\x094B\x0932\x0938\x0915\x0924\x0947\ + \\x0939\x0948\x0902" + , "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd") + , ("(G) Japanese (kanji and hiragana)" + , "\x306A\x305C\x307F\x3093\x306A\x65E5\x672C\x8A9E\x3092\ + \\x8A71\x3057\x3066\x304F\x308C\x306A\x3044\x306E\x304B" + , "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa") + , ("(H) Korean (Hangul syllables)" + , "\xC138\xACC4\xC758\xBAA8\xB4E0\xC0AC\xB78C\xB4E4\xC774\ + \\xD55C\xAD6D\xC5B4\xB97C\xC774\xD574\xD55C\xB2E4\xBA74\ + \\xC5BC\xB9C8\xB098\xC88B\xC744\xAE4C" + , "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j\ + \psd879ccm6fea98c") + -- RFC 3492 as originally published spells the encoded form + -- with an uppercase @D@ in the middle: + -- @\"b1abfaaepdrnnbgefbaDotcwatmq2g4l\"@. Per the published + -- erratum that's a typo -- the encoder produces an + -- all-lowercase body, since the input is all-lowercase + -- Cyrillic and Punycode preserves the case of the basic + -- prefix, of which this example has none. + -- See . + , ("(I) Russian (Cyrillic)" + , "\x043F\x043E\x0447\x0435\x043C\x0443\x0436\x0435\x043E\ + \\x043D\x0438\x043D\x0435\x0433\x043E\x0432\x043E\x0440\ + \\x044F\x0442\x043F\x043E\x0440\x0443\x0441\x0441\x043A\ + \\x0438" + , "b1abfaaepdrnnbgefbadotcwatmq2g4l") + , ("(J) Spanish" + , "Porqu\x00E9nopuedensimplementehablarenEspa\x00F1ol" + , "PorqunopuedensimplementehablarenEspaol-fmd56a") + , ("(K) Vietnamese" + , "T\x1EA1isaoh\x1ECDkh\x00F4ngth\x1EC3\&ch\x1EC9n\x00F3it\ + \i\x1EBFngVi\x1EC7t" + , "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g") + , ("(L) 3B...kinpachi sensei" + , "3\x5E74\&B\x7D44\x91D1\x516B\x5148\x751F" + , "3B-ww4c5e180e575a65lsy2b") + , ("(M) Amuro Namie -with-SUPER-MONKEYS" + , "\x5B89\x5BA4\x5948\x7F8E\x6075-with-SUPER-MONKEYS" + , "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n") + , ("(N) Hello-Another-Way-" + , "Hello-Another-Way-\x305D\x308C\x305E\x308C\x306E\x5834\ + \\x6240" + , "Hello-Another-Way--fc4qua05auwb3674vfr0b") + , ("(O) 2" + , "\x3072\x3068\x3064\x5C4B\x6839\x306E\x4E0B\&2" + , "2-u9tlzr9756bt3uc0v") + , ("(P) Maji-de-Koi-suru-5-byou-mae" + , "Maji\x3067\&Koi\x3059\x308B\&5\x79D2\x524D" + , "MajiKoi5-783gue6qz075azm5e") + , ("(Q) de Rumba" + , "\x30D1\x30D5\x30A3\x30FC\&de\x30EB\x30F3\x30D0" + , "de-jg4avhby1noc0d") + , ("(R) " + , "\x305D\x306E\x30B9\x30D4\x30FC\x30C9\x3067" + , "d9juau41awczczp") + , ("(S) -> $1.00 <-" + , "-> $1.00 <-" + , "-> $1.00 <--") + ] + +---------------------------------------------------------------------- +-- Test harness. +---------------------------------------------------------------------- + +-- | Check that encoding the U-label produces the expected Punycode +-- body, /and/ that decoding the body recovers the U-label. +roundTrip :: Vector -> TestTree +roundTrip (name, u, expected) = testCase name do + case encode u of + Left e -> assertFailure ("encode failed: " ++ show e) + Right got -> assertEqual "encode" expected got + case decode expected of + Left e -> assertFailure ("decode failed: " ++ show e) + Right got -> assertEqual "decode (round-trip)" u got + +-- | Edge cases not in RFC 3492 section 7.1 but worth pinning down: +-- empty input, all-basic input (no delimiter, no extension), and a +-- single non-ASCII codepoint. +edgeCases :: TestTree +edgeCases = testGroup "edge cases" + [ testCase "empty input encodes to empty" $ + encode "" @?= Right BS.empty + , testCase "empty input decodes from empty" $ + decode BS.empty @?= Right "" + , testCase "all-basic input gets a trailing delimiter" $ + -- RFC 3492 section 6.3: copy basics in order, followed by + -- a delimiter if b > 0 -- including when b == length(input). + encode "abc-123" @?= Right "abc-123-" + , testCase "all-basic body round-trips through trailing delimiter" $ + decode "abc-123-" @?= Right "abc-123" + , testCase "all-basic body without delimiter is malformed" $ + -- The last hyphen in a valid Punycode body terminates the + -- basic prefix; in @\"abc-123\"@ that's the hyphen between + -- @c@ and @1@, leaving @\"123\"@ as a base-36 integer that + -- doesn't terminate. + decode "abc-123" @?= Left PunycodeTruncated + , testCase "single non-ASCII codepoint" $ do + encode "\x00FC" @?= Right "tda" + decode "tda" @?= Right "\x00FC" + ] + where + -- Locally-scoped operator to keep the import list tidy. + (@?=) :: (Eq a, Show a) => a -> a -> Assertion + x @?= y = assertEqual "" y x + +main :: IO () +main = defaultMain $ testGroup "Punycode (RFC 3492)" + [ testGroup "section 7.1 sample strings" + (map roundTrip rfc3492Vectors) + , edgeCases + ] diff --git a/tests/unicode_conformance.hs b/tests/unicode_conformance.hs new file mode 100644 --- /dev/null +++ b/tests/unicode_conformance.hs @@ -0,0 +1,819 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | +-- Module : Main +-- Description : Unicode @IdnaTestV2.txt@ conformance harness. +-- Copyright : (c) Viktor Dukhovni, 2026 +-- License : BSD-3-Clause +-- +-- Maintainer : ietf-dane@dukhovni.org +-- Stability : unstable +-- +-- Runs the Unicode Consortium's @IdnaTestV2.txt@ vectors against +-- 'parseDomainOpts' chained with 'unparseDomainOpts' (for the +-- cross-label Bidi check), reporting agreement and disagreement +-- with the @toAsciiN@ (Nontransitional) column of the file -- +-- the column closest to strict RFC 5891 IDNA2008. Conformance +-- is a binary check per the file's own format header: +-- +-- @ +-- Implementations need only record that there is an error: they +-- need not reproduce the precise status codes (after removing +-- the ignored status values). +-- @ +-- +-- so vectors pass whenever the library's accept\/reject verdict +-- matches UTS \#46's, regardless of the specific error reason. +-- +-- The harness is gated on the @unicode-conformance@ Cabal flag (off +-- by default) so that the default @cabal test@ invocation has no +-- network dependency, no extra build-dependencies, and no +-- multi-thousand-vector test report cluttering output. Enable with: +-- +-- @ +-- cabal test --flags=+unicode-conformance +-- @ +-- +-- Known systematic disagreements between strict IDNA2008 and UTS #46 +-- Nontransitional are listed in +-- @tests\/data\/idna-test-v2-expected-diffs.txt@. Each entry there pairs a +-- source string with a disposition (@skip@ \/ @xfail@) plus a brief reason. +-- See the file's header comment for the format. +module Main (main) where + +import qualified Codec.Compression.GZip as GZip +import qualified Data.ByteString.Lazy as BL +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import Control.Exception (catch) +import Control.Monad (unless, when) +#if !MIN_VERSION_base(4,20,0) +import Data.Foldable(foldl') +#endif +import Data.List (isPrefixOf) +import Data.Map.Strict (Map) +import Data.Maybe (fromMaybe) +import Data.Set (Set) +import Numeric (readHex) +import System.Directory (doesFileExist) +import System.Environment (lookupEnv) +import System.IO (hPutStrLn, stderr) + +import Test.Tasty (TestTree, defaultMain, testGroup) +import Test.Tasty.HUnit (testCase, assertFailure) + +import Text.IDNA2008 + ( AceReason(..) + , IdnaError(..) + , IdnaFlags + , LabelFormSet + , LabelReason(..) + , allLabelForms + , defaultIdnaFlags + , parseDomainOpts + , parseIdnaFlagsStr + , parseLabelFormSetStr + , unparseDomainOpts + ) + +---------------------------------------------------------------------- +-- Knobs / source resolution +---------------------------------------------------------------------- + +-- | Read the @x-unicode-version@ custom field from @idna2008.cabal@, +-- which is the single source of truth for the Unicode version the +-- library's generated tables track. Reported in stderr alongside +-- the loaded-vector count so the user knows which Unicode version +-- the bundled @tests\/data\/\*.xz@ snapshots reflect. +cabalUnicodeVersion :: IO String +cabalUnicodeVersion = do + body <- readFile cabalPath `catch` \(e :: IOError) -> + fail $ "unicode-conformance: could not read " ++ cabalPath + ++ ": " ++ show e + case extract body of + Just v -> pure v + Nothing -> fail $ + "unicode-conformance: x-unicode-version field not found in " + ++ cabalPath + where + cabalPath = "idna2008.cabal" + fieldName = "x-unicode-version:" + extract body = + case dropWhile (not . (fieldName `isPrefixOf`)) (lines body) of + (line:_) -> Just (trim (drop (length fieldName) line)) + [] -> Nothing + +-- | Path to the bundled gzip-compressed @IdnaTestV2.txt@ snapshot. +-- Listed under @extra-source-files@ in @idna2008.cabal@ so it +-- travels with @cabal sdist@. +bundledVectorsFile :: FilePath +bundledVectorsFile = "tests/data/IdnaTestV2.txt.gz" + +-- | Path to the bundled gzip-compressed @IdnaMappingTable.txt@ +-- snapshot, used by the precise cp-disagreement filter. +bundledMappingFile :: FilePath +bundledMappingFile = "tests/data/IdnaMappingTable.txt.gz" + +-- | Where to look for the expected-diffs file (relative to the +-- directory @cabal test@ is invoked from, normally the package root). +-- Missing file is treated as \"no known diffs\". +defaultExpectedDiffsFile :: FilePath +defaultExpectedDiffsFile = "tests/data/idna-test-v2-expected-diffs.txt" + +-- | Read a UTS \#46 data file. Honours an environment-variable +-- override (callers pass e.g. @\"IDNA_TEST_V2_FILE\"@) pointing at +-- an /uncompressed/ plain-text local copy; otherwise reads the +-- bundled gzip-compressed snapshot at @bundledPath@ and +-- decompresses in memory. The env override is the supported way +-- to test against a Unicode version other than the bundled one +-- without repacking @tests\/data\/@. +readUnicodeFile :: FilePath -> String -> IO String +readUnicodeFile bundledPath envOverride = do + mPath <- lookupEnv envOverride + case mPath of + Just path -> readFile path + Nothing -> do + compressed <- BL.readFile bundledPath + let decompressed = GZip.decompress compressed + pure (T.unpack (T.decodeUtf8 (BL.toStrict decompressed))) + +---------------------------------------------------------------------- +-- Parser for IdnaTestV2.txt +-- +-- File format (as documented in the file's own header): +-- +-- * @#@ starts a comment that runs to end-of-line. +-- * Blank lines are ignored. +-- * Other lines have seven semicolon-separated fields: +-- +-- source ; toUnicode ; toUnicodeStatus ; +-- toAsciiN ; toAsciiNStatus ; +-- toAsciiT ; toAsciiTStatus +-- +-- * An empty @toUnicode@ field means "same as @source@". +-- * An empty @toAsciiN@ field means "same as @toUnicode@". +-- * An empty @toAsciiT@ field means "same as @toAsciiN@". +-- * An empty @toUnicodeStatus@\/@toAsciiNStatus@\/@toAsciiTStatus@ +-- field means "OK" (no errors expected). +-- * Status fields, when non-empty, are space-separated lists of +-- bracketed codes like @[V5]@, @[A3]@, @[Bn]@, @[Pn]@, @[Cn]@. +-- UTS #46 section 6 defines the code vocabulary. +-- * Source strings can contain @\\uXXXX@ escapes that resolve to the +-- literal codepoint. +---------------------------------------------------------------------- + +-- | One vector loaded from @IdnaTestV2.txt@, with inheritance applied +-- (empty fields filled in from earlier columns per the file's rules). +data Vector = Vector + { vSource :: !String -- ^ Input string. + , vToUnicode :: !String -- ^ Expected toUnicode output. + , vToUnicodeStatus :: ![String] -- ^ Expected toUnicode status codes. + , vToAsciiN :: !String -- ^ Expected toAsciiN output. + , vToAsciiNStatus :: ![String] -- ^ Expected toAsciiN status codes. + , vToAsciiT :: !String -- ^ Expected toAsciiT output. + , vToAsciiTStatus :: ![String] -- ^ Expected toAsciiT status codes. + , vLineNo :: !Int -- ^ Source-file line for diagnostics. + } deriving (Show, Eq) + +-- | Load and parse the bundled (or env-overridden) @IdnaTestV2.txt@. +loadVectors :: IO [Vector] +loadVectors = do + raw <- readUnicodeFile bundledVectorsFile "IDNA_TEST_V2_FILE" + pure (parseVectors raw) + +parseVectors :: String -> [Vector] +parseVectors body = + [ v + | (lineNo, line) <- zip [1..] (lines body) + , let stripped = stripComment line + , not (allBlank stripped) + , Just v <- [parseLine lineNo stripped] + ] + where + stripComment s = takeWhile (/= '#') s + allBlank = all (`elem` (" \t" :: String)) + +parseLine :: Int -> String -> Maybe Vector +parseLine lineNo s = + case splitOn ';' s of + [src, tu, tus, taN, taNs, taT, taTs] -> + let src' = decodeFieldString (trim src) "" + tu' = decodeFieldString (trim tu) src' + taN' = decodeFieldString (trim taN) tu' + taT' = decodeFieldString (trim taT) taN' + -- Status-column inheritance per the file's own format + -- header (column 3-7 documentation): + -- + -- * Column 3 (toUnicodeStatus): blank means @[]@ (no + -- errors). + -- * Column 5 (toAsciiNStatus): blank means /same as + -- toUnicodeStatus/. An explicit @[]@ means no + -- errors. + -- * Column 7 (toAsciiTStatus): blank means /same as + -- toAsciiNStatus/. An explicit @[]@ means no + -- errors. + -- + -- 'parseStatusField' distinguishes blank ('Nothing') + -- from explicit ('Just'), so we can resolve inheritance. + tuStat = fromMaybe [] (parseStatusField tus) + taNStat = fromMaybe tuStat (parseStatusField taNs) + taTStat = fromMaybe taNStat (parseStatusField taTs) + in Just Vector + { vSource = src' + , vToUnicode = tu' + , vToUnicodeStatus = tuStat + , vToAsciiN = taN' + , vToAsciiNStatus = taNStat + , vToAsciiT = taT' + , vToAsciiTStatus = taTStat + , vLineNo = lineNo + } + _ -> Nothing + +-- | Parse a status field. Distinguishes: +-- +-- * @Nothing@ -- the field was blank (no characters between the +-- semicolon separators). Per the file's format, blank in +-- toAsciiN\/T status columns means \"inherit the previous +-- column's value\"; the caller resolves that. +-- * @Just []@ -- the field was an explicit @[]@ marker meaning +-- \"no errors expected\". +-- * @Just xs@ -- the field carried one or more bracketed status +-- codes (e.g. @[V5]@, @[V5, B3]@, @[V5] [B3]@); 'xs' is the +-- code list with brackets, commas, and inter-code whitespace +-- stripped. +parseStatusField :: String -> Maybe [String] +parseStatusField raw = + case trim raw of + "" -> Nothing + body -> Just (filter (not . null) (words (map normalize body))) + where + normalize c + | c == '[' || c == ']' || c == ',' = ' ' + | otherwise = c + +-- | Interpret a trimmed value-column field per the file's format +-- header (column 1, 2, 4, 6 documentation): +-- +-- * @\"\"@ (literal two double-quotes) means the empty string. +-- * Any other non-empty content is the literal field value, with +-- @\\uXXXX@ \/ @\\x{HEX}@ escapes decoded. +-- * Empty content (after trimming) inherits the supplied parent. +-- +-- 'parent' is the value to inherit on empty (caller chooses what's +-- meaningful for the column being parsed). +decodeFieldString :: String -> String -> String +decodeFieldString trimmed parent = case trimmed of + "" -> parent + "\"\"" -> "" + other -> decodeEscapes other + +-- | UTS #46's source field uses @\\uXXXX@ for non-printable / non-ASCII +-- codepoints. Decode those (and the few backslash escapes the file +-- uses) into literal characters. +decodeEscapes :: String -> String +decodeEscapes = go + where + go [] = [] + go ('\\':'u':a:b:c:d:rest) + | all isHex [a,b,c,d] = + toEnum (readHex4 [a,b,c,d]) : go rest + go ('\\':'x':'{':rest) = + let (hex, more) = break (== '}') rest + in case more of + '}':more' | not (null hex) && all isHex hex -> + toEnum (readHexAny hex) : go more' + _ -> '\\' : 'x' : '{' : go rest + go (c:rest) = c : go rest + + isHex c = (c >= '0' && c <= '9') + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F') + readHex4 cs = readHexAny cs + readHexAny = foldl (\acc c -> acc * 16 + hexDigit c) 0 + hexDigit c + | c >= '0' && c <= '9' = fromEnum c - fromEnum '0' + | c >= 'a' && c <= 'f' = 10 + fromEnum c - fromEnum 'a' + | c >= 'A' && c <= 'F' = 10 + fromEnum c - fromEnum 'A' + | otherwise = 0 + +splitOn :: Char -> String -> [String] +splitOn sep = foldr go [[]] + where + go c acc@(cur:rest) + | c == sep = [] : acc + | otherwise = (c:cur) : rest + go _ [] = [[]] + +trim :: String -> String +trim = dropWhile isSpace . reverse . dropWhile isSpace . reverse + where isSpace c = c == ' ' || c == '\t' + +---------------------------------------------------------------------- +-- Parser for IdnaMappingTable.txt +-- +-- File format (per UTS \#46 \"Section 5: IDNA Mapping Table\"): +-- +-- [..] ; [; ] [; ] +-- # comment +-- +-- Statuses we care about for marking a codepoint as \"UTS \#46 treats +-- this more permissively than strict IDNA2008\": +-- +-- * @valid@ with a 4th-column tag of @NV8@ or @XV8@ -- the cp is +-- valid under IDNA2003\/legacy IDNA but disallowed by IDNA2008's +-- §5892 disposition table. The file header explicitly tells +-- strict-IDNA2008 implementations to skip these. +-- * @mapped@ -- UTS \#46 rewrites the cp to the mapping target. +-- We don't apply that rewrite, so we see the original cp and +-- reject it. +-- * @deviation@ -- UTS \#46 conditionally rewrites under +-- Transitional Processing; under Nontransitional it's left +-- valid, but RFC 5891 may not treat the cp the same way. +-- * @ignored@ -- UTS \#46 removes the cp. We keep it, which can +-- turn an otherwise-valid label into a leading-combining-mark +-- violation (e.g. variation selectors at the start of a label). +-- * @disallowed_STD3_mapped@ -- under default UTS \#46 settings +-- (STD3 = true) UTS \#46 rejects; under STD3 = false it maps. +-- IDNA2008 always rejects. Treat as permissive for safety. +-- +-- All other statuses (@valid@ without NV8\/XV8, @disallowed@, +-- @disallowed_STD3_valid@) are agreed-on between the two specs. +---------------------------------------------------------------------- + +-- | Codepoints UTS \#46 treats more permissively than strict +-- IDNA2008, as extracted from @IdnaMappingTable.txt@. Membership in +-- this set is the precise signal the comparator uses to decide that +-- a 'DisallowedCodepoint' or 'LeadingCombiningMark' rejection from +-- our library reflects a documented spec disagreement rather than a +-- library bug. +loadMappingTable :: IO (Set Int) +loadMappingTable = do + body <- readUnicodeFile bundledMappingFile "IDNA_MAPPING_TABLE_FILE" + pure $ Set.fromList + [ cp + | line <- lines body + , let payload = takeWhile (/= '#') line + , not (allBlank payload) + , Just (lo, hi) <- [parseMappingLine payload] + , cp <- [lo .. hi] + ] + where + allBlank = all (`elem` (" \t" :: String)) + +-- | Parse one non-comment line of @IdnaMappingTable.txt@; return the +-- inclusive range @(lo, hi)@ if and only if the line's status marks +-- the codepoint(s) as a UTS \#46-vs-IDNA2008 disagreement. +parseMappingLine :: String -> Maybe (Int, Int) +parseMappingLine line = case map trim (splitOn ';' line) of + (cpField : status : rest) | isPermissive status rest -> + parseRange cpField + _ -> Nothing + +-- | A status (column 2) and the trailing columns indicate a +-- UTS \#46-more-permissive disposition iff one of the following: +isPermissive :: String -> [String] -> Bool +isPermissive status rest = case status of + "mapped" -> True + "deviation" -> True + "ignored" -> True + "disallowed_STD3_mapped" -> True + "valid" -> hasNvXv rest + _ -> False + where + -- @valid@ rows tag NV8/XV8 in either the 3rd or 4th column, + -- depending on whether the mapping field is present. + hasNvXv fields = any (\f -> trim f == "NV8" || trim f == "XV8") + fields + +-- | Parse a @cp@ or @cp..cp@ field. Hex with no @0x@ prefix. +parseRange :: String -> Maybe (Int, Int) +parseRange s = case break (== '.') (trim s) of + (lo, "") -> singleton (parseHex lo) + (lo, '.':'.':hiRest) -> do + a <- parseHex lo + b <- parseHex hiRest + pure (a, b) + _ -> Nothing + where + singleton (Just a) = Just (a, a) + singleton Nothing = Nothing + + parseHex :: String -> Maybe Int + parseHex hex = case readHex (trim hex) of + [(n, "")] -> Just n + _ -> Nothing + +---------------------------------------------------------------------- +-- Expected-diffs file +---------------------------------------------------------------------- + +-- | How to treat a vector that the expected-diffs file mentions. +data Disposition + = DSkip -- ^ Drop the vector before running it. + | DXFail -- ^ Expect the comparator to report a disagreement; if + -- it agrees instead, that's an unexpected pass worth + -- investigating. + deriving (Eq, Show) + +-- | Load the expected-diffs file. Each non-blank, non-comment line is +-- @\\\t\\\t\@; lines starting with @#@ are +-- comments. Missing file returns an empty map. +loadExpectedDiffs :: FilePath -> IO (Map String (Disposition, String)) +loadExpectedDiffs path = do + exists <- doesFileExist path + if not exists + then pure Map.empty + else do + body <- readFile path + pure $ Map.fromList + [ (decodeEscapes src, (disp, reason)) + | line <- lines body + , not (isCommentLine line) + , Just (disp, src, reason) <- [parseDiffLine line] + ] + where + isCommentLine line = case dropWhile (`elem` (" \t" :: String)) line of + ('#':_) -> True + "" -> True + _ -> False + + parseDiffLine line = case splitOn '\t' line of + (d:s:rs) -> do + disp <- case dropWhile (`elem` (" \t" :: String)) d of + "skip" -> Just DSkip + "xfail" -> Just DXFail + _ -> Nothing + pure (disp, s, unwords (filter (not . null) rs)) + _ -> Nothing + +---------------------------------------------------------------------- +-- Library flag and label-form set chosen to approximate UTS #46 +-- Nontransitional semantics. +---------------------------------------------------------------------- + +-- | Closest match to UTS #46 Nontransitional: the parser default +-- (which already enables strict A-label check, strict NFC validation, +-- and per-label Bidi check) plus all four mapping toggles +-- (@map-case@, @map-width@, @map-nfc@, @map-dots@). Combined under +-- the @\"default,map\"@ token preset; we feed the library default +-- 'IdnaFlags' as the base value the @default@ token resolves to. +uts46Flags :: IdnaFlags +uts46Flags = case parseIdnaFlagsStr defaultIdnaFlags "default,map" of + Right fs -> fs + Left e -> error ("uts46Flags: " ++ e) + +-- | UTS #46 only admits A-, U-, and conventional LDH labels. FAKEA, +-- OCTET, ATTRLEAF, BLANK, WILDLABEL are not part of its domain-name +-- vocabulary; we use the @idn@ preset to reject them up front so the +-- comparator doesn't try to map their library-internal classifications +-- onto UTS #46 codes that don't exist. 'allLabelForms' is fed as the +-- base value for the @default@ token (not used in our preset, but the +-- parser requires a default to resolve to). +uts46Forms :: LabelFormSet +uts46Forms = case parseLabelFormSetStr allLabelForms "idn" of + Right fs -> fs + Left e -> error ("uts46Forms: " ++ e) + +---------------------------------------------------------------------- +-- (No specific-code matching: per the file's own header, conformance +-- is binary error-or-no-error. The previous attempt to map our +-- 'IdnaError' constructors onto UTS \#46 codes is intentionally +-- absent here.) +---------------------------------------------------------------------- + +---------------------------------------------------------------------- +-- Comparator +---------------------------------------------------------------------- + +data Outcome + = OPass + -- ^ Library matched UTS #46. + | OUnexpectedAccept ![String] + -- ^ UTS expected error; we succeeded. The list is the + -- expected status set. + | OUnexpectedReject !IdnaError + -- ^ UTS expected success; we rejected. + | OCpDisagreement !IdnaError + -- ^ Library rejection traces back to a codepoint UTS \#46 + -- would have preprocessed away (or accepted) but strict + -- IDNA2008 can't. Covers three flavours: + -- + -- * 'DisallowedCodepoint' (direct or via 'DecodedInvalid') + -- -- the NV8\/XV8 cases the file header explicitly tells + -- strict implementations to skip, plus UTS \#46's + -- NFKC-style mapped codepoints we don't apply. + -- * 'LeadingCombiningMark' on a codepoint UTS \#46 treats + -- as @ignored@ (e.g. variation selectors @U+FE00@ - + -- @U+FE0F@): UTS \#46 removes them, exposing the next + -- codepoint as the label start; we keep them and + -- reject under RFC 5891 §4.2.3.2. + -- + -- Auto-skipped to avoid drowning real bugs. + | ORootDifference ![String] + -- ^ Library accepted a domain UTS #46 rejects as A4_1\/A4_2 + -- \/X4_2, and the source is empty, all-dot-mappings, or + -- ends in one. Captures the DNS convention (trailing + -- dot = absolute root) versus UTS #46's \"no empty label + -- anywhere\". Auto-skipped. + +-- | The UTS \#46 file header is explicit that conformance is a +-- binary error-or-no-error check, not a specific-code match: +-- +-- @ +-- Implementations need only record that there is an error: they +-- need not reproduce the precise status codes (after removing +-- the ignored status values). +-- @ +-- +-- So the comparator collapses to four outcomes: +-- +-- * Both sides agree on success -> 'OPass' +-- * Both sides agree on /some/ error -> 'OPass' +-- * UTS \#46 expected an error, library accepted -> 'OUnexpectedAccept' +-- (or 'ORootDifference' for the DNS-root subset) +-- * UTS \#46 expected success, library rejected -> 'OUnexpectedReject' +-- (or 'OCpDisagreement' for the documented spec- +-- disagreement subset) +runVector :: Set Int -> Vector -> Outcome +runVector permissiveCps v = case fullPipeline input of + Right () + | null expected -> OPass + | isRootLike (vSource v) + && any isLenCode expected -> ORootDifference expected + | otherwise -> OUnexpectedAccept expected + Left err + | not (null expected) -> OPass + | isCpDisagreement err -> OCpDisagreement err + | otherwise -> OUnexpectedReject err + where + input = T.pack (vSource v) + expected = vToAsciiNStatus v + + isLenCode c = c == "A4_1" || c == "A4_2" || c == "X4_2" + + -- UTS #46 treats every empty label (including the canonical + -- DNS \"trailing dot = root\" form) as A4_2; the library + -- treats a trailing dot as the absolute-root indicator + -- consistent with master-file syntax. These two views + -- disagree by definition on inputs that are empty, consist + -- solely of label-separator characters, or end in one. + isRootLike s = null s || all isDotMap s || lastIsDotMap s + where + lastIsDotMap [] = False + lastIsDotMap xs = isDotMap (last xs) + isDotMap c = c == '.' -- ASCII full stop + || c == '\x3002' -- ideographic full stop + || c == '\xFF0E' -- fullwidth full stop + || c == '\xFF61' -- halfwidth ideographic full stop + + -- Precise cp-disagreement test: the offending codepoint is one + -- IdnaMappingTable.txt marks as @mapped@, @deviation@, + -- @ignored@, or @valid;NV8/XV8@ -- the documented UTS \#46-vs- + -- IDNA2008 disagreement set. If 'permissiveCps' is empty + -- (mapping table failed to load) the check degrades to + -- \"always false\", which makes such disagreements surface as + -- 'OUnexpectedReject' rather than being silently absorbed. + -- Only consulted when @UTS \#46 expected success@; if UTS + -- already expected /some/ error, the comparator counts our + -- error as PASS regardless of the specific reason. + isCpDisagreement = \case + ErrLabelInvalid _ (DisallowedCodepoint cp) -> + cp `Set.member` permissiveCps + ErrAceInvalid _ (DecodedInvalid (DisallowedCodepoint cp)) -> + cp `Set.member` permissiveCps + ErrLabelInvalid _ (LeadingCombiningMark cp) -> + cp `Set.member` permissiveCps + ErrAceInvalid _ (DecodedInvalid (LeadingCombiningMark cp)) -> + cp `Set.member` permissiveCps + _ -> + False + +-- | UTS #46 \"toAsciiN\" composes per-label parsing with the +-- cross-label Bidi check ('renderUnicodeTextChecked' inside +-- 'unparseDomainOpts'). 'parseDomainOpts' alone only enforces the +-- per-label half of RFC 5893, so we chain the two: parse, then if +-- parse succeeded, run the cross-label check. We discard the +-- rendered Unicode text the second step returns -- only its +-- error\/success status matters for conformance. +fullPipeline :: T.Text -> Either IdnaError () +fullPipeline input = do + (dom, _info) <- parseDomainOpts uts46Forms uts46Flags input + _ <- unparseDomainOpts allLabelForms uts46Flags dom + pure () + +---------------------------------------------------------------------- +-- Aggregation +---------------------------------------------------------------------- + +data Counts = Counts + { cTotal :: !Int + , cPass :: !Int + , cSkip :: !Int -- ^ Explicit skip via expected-diffs. + , cXFail :: !Int -- ^ Expected-diffs xfail confirmed. + , cUPass :: !Int -- ^ XFail predicted; actually agreed. + , cCpDisagree :: !Int -- ^ Auto-skip: NV8\/XV8\/NFKC-mapped. + , cRootDifference :: !Int -- ^ Auto-skip: DNS root vs UTS #46. + , cDiff :: !Int + , cDiffSamples :: ![String] -- ^ First N unexpected-diff lines. + , cUPassSamples :: ![String] -- ^ First N unexpected-pass lines. + } + +emptyCounts :: Counts +emptyCounts = Counts 0 0 0 0 0 0 0 0 [] [] + +-- | Cap on how many lines of detail we accumulate in the failure +-- message. Anything beyond this is summarised numerically. +sampleLimit :: Int +sampleLimit = 30 + +step + :: Set Int + -> Map String (Disposition, String) + -> Counts -> Vector -> Counts +step permissiveCps diffs c v = + let total' = cTotal c + 1 + bumped = c { cTotal = total' } + in case Map.lookup (vSource v) diffs of + Just (DSkip, _) -> bumped { cSkip = cSkip c + 1 } + mDiff -> case runVector permissiveCps v of + OPass -> case mDiff of + Just (DXFail, reason) -> + bumped { cUPass = cUPass c + 1 + , cUPassSamples = + addSample sampleLimit (renderUPass v reason) + (cUPassSamples c) + } + _ -> bumped { cPass = cPass c + 1 } + OCpDisagreement{} -> + -- Heuristic auto-skip: never counted as a diff, + -- regardless of xfail entries. An explicit xfail + -- entry for one of these would be redundant. + bumped { cCpDisagree = cCpDisagree c + 1 } + ORootDifference{} -> + bumped { cRootDifference = cRootDifference c + 1 } + outcome -> case mDiff of + Just (DXFail, _) -> + bumped { cXFail = cXFail c + 1 } + _ -> + bumped { cDiff = cDiff c + 1 + , cDiffSamples = + addSample sampleLimit (renderDiff v outcome) + (cDiffSamples c) + } + +addSample :: Int -> a -> [a] -> [a] +addSample lim x xs + | length xs < lim = xs ++ [x] + | otherwise = xs + +renderDiff :: Vector -> Outcome -> String +renderDiff v outcome = case outcome of + OPass -> "" + OCpDisagreement{} -> "" -- auto-skipped, never rendered + ORootDifference{} -> "" -- auto-skipped, never rendered + OUnexpectedAccept exp_ -> concat + [ " L", pad6 (vLineNo v) + , " accepted but UTS expected ", show exp_ + , ": ", show (vSource v) + ] + OUnexpectedReject err -> concat + [ " L", pad6 (vLineNo v) + , " UTS expected success, library returned " + , showError err + , ": ", show (vSource v) + ] + where + pad6 n = let s = show n in replicate (max 0 (6 - length s)) ' ' ++ s + +renderUPass :: Vector -> String -> String +renderUPass v reason = concat + [ " L", show (vLineNo v) + , " xfail predicted but library agreed with UTS" + , (if null reason then "" else " (" ++ reason ++ ")") + , ": ", show (vSource v) + ] + +-- | Compact one-liner for an 'IdnaError', avoiding the verbose +-- generic 'show'. +showError :: IdnaError -> String +showError = \case + ErrEmptyLabel{} -> "EmptyLabel" + ErrLabelTooLong _ n -> "LabelTooLong(" ++ show n ++ ")" + ErrNameTooLong n -> "NameTooLong(" ++ show n ++ ")" + ErrBadEscape{} -> "BadEscape" + ErrInvalidUtf8{} -> "InvalidUtf8" + ErrCodepointTooLarge _ cp -> "CodepointTooLarge(U+" ++ hex cp ++ ")" + ErrUnpresentableLabel{} -> "UnpresentableLabel" + ErrFormNotAllowed _ f -> "FormNotAllowed(" ++ show f ++ ")" + ErrLabelInvalid _ r -> "LabelInvalid(" ++ showReason r ++ ")" + ErrAceInvalid _ r -> "AceInvalid(" ++ showAceReason r ++ ")" + ErrPunycodeOverflow{} -> "PunycodeOverflow" + ErrCrossLabelBidi i rule -> "CrossLabelBidi(" ++ show i ++ "," ++ show rule ++ ")" + where + hex cp = let s = showHex' cp in replicate (max 0 (4 - length s)) '0' ++ s + showHex' 0 = "0" + showHex' n = go n "" + where go 0 acc = acc + go k acc = go (k `div` 16) (h (k `mod` 16) : acc) + h x | x < 10 = toEnum (x + fromEnum '0') + | otherwise = toEnum (x - 10 + fromEnum 'a') + +showReason :: LabelReason -> String +showReason = \case + DisallowedCodepoint cp -> "Disallowed(U+" ++ hex4 cp ++ ")" + ContextRule cp -> "Context(U+" ++ hex4 cp ++ ")" + NotNFC -> "NotNFC" + LabelBidi rule -> "Bidi(" ++ show rule ++ ")" + HyphenViolation -> "Hyphen" + LeadingCombiningMark cp -> "LeadingCM(U+" ++ hex4 cp ++ ")" + where + hex4 cp = let s = sh cp in replicate (max 0 (4 - length s)) '0' ++ s + sh 0 = "0" + sh n = go n "" + where go 0 acc = acc + go k acc = go (k `div` 16) (h (k `mod` 16) : acc) + h x | x < 10 = toEnum (x + fromEnum '0') + | otherwise = toEnum (x - 10 + fromEnum 'a') + +showAceReason :: AceReason -> String +showAceReason = \case + BadPunycode -> "BadPunycode" + DecodedInvalid r -> "DecodedInvalid(" ++ showReason r ++ ")" + RoundTripMismatch -> "RoundTripMismatch" + +summary :: Counts -> String +summary c = unlines + [ " total : " ++ show (cTotal c) + , " pass : " ++ show (cPass c) + , " skip (explicit) : " ++ show (cSkip c) + , " xfail (explicit) : " ++ show (cXFail c) + , " auto-skip: cp-disagree : " ++ show (cCpDisagree c) + , " auto-skip: root-diff : " ++ show (cRootDifference c) + , " unexpected pass : " ++ show (cUPass c) + , " unexpected diff : " ++ show (cDiff c) + ] + +---------------------------------------------------------------------- +-- Test tree +---------------------------------------------------------------------- + +main :: IO () +main = do + ver <- cabalUnicodeVersion + vectors <- loadVectors + permissiveCps <- loadMappingTable + diffs <- loadExpectedDiffs defaultExpectedDiffsFile + hPutStrLn stderr $ + "unicode-conformance: Unicode " ++ ver + ++ "; " ++ show (length vectors) ++ " vectors, " + ++ show (Set.size permissiveCps) ++ " UTS#46-permissive codepoints" + unless (Map.null diffs) $ + hPutStrLn stderr $ + "unicode-conformance: loaded " ++ show (Map.size diffs) + ++ " expected-diff entries from " + ++ defaultExpectedDiffsFile + defaultMain (conformanceTree ver vectors diffs permissiveCps) + +conformanceTree + :: String + -> [Vector] + -> Map String (Disposition, String) + -> Set Int + -> TestTree +conformanceTree ver vectors diffs permissiveCps = + testGroup ("IdnaTestV2.txt for Unicode " ++ ver) + [ testCase "conformance" $ do + let !counts = foldl' (step permissiveCps diffs) emptyCounts vectors + hPutStrLn stderr (summary counts) + when (cDiff counts > 0 || cUPass counts > 0) $ do + let report = unlines $ + [ "" + , "unicode-conformance: comparator disagreed with UTS #46" + , "Nontransitional on " ++ show (cDiff counts) ++ " vector(s)" + , "and saw " ++ show (cUPass counts) + ++ " unexpected pass(es)." + , "" + , summary counts + ] + ++ (if null (cDiffSamples counts) + then [] + else ("First " ++ show (length (cDiffSamples counts)) + ++ " unexpected diffs:") + : cDiffSamples counts) + ++ (if null (cUPassSamples counts) + then [] + else ("First " ++ show (length (cUPassSamples counts)) + ++ " unexpected passes:") + : cUPassSamples counts) + ++ [ "" + , "Tune tests/data/idna-test-v2-expected-diffs.txt to mark" + , "known UTS#46-vs-IDNA2008 disagreements (skip/xfail)." + ] + assertFailure report + ] diff --git a/tests/vectors.json b/tests/vectors.json new file mode 100644 --- /dev/null +++ b/tests/vectors.json @@ -0,0 +1,2794 @@ +{ + "version": 1, + "description": "Conformance vectors for the idna2008 library. Hand-curated initial set; expand by adding more cases as the implementation evolves and as ports to other languages turn up edge cases worth pinning.", + "tests": [ + { + "name": "ldh-three-labels", + "input": "www.example.com", + "classes": "ldh", + "expect": { + "ok": { + "wireHex": "03777777076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "ldh-name-at-max-length", + "input": "a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a", + "classes": "ldh", + "expect": { + "ok": { + "wireHex": "016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016101610161016100", + "classes": [ + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "ldh-with-digits-and-hyphen", + "input": "abc-123.example", + "classes": "ldh", + "expect": { + "ok": { + "wireHex": "076162632d313233076578616d706c6500", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "wild-label-prefix", + "input": "*.example.com", + "classes": "all", + "expect": { + "ok": { + "wireHex": "012a076578616d706c6503636f6d00", + "classes": [ + "WILDLABEL", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "attrleaf-dmarc", + "input": "_dmarc.example.com", + "classes": "all", + "expect": { + "ok": { + "wireHex": "065f646d617263076578616d706c6503636f6d00", + "classes": [ + "ATTRLEAF", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "attrleaf-srv-style", + "input": "_25._tcp.example.com", + "classes": "all", + "expect": { + "ok": { + "wireHex": "035f3235045f746370076578616d706c6503636f6d00", + "classes": [ + "ATTRLEAF", + "ATTRLEAF", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "leading-hyphen-as-octet", + "input": "-foo.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "042d666f6f076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "trailing-hyphen-as-octet", + "input": "foo-.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "04666f6f2d076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "empty-label-middle", + "input": "foo..bar", + "classes": "all", + "expect": { + "err": { + "kind": "ErrEmptyLabel", + "labelIndex": 1 + } + } + }, + { + "name": "leading-hyphen-rejected-by-hostname-set", + "input": "-foo.example", + "classes": "host", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "trailing-hyphen-rejected-by-hostname-set", + "input": "foo-.example", + "classes": "host", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "attrleaf-rejected-by-hostname-set", + "input": "_dmarc.example", + "classes": "host", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "wildcard-rejected-by-hostname-set", + "input": "*.example", + "classes": "host", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "leading-digit-with-rtl-rejected", + "input": "0\u05d0\u05d1.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "LabelBidi", + "rule": "BidiRule1FirstNotLRAL", + "labelIndex": 0 + } + }, + "flags": "alabel-check,bidi-check" + }, + { + "name": "empty", + "input": "", + "classes": "all", + "expect": { + "ok": { + "wireHex": "00", + "classes": [] + } + } + }, + { + "name": "lone-dot", + "input": ".", + "classes": "all", + "expect": { + "ok": { + "wireHex": "00", + "classes": [] + } + } + }, + { + "name": "double-dot", + "input": "..", + "classes": "all", + "expect": { + "err": { + "kind": "ErrEmptyLabel" + } + } + }, + { + "name": "leading-dot", + "input": ".foo", + "classes": "all", + "expect": { + "err": { + "kind": "ErrEmptyLabel" + } + } + }, + { + "name": "label-63-bytes-ok", + "input": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "classes": "ldh", + "expect": { + "ok": { + "wireHex": "3f61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616100", + "classes": [ + "LDH" + ] + } + } + }, + { + "name": "label-64-bytes-too-long", + "input": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelTooLong", + "labelIndex": 0, + "length": 64 + } + } + }, + { + "name": "ldh-trailing-dot", + "input": "example.com.", + "classes": "ldh", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "attrleaf-not-allowed-by-idnForms", + "input": "_25._tcp.example.com", + "classes": "idn", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "r-ldh", + "input": "l---l.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "056c2d2d2d6c076578616d706c6500", + "classes": [ + "RLDH", + "LDH" + ] + } + } + }, + { + "name": "r-ldh-multi-letter-tag", + "input": "cd--storage-shelves.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "1363642d2d73746f726167652d7368656c766573076578616d706c6500", + "classes": [ + "RLDH", + "LDH" + ] + } + } + }, + { + "name": "decimal-escape", + "input": "f\\111oba.r", + "classes": "all", + "expect": { + "ok": { + "wireHex": "05666f6f6261017200", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "non-octet-with-non-ldh-ascii", + "input": "χν_foo.example.com", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid" + } + } + }, + { + "name": "u-label-double-hyphen-3-4-rejected", + "input": "αβ--γδ.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "ldh-name-too-long", + "input": "a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a.a", + "classes": "ldh", + "expect": { + "err": { + "kind": "ErrNameTooLong", + "length": 257 + } + } + }, + { + "name": "wildcard-leaf", + "input": "*.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "012a076578616d706c6500", + "classes": [ + "WILDLABEL", + "LDH" + ] + } + } + }, + { + "name": "wildcard-rejected-by-idnForms", + "input": "*.example", + "classes": "idn", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + }, + { + "name": "wildcard-allowed-by-explicit-set", + "input": "*.example", + "classes": "wildlabel,ldh", + "expect": { + "ok": { + "wireHex": "012a076578616d706c6500", + "classes": [ + "WILDLABEL", + "LDH" + ] + } + } + }, + { + "name": "asterisk-among-others-is-octet", + "input": "*foo.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "042a666f6f076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "two-asterisks-is-octet", + "input": "**.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "022a2a076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "all-specials-escaped-octet", + "input": "\"$()\\.;@\\\\foo.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0b222428292e3b405c666f6f076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "\\\"\\$\\(\\)\\.\\;\\@\\\\foo.example", + "displayFormLax": "\\\"\\$\\(\\)\\.\\;\\@\\\\foo.example", + "displayFormAscii": "\\\"\\$\\(\\)\\.\\;\\@\\\\foo.example" + } + } + }, + { + "name": "escaped-star-is-wildlabel", + "input": "\\*.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "012a076578616d706c6500", + "classes": [ + "WILDLABEL", + "LDH" + ] + } + } + }, + { + "name": "decimal-escaped-star-is-wildlabel", + "input": "\\042.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "012a076578616d706c6500", + "classes": [ + "WILDLABEL", + "LDH" + ] + } + } + }, + { + "name": "a-label-input", + "input": "xn--mnchen-3ya.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "u-label-encode-latin1", + "input": "münchen.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "real-a-label", + "input": "xn--mnchen-3ya.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "fake-a-label-disallowed-codepoint", + "input": "xn--a.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "05786e2d2d61076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ] + } + } + }, + { + "name": "loose-mode-classifies-as-alabel", + "input": "xn--a.example", + "classes": "all", + "flags": "", + "expect": { + "ok": { + "wireHex": "05786e2d2d61076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "fake-a-rejected-by-idnForms", + "input": "xn--a.example", + "classes": "idn", + "flags": "alabel-check", + "expect": { + "err": { + "kind": "ErrAceInvalid", + "labelIndex": 0, + "reason": "DecodedInvalid", + "innerReason": "DisallowedCodepoint", + "codepoint": 128 + } + } + }, + { + "name": "fake-a-emoji", + "input": "xn--ls8h.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ] + } + } + }, + { + "name": "strict-nfc-ascii-passes", + "input": "example.com", + "classes": "all", + "flags": "alabel-check,nfc-check", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "strict-nfc-precomposed-latin1-passes", + "input": "münchen.example", + "classes": "all", + "flags": "alabel-check,nfc-check", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "strict-nfc-decomposed-rejected", + "input": "a\u0301.example", + "classes": "all", + "flags": "alabel-check,nfc-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "NotNFC" + } + } + }, + { + "name": "default-nfc-decomposed-passes", + "input": "a\u0301.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "09786e2d2d612d786262076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a3-middle-dot-l-l", + "input": "l·l.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0a786e2d2d6c6c2d306561076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a3-middle-dot-leading", + "input": "·l.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 183, + "labelIndex": 0 + } + } + }, + { + "name": "a3-middle-dot-trailing", + "input": "l·.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 183, + "labelIndex": 0 + } + } + }, + { + "name": "a3-middle-dot-bad-neighbours", + "input": "x·y.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 183, + "labelIndex": 0 + } + } + }, + { + "name": "a4-greek-numeral-followed-by-greek", + "input": "͵α.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "09786e2d2d777661346a076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a4-greek-numeral-trailing", + "input": "α͵.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 885, + "labelIndex": 0 + } + } + }, + { + "name": "a4-greek-numeral-followed-by-non-greek", + "input": "͵a.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 885, + "labelIndex": 0 + } + } + }, + { + "name": "a5-hebrew-geresh-after-hebrew", + "input": "\u05d0\u05f3.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "09786e2d2d3464623465076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a5-hebrew-geresh-leading", + "input": "\u05f3\u05d0.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 1523, + "labelIndex": 0 + } + } + }, + { + "name": "a6-hebrew-gershayim-after-hebrew", + "input": "\u05d0\u05f4.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "09786e2d2d3464623665076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a6-hebrew-gershayim-after-non-hebrew", + "input": "a\u05f4.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 1524, + "labelIndex": 0 + } + } + }, + { + "name": "a7-katakana-middle-dot-with-hiragana", + "input": "あ・あ.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0b786e2d2d6c386a61383663076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a7-katakana-middle-dot-with-han", + "input": "一・一.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0d786e2d2d76656b373638666261076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a7-katakana-middle-dot-alone", + "input": "・.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 12539, + "labelIndex": 0 + } + } + }, + { + "name": "a7-katakana-middle-dot-without-hkh", + "input": "a・b.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 12539, + "labelIndex": 0 + } + } + }, + { + "name": "a8-arabic-indic-pure", + "input": "\u0660\u0661.example", + "classes": "all", + "flags": "", + "expect": { + "ok": { + "wireHex": "08786e2d2d38686263076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a9-extended-arabic-indic-pure", + "input": "\u06f0\u06f1.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "08786e2d2d646d6263076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a8-a9-mixed-rejected", + "input": "\u0660\u06f0.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 1632, + "labelIndex": 0 + } + } + }, + { + "name": "a2-zwj-after-virama", + "input": "\u0915\u094d\u200d\u0936.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0f786e2d2d313162306532637639336a076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a1-zwnj-after-virama", + "input": "\u0915\u094d\u200c\u0936.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0f786e2d2d313162306532637239336a076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a1-zwnj-d-d", + "input": "\u0644\u200c\u0644.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0c786e2d2d6768626133313971076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a1-zwnj-with-transparents", + "input": "\u0644\u064b\u200c\u064b\u0644.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0f786e2d2d6768626175613837303461076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "a2-zwj-without-virama", + "input": "\u0915\u200d\u0936.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 8205, + "labelIndex": 0 + } + } + }, + { + "name": "a2-zwj-leading", + "input": "\u200d\u0915.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 8205, + "labelIndex": 0 + } + } + }, + { + "name": "a1-zwnj-no-context", + "input": "a\u200cb.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 8204, + "labelIndex": 0 + } + } + }, + { + "name": "a1-zwnj-half-context", + "input": "\u0644\u200cb.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 8204, + "labelIndex": 0 + } + } + }, + { + "name": "a1-zwnj-leading", + "input": "\u200c\u0644.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "ContextRule", + "codepoint": 8204, + "labelIndex": 0 + } + } + }, + { + "name": "default-rejects-emoji-cp", + "input": "💩.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 128169, + "labelIndex": 0 + } + } + }, + { + "name": "allow-emoji-admits-pile-of-poo", + "input": "💩.example", + "classes": "all", + "flags": "emoji-ok", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "+emoji-ok", + "ok": "💩.example" + } + } + } + }, + { + "name": "allow-emoji-still-rejects-euro-sign", + "input": "€.example", + "classes": "all", + "flags": "emoji-ok", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 8364, + "labelIndex": 0 + } + } + }, + { + "name": "strict-alabel-with-allow-emoji-classifies-alabel", + "input": "xn--ls8h.example", + "classes": "all", + "flags": "alabel-check,emoji-ok", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "strict-alabel-without-allow-emoji-classifies-fakea", + "input": "xn--ls8h.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ] + } + } + }, + { + "name": "u-3002-rejected-without-mapdots", + "input": "abc。def", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 12290, + "labelIndex": 0 + } + } + }, + { + "name": "u-3002-separates-with-mapdots", + "input": "abc。def", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "036162630364656600", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "u-ff0e-separates-with-mapdots", + "input": "abc．def", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "036162630364656600", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "u-ff61-separates-with-mapdots", + "input": "abc｡def", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "036162630364656600", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "u-3002-trailing-with-mapdots", + "input": "abc。", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "0361626300", + "classes": [ + "LDH" + ] + } + } + }, + { + "name": "u-3002-alone-is-root-with-mapdots", + "input": "。", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "00", + "classes": [] + } + } + }, + { + "name": "mixed-separators-with-mapdots", + "input": "abc。def．ghi.jkl", + "classes": "all", + "flags": "map-dots", + "expect": { + "ok": { + "wireHex": "036162630364656603676869036a6b6c00", + "classes": [ + "LDH", + "LDH", + "LDH", + "LDH" + ] + } + } + }, + { + "name": "escaped-u-3002-rejected-with-mapdots", + "input": "abc\\。def", + "classes": "all", + "flags": "map-dots", + "expect": { + "err": { + "kind": "ErrBadEscape" + } + } + }, + { + "name": "mapnfc-normalizes-decomposed", + "input": "a\u0301.example", + "classes": "all", + "flags": "alabel-check,nfc-check,map-nfc", + "expect": { + "ok": { + "wireHex": "07786e2d2d316361076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapnfc-normalizes-without-strict", + "input": "a\u0301.example", + "classes": "all", + "flags": "alabel-check,map-nfc", + "expect": { + "ok": { + "wireHex": "07786e2d2d316361076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapnfc-precomposed-passthrough", + "input": "á.example", + "classes": "all", + "flags": "alabel-check,map-nfc", + "expect": { + "ok": { + "wireHex": "07786e2d2d316361076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapnfc-ldh-passthrough", + "input": "example.com", + "classes": "all", + "flags": "alabel-check,map-nfc", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "default-preserves-ascii-case", + "input": "EXAMPLE.com", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "074558414d504c4503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "mapcase-lowercases-ascii-ldh", + "input": "EXAMPLE.com", + "classes": "all", + "flags": "alabel-check,map-case", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "mapcase-lowercases-xn-prefix", + "input": "XN--mnchen-3ya.example", + "classes": "all", + "flags": "alabel-check,map-case", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "default-rejects-uppercase-in-ulabel", + "input": "München.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 77, + "labelIndex": 0 + } + } + }, + { + "name": "mapcase-admits-mixed-case-ulabel", + "input": "München.example", + "classes": "all", + "flags": "alabel-check,map-case", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapcase-lowercases-nonascii", + "input": "Übung.example", + "classes": "all", + "flags": "alabel-check,map-case", + "expect": { + "ok": { + "wireHex": "0c786e2d2d62756e672d7a7261076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapcase-matches-lowercase-input", + "input": "übung.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "0c786e2d2d62756e672d7a7261076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapwidth-fullwidth-latin-equals-ascii", + "input": "ａｂｃ.example", + "classes": "all", + "flags": "alabel-check,map-case,map-width", + "expect": { + "ok": { + "wireHex": "03616263076578616d706c6500", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "mapwidth-halfwidth-katakana", + "input": "ｱｲｳ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "ok": { + "wireHex": "09786e2d2d63636b6567076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "mapwidth-fullwidth-katakana-baseline", + "input": "アイウ.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "09786e2d2d63636b6567076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayForm": "アイウ.example", + "displayFormOpts": { + "flags": "bidi-check", + "ok": "アイウ.example" + } + } + } + }, + { + "name": "mapwidth-implies-mapdots-fullwidth-period", + "input": "abc．def", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "ok": { + "wireHex": "036162630364656600", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "mapwidth-implies-mapdots-halfwidth-period", + "input": "abc｡def", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "ok": { + "wireHex": "036162630364656600", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "default-rejects-fullwidth-period", + "input": "abc．def", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 65294, + "labelIndex": 0 + } + } + }, + { + "name": "mapwidth-rejects-non-ldh-ascii-result", + "input": "ａｂ＿ｃ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 95, + "labelIndex": 0 + } + } + }, + { + "name": "mapwidth-rejects-leading-hyphen-result", + "input": "－ａｂｃ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "HyphenViolation", + "labelIndex": 0 + } + } + }, + { + "name": "mapwidth-rejects-trailing-hyphen-result", + "input": "ａｂｃ－.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "HyphenViolation", + "labelIndex": 0 + } + } + }, + { + "name": "mapwidth-rejects-hy34-result", + "input": "ａｂ－－ｃｄ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "HyphenViolation", + "labelIndex": 0 + } + } + }, + { + "name": "default-fullwidth-underscore-input-rejected", + "input": "ａｂ＿ｃ.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 65345, + "labelIndex": 0 + } + } + }, + { + "name": "default-fullwidth-hy34-input-rejected", + "input": "ａｂ－－ｃｄ.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 65345, + "labelIndex": 0 + } + } + }, + { + "name": "ltr-only-no-trigger", + "input": "café.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "ok": { + "wireHex": "0b786e2d2d6361662d646d61076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "rtl-pure-hebrew-ok", + "input": "\u05d0\u05d1\u05d2.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "ok": { + "wireHex": "09786e2d2d3464626364076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayForm": "\u05d0\u05d1\u05d2.example" + } + } + }, + { + "name": "rtl-pure-arabic-ok", + "input": "\u0627\u0628.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "ok": { + "wireHex": "08786e2d2d6d676263076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayForm": "\u0627\u0628.example" + } + } + }, + { + "name": "rtl-with-l-rejected-with-checkbidi", + "input": "\u05d0a\u05d1.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "LabelBidi", + "rule": "BidiRule2RTLDisallowed", + "labelIndex": 0 + } + } + }, + { + "name": "rtl-with-l-ok-without-checkbidi", + "input": "\u05d0a\u05d1.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "0a786e2d2d612d7a686365076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "rtl-en-an-mix-rejected", + "input": "\u05d00\u0660.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "LabelBidi", + "rule": "BidiRule4ENANMix", + "labelIndex": 0 + } + } + }, + { + "name": "ltr-with-rtl-content-rejected", + "input": "a\u05d0.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "LabelBidi", + "rule": "BidiRule5LTRDisallowed", + "labelIndex": 0 + } + } + }, + { + "name": "rtl-trailing-nsm-ok", + "input": "\u05d0\u05d1\u05b0.example", + "classes": "all", + "flags": "alabel-check,bidi-check", + "expect": { + "ok": { + "wireHex": "0a786e2d2d376362376464076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "ldh-passthrough", + "input": "example.com", + "classes": "all", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ], + "displayForm": "example.com" + } + } + }, + { + "name": "real-a-label-decodes", + "input": "xn--mnchen-3ya.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ], + "displayForm": "münchen.example" + } + } + }, + { + "name": "u-label-input-round-trips", + "input": "münchen.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayForm": "münchen.example" + } + } + }, + { + "name": "fake-a-passthrough-control", + "input": "xn--a.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "05786e2d2d61076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayForm": "xn--a.example" + } + } + }, + { + "name": "fake-a-passthrough-emoji", + "input": "xn--ls8h.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayForm": "xn--ls8h.example" + } + } + }, + { + "name": "passthrough-escapes-dot", + "input": "xn--foo\\.bar.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0b786e2d2d666f6f2e626172076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "xn--foo\\.bar.example" + } + } + }, + { + "name": "passthrough-escapes-backslash", + "input": "a\\\\b.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "03615c62076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "a\\\\b.example" + } + } + }, + { + "name": "passthrough-escapes-at-sign", + "input": "foo\\@bar.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "07666f6f40626172076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "foo\\@bar.example" + } + } + }, + { + "name": "passthrough-escapes-control-byte", + "input": "a\\009b!.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0461096221076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "a\\009b!.example" + } + } + }, + { + "name": "passthrough-escapes-high-byte", + "input": "a\\252b!.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0461fc6221076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "a\\252b!.example" + } + } + }, + { + "name": "passthrough-domain-bytes-not-utf8-decoded", + "input": "a\\195\\188b!.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0561c3bc6221076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "a\\195\\188b!.example" + } + } + }, + { + "name": "lax-real-a-label", + "input": "xn--mnchen-3ya.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ], + "displayFormLax": "münchen.example" + } + } + }, + { + "name": "lax-fake-a-emoji-shows-codepoint", + "input": "xn--ls8h.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayFormLax": "💩.example" + } + } + }, + { + "name": "lax-fake-a-control-shows-codepoint", + "input": "xn--a.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "05786e2d2d61076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayFormLax": "\u0080.example" + } + } + }, + { + "name": "xn-prefix-with-non-ldh-ascii-inside-punycode-body", + "input": "xn--foo$bar--e6a.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "10786e2d2d666f6f246261722d2d653661076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ], + "displayForm": "xn--foo\\$bar--e6a.example", + "displayFormLax": "xn--foo\\$bar--e6a.example", + "displayFormAscii": "xn--foo\\$bar--e6a.example" + } + } + }, + { + "name": "lax-bad-punycode-falls-back", + "input": "xn---hgi.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "08786e2d2d2d686769076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayFormLax": "xn---hgi.example" + } + } + }, + { + "name": "lax-non-ace-label-passthrough", + "input": "example.com", + "classes": "all", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ], + "displayFormLax": "example.com" + } + } + }, + { + "name": "nocheck-plain-ldh", + "input": "example.com.", + "classes": "host", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ], + "displayFormOpts": { + "flags": "", + "ok": "example.com" + } + } + } + }, + { + "name": "nocheck-ace-decodes", + "input": "xn--mnchen-3ya.example.", + "classes": "host", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "", + "ok": "münchen.example" + } + } + } + }, + { + "name": "check-pure-latin-ok", + "input": "xn--mnchen-3ya.example.", + "classes": "host", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "bidi-check", + "ok": "münchen.example" + } + } + } + }, + { + "name": "check-pure-hebrew-ok", + "input": "\u05d0\u05d1\u05d2.example.", + "classes": "host", + "expect": { + "ok": { + "wireHex": "09786e2d2d3464626364076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "bidi-check", + "ok": "\u05d0\u05d1\u05d2.example" + } + } + } + }, + { + "name": "check-attrleaf-alongside-rtl-rejected", + "input": "_tcp.\u05d0\u05d1\u05d2.example.", + "classes": "all", + "expect": { + "ok": { + "wireHex": "045f74637009786e2d2d3464626364076578616d706c6500", + "classes": [ + "ATTRLEAF", + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "bidi-check", + "err": { + "kind": "ErrCrossLabelBidi", + "labelIndex": 0, + "rule": "BidiRule1FirstNotLRAL" + } + } + } + } + }, + { + "name": "check-fallback-renders-as-ascii", + "input": "_tcp.\u05d0\u05d1\u05d2.example.", + "classes": "all", + "expect": { + "ok": { + "wireHex": "045f74637009786e2d2d3464626364076578616d706c6500", + "classes": [ + "ATTRLEAF", + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "bidi-check,ascii-fallback", + "ok": "_tcp.xn--4dbcd.example" + } + } + } + }, + { + "name": "fallback-implies-checkbidi", + "input": "_tcp.\u05d0\u05d1\u05d2.example.", + "classes": "all", + "expect": { + "ok": { + "wireHex": "045f74637009786e2d2d3464626364076578616d706c6500", + "classes": [ + "ATTRLEAF", + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "ascii-fallback", + "ok": "_tcp.xn--4dbcd.example" + } + } + } + }, + { + "name": "check-digit-label-alongside-rtl-rejected", + "input": "123.\u05d0\u05d1\u05d2.example.", + "classes": "host", + "expect": { + "ok": { + "wireHex": "0331323309786e2d2d3464626364076578616d706c6500", + "classes": [ + "LDH", + "ULABEL", + "LDH" + ], + "displayFormOpts": { + "flags": "bidi-check", + "err": { + "kind": "ErrCrossLabelBidi", + "labelIndex": 0, + "rule": "BidiRule1FirstNotLRAL" + } + } + } + } + }, + { + "name": "empty-domain", + "input": ".", + "classes": "all", + "expect": { + "ok": { + "wireHex": "00", + "classes": [], + "displayFormOpts": { + "flags": "bidi-check", + "ok": "." + } + } + } + }, + { + "name": "laxulabel-admits-fakea-emoji", + "input": "xn--ls8h.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayFormOpts": { + "flags": "", + "classes": "+laxulabel", + "ok": "💩.example" + } + } + } + }, + { + "name": "bad-escape-trailing-backslash", + "input": "foo\\", + "classes": "all", + "expect": { + "err": { + "kind": "ErrBadEscape", + "labelIndex": 0 + } + } + }, + { + "name": "bad-escape-truncated-ddd", + "input": "foo\\1", + "classes": "all", + "expect": { + "err": { + "kind": "ErrBadEscape", + "labelIndex": 0 + } + } + }, + { + "name": "bad-escape-numeric-out-of-range", + "input": "foo\\999.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrBadEscape", + "labelIndex": 0 + } + } + }, + { + "name": "bad-escape-mid-ddd-non-digit", + "input": "foo\\1ab.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrBadEscape", + "labelIndex": 0 + } + } + }, + { + "name": "bad-escape-non-numeric-multi-byte-too-large", + "input": "foo\\€.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrBadEscape", + "labelIndex": 0 + } + } + }, + { + "name": "escape-non-numeric-multi-byte-in-range", + "input": "foo\\§.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "04666f6fa7076578616d706c6500", + "classes": [ + "OCTET", + "LDH" + ] + } + } + }, + { + "name": "u-label-leading-hyphen-rejected", + "input": "-α.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "u-label-trailing-hyphen-rejected", + "input": "α-.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "u-label-leading-combining-mark-rejected", + "input": "\u0300abc.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "LeadingCombiningMark", + "codepoint": 768 + } + } + }, + { + "name": "u-label-hy34-rejected", + "input": "хn----7sbahcidia5fd3ae5rh.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "a-label-decoded-hy34-rejected", + "input": "xn--n----7sbahcidia5fd3ae5rh-kyt.example", + "classes": "idn", + "flags": "alabel-check", + "expect": { + "err": { + "kind": "ErrAceInvalid", + "labelIndex": 0, + "reason": "DecodedInvalid", + "innerReason": "HyphenViolation" + } + } + }, + { + "name": "mapwidth-creates-hy34-nonascii-rejected", + "input": "ａｂ－－αβ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "mapwidth-creates-leading-hyphen-nonascii-rejected", + "input": "－αβγ.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "mapwidth-creates-trailing-hyphen-nonascii-rejected", + "input": "αβγ－.example", + "classes": "all", + "flags": "alabel-check,map-width", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "HyphenViolation" + } + } + }, + { + "name": "zwnj-at-label-end-rejected", + "input": "\u0628\u200c.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "ContextRule", + "codepoint": 8204 + } + } + }, + { + "name": "escape-mixed-with-non-ascii-rejected", + "input": "\\Aα.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrUnpresentableLabel", + "labelIndex": 0 + } + } + }, + { + "name": "lone-dot-lax-renders-as-dot", + "input": ".", + "classes": "all", + "expect": { + "ok": { + "wireHex": "00", + "classes": [], + "displayForm": ".", + "displayFormLax": ".", + "displayFormAscii": "." + } + } + }, + { + "name": "u-label-rejected-with-ldh-only-classes", + "input": "münchen.example", + "classes": "ldh", + "expect": { + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0, + "form": "ULABEL" + } + } + }, + { + "name": "unassigned-codepoint-rejected", + "input": "԰.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "DisallowedCodepoint", + "codepoint": 1328 + } + } + }, + { + "name": "mapwidth-collapses-to-four-char-ldh", + "input": "ａｂｃｄ.example", + "classes": "all", + "flags": "alabel-check,map-case,map-width", + "expect": { + "ok": { + "wireHex": "0461626364076578616d706c6500", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "uppercase-ace-body-classified-alabel", + "input": "xn--MNCHEN-3ya.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0e786e2d2d4d4e4348454e2d337961076578616d706c6500", + "classes": [ + "ALABEL", + "LDH" + ] + } + } + }, + { + "name": "maputs46-folds-reiwa-era", + "input": "㋿.example", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "ok": { + "wireHex": "0a786e2d2d6e6e7174316c076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "maputs46-folds-circled-cjk-ideograph", + "input": "㊀.example", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "ok": { + "wireHex": "07786e2d2d346771076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "default-rejects-reiwa-era-without-maputs46", + "input": "㋿.example", + "classes": "all", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 13055, + "labelIndex": 0 + } + } + }, + { + "name": "maputs46-rejects-dropped-u3297", + "input": "㊗.example", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 12951, + "labelIndex": 0 + } + } + }, + { + "name": "maputs46-rejects-dropped-u3299", + "input": "㊙.example", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 12953, + "labelIndex": 0 + } + } + }, + { + "name": "emojiok-rejects-squared-kanji-emoji-1F238", + "input": "🈸.example", + "classes": "all", + "flags": "+emoji-ok", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 127544, + "labelIndex": 0 + } + } + }, + { + "name": "emojiok-admits-non-mapped-emoji", + "input": "😀.example", + "classes": "all", + "flags": "+emoji-ok", + "expect": { + "ok": { + "wireHex": "08786e2d2d65323868076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "maputs46-rejects-squared-kanji-emoji-even-with-emoji-ok", + "input": "🈸.example", + "classes": "all", + "flags": "+map-uts46,+emoji-ok", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "reason": "DisallowedCodepoint", + "codepoint": 127544, + "labelIndex": 0 + } + } + }, + { + "name": "maputs46-implies-mapcase-mapwidth", + "input": "ＥＸＡＭＰＬＥ.com", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "ok": { + "wireHex": "076578616d706c6503636f6d00", + "classes": [ + "LDH", + "LDH" + ] + } + } + }, + { + "name": "maputs46-implies-mapdots-fullwidth-period", + "input": "㋿．example", + "classes": "all", + "flags": "+map-uts46", + "expect": { + "ok": { + "wireHex": "0a786e2d2d6e6e7174316c076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ] + } + } + }, + { + "name": "strict-set-uppercase-ascii-prefix-tolerated", + "input": "xn--Mller-kva.de", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0d786e2d2d4d6c6c65722d6b766102646500", + "classes": [ + "ALABEL", + "LDH" + ], + "displayForm": "müller.de", + "displayFormOpts": { + "flags": "default", + "classes": "-fakea,-laxulabel", + "ok": "müller.de" + } + } + } + }, + { + "name": "strict-set-rejects-uppercase-in-decoded-ulabel", + "input": "xn--mller-2pa.de", + "classes": "all", + "expect": { + "ok": { + "wireHex": "0d786e2d2d6d6c6c65722d32706102646500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayForm": "xn--mller-2pa.de", + "displayFormOpts": { + "flags": "default", + "classes": "-fakea,-laxulabel", + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + } + } + }, + { + "name": "strict-set-rejects-bad-punycode", + "input": "xn--z.example", + "classes": "all", + "expect": { + "ok": { + "wireHex": "05786e2d2d7a076578616d706c6500", + "classes": [ + "FAKEA", + "LDH" + ], + "displayForm": "xn--z.example", + "displayFormOpts": { + "flags": "default", + "classes": "-fakea,-laxulabel", + "err": { + "kind": "ErrFormNotAllowed", + "labelIndex": 0 + } + } + } + } + }, + { + "name": "laxulabel-admits-bidi-rule1-violation", + "input": "0\u05d0\u05d1.example", + "classes": "all,+laxulabel", + "flags": "alabel-check,bidi-check", + "expect": { + "ok": { + "wireHex": "0a786e2d2d302d30686364076578616d706c6500", + "classes": [ + "LAXULABEL", + "LDH" + ], + "displayFormLax": "0\u05d0\u05d1.example" + } + } + }, + { + "name": "laxulabel-admits-non-nfc", + "input": "a\u0301.example", + "classes": "all,+laxulabel", + "flags": "alabel-check,nfc-check", + "expect": { + "ok": { + "wireHex": "09786e2d2d612d786262076578616d706c6500", + "classes": [ + "LAXULABEL", + "LDH" + ], + "displayFormLax": "a\u0301.example" + } + } + }, + { + "name": "laxulabel-admits-trailing-hyphen-on-u-label", + "input": "ä-.example", + "classes": "all,+laxulabel", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "09786e2d2d2d2d7a6661076578616d706c6500", + "classes": [ + "LAXULABEL", + "LDH" + ], + "displayFormLax": "ä-.example" + } + } + }, + { + "name": "laxulabel-admits-emoji-codepoint", + "input": "💩.example", + "classes": "all,+laxulabel", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "08786e2d2d6c733868076578616d706c6500", + "classes": [ + "LAXULABEL", + "LDH" + ], + "displayFormLax": "💩.example" + } + } + }, + { + "name": "laxulabel-not-needed-for-clean-ulabel", + "input": "münchen.example", + "classes": "all", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "0e786e2d2d6d6e6368656e2d337961076578616d706c6500", + "classes": [ + "ULABEL", + "LDH" + ], + "displayForm": "münchen.example" + } + } + }, + { + "name": "laxulabel-admits-good-ldh-ascii-plus-invalid-unicode", + "input": "hello💩.example", + "classes": "all,+laxulabel", + "flags": "alabel-check", + "expect": { + "ok": { + "wireHex": "0f786e2d2d68656c6c6f2d6266313464076578616d706c6500", + "classes": [ + "LAXULABEL", + "LDH" + ], + "displayFormLax": "hello💩.example" + } + } + }, + { + "name": "laxulabel-not-rescued-when-non-ldh-ascii-mixed", + "input": "χν_foo.example.com", + "classes": "all", + "flags": "alabel-check", + "expect": { + "err": { + "kind": "ErrLabelInvalid", + "labelIndex": 0, + "reason": "DisallowedCodepoint", + "codepoint": 95 + } + } + } + ] +}