update

#!/bin/sh
# Regenerate every Unicode-derived asset in the package from the
# upstream Unicode sources for the requested version:
#
#   * The library's table modules under
#     internal/Text/IDNA2008/Internal/*/Data.hs (Property, NFC,
#     Joining, Script, Width, Case, Bidi, Combining, Emoji).
#   * The UTS #46 reference data the `unicode-conformance` test
#     suite consults, bundled XZ-compressed under tests/data/.
#
# Usage:
#     internal/tools/update [<unicode-version>]
#
# When no version argument is supplied the script reads the default
# from the @x-unicode-version@ custom field in @idna2008.cabal@,
# which is the single source of truth for the Unicode version the
# library tracks.  Bump the field there and re-run this script to
# move to a new Unicode release; everything Unicode-derived stays
# in lockstep.
#
# All output paths are anchored at this script's directory, so the
# script can be run from anywhere.
#
# Override the Python interpreter via the PYTHON environment variable
# (e.g. PYTHON=python3.12 ./update 17.0.0).
#
# The IDNA disposition table is derived locally per RFC 5892 section 3 by
# genIdnaDerive.py from UCD inputs; we no longer fetch from the IANA
# IDNA-tables-properties registry, which lags Unicode releases (it
# tops out at 12.0.0 as of mid-2025).  The local derivation has been
# verified to reproduce IANA's published 12.0.0 CSV byte-for-byte.

set -eu

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
CABAL_FILE="${SCRIPT_DIR}/../../idna2008.cabal"

read_unicode_version() {
    awk '/^x-unicode-version:/ { print $2; exit }' "$CABAL_FILE"
}

DEFAULT_VERSION=$(read_unicode_version)
if [ -z "$DEFAULT_VERSION" ]; then
    echo "update: could not read x-unicode-version from $CABAL_FILE" >&2
    exit 1
fi

VERSION="${1:-$DEFAULT_VERSION}"
PYTHON="${PYTHON:-python3}"

#-----------------------------------------------------------------------
# Source URL templates -- edit here if Unicode reorganises its
# filesystem layout.  '%s' is the Unicode version string passed in
# as the first argument (e.g. "17.0.0").
#-----------------------------------------------------------------------

# Unicode UCD bundle: most files live under /Public/<ver>/ucd/.
UCD_BASE="https://www.unicode.org/Public/%s/ucd"

UNICODE_DATA_URL="${UCD_BASE}/UnicodeData.txt"
COMPOSITION_EXCLUSIONS_URL="${UCD_BASE}/CompositionExclusions.txt"
DERIVED_CORE_PROPERTIES_URL="${UCD_BASE}/DerivedCoreProperties.txt"
DERIVED_NORM_PROPS_URL="${UCD_BASE}/DerivedNormalizationProps.txt"
PROP_LIST_URL="${UCD_BASE}/PropList.txt"
HANGUL_SYLLABLE_TYPE_URL="${UCD_BASE}/HangulSyllableType.txt"
SCRIPTS_URL="${UCD_BASE}/Scripts.txt"
DERIVED_JOINING_TYPE_URL="${UCD_BASE}/extracted/DerivedJoiningType.txt"
DERIVED_COMBINING_CLASS_URL="${UCD_BASE}/extracted/DerivedCombiningClass.txt"
DERIVED_BIDI_CLASS_URL="${UCD_BASE}/extracted/DerivedBidiClass.txt"

# Emoji file: in 13.0+ it's part of the UCD bundle (Public/<ver>/ucd/emoji/);
# in 12.x and earlier it lived in a separate Public/emoji/<MAJ.MIN>/ tree.
EMOJI_UCD_URL="${UCD_BASE}/emoji/emoji-data.txt"
EMOJI_LEGACY_URL="https://www.unicode.org/Public/emoji/%s/emoji-data.txt"

# UTS #46 reference data used by the `unicode-conformance` test suite.
# The Unicode Consortium reorganised the IDNA data-file layout at
# Unicode 17.0.0 -- for >=17 the files live alongside the main UCD
# at /Public/<ver>/idna/, for <=16 at /Public/idna/<ver>/, and the
# /Public/idna/latest/ symlink continues to track the most-recently
# published version regardless of layout.
UTS46_NEW_BASE="https://www.unicode.org/Public/%s/idna"
UTS46_OLD_BASE="https://www.unicode.org/Public/idna/%s"

#-----------------------------------------------------------------------
# Compute concrete URLs and output paths.
#-----------------------------------------------------------------------

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
IDNA_DIR="${SCRIPT_DIR}/../Text/IDNA2008/Internal"

unidata_url=$(printf "$UNICODE_DATA_URL"            "$VERSION")
compex_url=$(printf  "$COMPOSITION_EXCLUSIONS_URL"  "$VERSION")
dcp_url=$(printf     "$DERIVED_CORE_PROPERTIES_URL" "$VERSION")
nfcprops_url=$(printf "$DERIVED_NORM_PROPS_URL"     "$VERSION")
plist_url=$(printf   "$PROP_LIST_URL"               "$VERSION")
hst_url=$(printf     "$HANGUL_SYLLABLE_TYPE_URL"    "$VERSION")
scripts_url=$(printf "$SCRIPTS_URL"                 "$VERSION")
jt_url=$(printf      "$DERIVED_JOINING_TYPE_URL"    "$VERSION")
ccc_url=$(printf     "$DERIVED_COMBINING_CLASS_URL" "$VERSION")
bidi_url=$(printf    "$DERIVED_BIDI_CLASS_URL"      "$VERSION")

case "$VERSION" in
    12.*|11.*|10.*|9.*|8.*|7.*|6.*)
        # Pre-13: emoji files in the legacy Public/emoji/<MAJ.MIN>/ tree.
        short_ver=$(echo "$VERSION" | cut -d. -f1,2)
        emoji_url=$(printf "$EMOJI_LEGACY_URL" "$short_ver")
        ;;
    *)
        emoji_url=$(printf "$EMOJI_UCD_URL" "$VERSION")
        ;;
esac

# Decide which UTS #46 layout to use based on the version's major
# component.  Versions >= 17 use the new under-UCD layout; older
# versions use /Public/idna/<ver>/.  An explicit "latest" value
# resolves through the symlink.
uts46_major=${VERSION%%.*}
case "$VERSION" in
    latest)
        uts46_test_url="https://www.unicode.org/Public/idna/latest/IdnaTestV2.txt"
        uts46_map_url="https://www.unicode.org/Public/idna/latest/IdnaMappingTable.txt"
        ;;
    *)
        case "$uts46_major" in
            ''|*[!0-9]*)
                uts46_base=$(printf "$UTS46_OLD_BASE" "$VERSION")
                ;;
            *)
                if [ "$uts46_major" -ge 17 ]; then
                    uts46_base=$(printf "$UTS46_NEW_BASE" "$VERSION")
                else
                    uts46_base=$(printf "$UTS46_OLD_BASE" "$VERSION")
                fi
                ;;
        esac
        uts46_test_url="${uts46_base}/IdnaTestV2.txt"
        uts46_map_url="${uts46_base}/IdnaMappingTable.txt"
        ;;
esac

TESTS_DATA_DIR="${SCRIPT_DIR}/../../tests/data"

#-----------------------------------------------------------------------
# Fetch all sources to a scratch dir, then run each generator.
#-----------------------------------------------------------------------

WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT

fetch() {
    # $1 = URL, $2 = local filename inside $WORK
    echo "fetching: $1"
    curl --fail --silent --show-error --location \
         --output "$WORK/$2" "$1"
}

fetch "$unidata_url"    UnicodeData.txt
fetch "$compex_url"     CompositionExclusions.txt
fetch "$dcp_url"        DerivedCoreProperties.txt
fetch "$nfcprops_url"   DerivedNormalizationProps.txt
fetch "$plist_url"      PropList.txt
fetch "$hst_url"        HangulSyllableType.txt
fetch "$scripts_url"    Scripts.txt
fetch "$jt_url"         DerivedJoiningType.txt
fetch "$ccc_url"        DerivedCombiningClass.txt
fetch "$bidi_url"       DerivedBidiClass.txt
fetch "$emoji_url"      emoji-data.txt
fetch "$uts46_test_url" IdnaTestV2.txt
fetch "$uts46_map_url"  IdnaMappingTable.txt

# Derive the IDNA disposition CSV locally per RFC 5892, then feed
# it into the existing CSV-to-Haskell generator.
echo "deriving: idna-tables-properties.csv (RFC 5892 section 3)"
"$PYTHON" "${SCRIPT_DIR}/genIdnaDerive.py" "$VERSION" \
    "$WORK/UnicodeData.txt"             \
    "$WORK/DerivedCoreProperties.txt"   \
    "$WORK/PropList.txt"                \
    "$WORK/HangulSyllableType.txt"      \
    "$WORK/DerivedNormalizationProps.txt" \
    > "$WORK/idna-tables-properties.csv"

run() {
    # $1 = output path; remaining args = command + args; stdout is redirected.
    out="$1"
    shift
    echo "writing:  $out"
    "$@" > "$out"
}

run "${IDNA_DIR}/Property/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaProperty.py" \
        "$VERSION" "$WORK/idna-tables-properties.csv"

run "${IDNA_DIR}/NFC/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaNFC.py" \
        "$VERSION" "$WORK/DerivedNormalizationProps.txt"

run "${IDNA_DIR}/NFC/Tables/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaNFCTables.py" \
        "$VERSION" "$WORK/UnicodeData.txt" "$WORK/CompositionExclusions.txt"

run "${IDNA_DIR}/Joining/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaJoining.py" \
        "$VERSION" "$WORK/DerivedJoiningType.txt" "$WORK/DerivedCombiningClass.txt"

run "${IDNA_DIR}/Script/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaScript.py" \
        "$VERSION" "$WORK/Scripts.txt"

run "${IDNA_DIR}/Width/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaWidth.py" \
        "$VERSION" "$WORK/UnicodeData.txt"

run "${IDNA_DIR}/Case/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaSimpleLower.py" \
        "$VERSION" "$WORK/UnicodeData.txt"

run "${IDNA_DIR}/Bidi/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaBidi.py" \
        "$VERSION" "$WORK/DerivedBidiClass.txt"

run "${IDNA_DIR}/Combining/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaCombining.py" \
        "$VERSION" "$WORK/UnicodeData.txt"

run "${IDNA_DIR}/Emoji/Data.hs" \
    "$PYTHON" "${SCRIPT_DIR}/genIdnaEmoji.py" \
        "$VERSION" "$WORK/emoji-data.txt" "$WORK/IdnaMappingTable.txt"

#-----------------------------------------------------------------------
# Bundle the UTS #46 reference data for the `unicode-conformance` test
# suite.  The files are committed gzip-compressed under tests/data/
# and unpacked at test time by Codec.Compression.GZip, so 'cabal
# test' runs offline without a .cache/ directory or network
# dependency.  Gzip (rather than xz) keeps the build deps to the
# Haskell @zlib@ package, which ships everywhere and doesn't need a
# pkg-config dance to find a system C library.
#-----------------------------------------------------------------------

mkdir -p "$TESTS_DATA_DIR"

bundle() {
    # $1 = source filename in $WORK; $2 = destination basename in
    # $TESTS_DATA_DIR (the .gz extension is appended).  The @-n@
    # flag suppresses the original filename and mtime from the gzip
    # header so that re-running @update@ against the same source
    # bytes produces a byte-identical output -- no spurious diffs
    # in the committed tree after a no-op regeneration.
    src="$WORK/$1"
    dst="${TESTS_DATA_DIR}/$2.gz"
    echo "bundling: $dst"
    gzip -9 -n -c "$src" > "$dst"
}

bundle IdnaTestV2.txt       IdnaTestV2.txt
bundle IdnaMappingTable.txt IdnaMappingTable.txt

echo
echo "All auto-generated IDNA tables are now aligned with Unicode $VERSION."
echo "UTS #46 conformance data refreshed under ${TESTS_DATA_DIR}/."