diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # CHANGELOG for crypton +## 2.2.1 + +**A binary built on one x86-64 machine runs on another again.** Since +2.1.8 the `-mavx2 -mbmi2` written for the ML-KEM and ML-DSA backends have +been compiling every C file in the library -- `cc-options` in a conditional +block belong to the component, not to the block -- so the compiler put AVX2 +and BMI2 into ordinary code that has no dispatch and asks nothing at +runtime: Ed25519, Argon2, Curve25519, P-256, Ed448, and the older hashes +among them. A binary built where AVX2 exists then died with `SIGILL` where +it does not, in Ed25519 rather than in anything post-quantum. + +The two backends also chose their implementation when the library was +compiled and called it unchecked. They ask the processor now, as everything +else in crypton that uses an instruction beyond the baseline already did, +and fall back to the portable C where the answer is no. AArch64 asks about +the ARMv8.4 Keccak instructions for the same reason. + +Nothing in the API changes. + +* fix: stop raising the x86-64 baseline for the whole library + [#317](https://github.com/kazu-yamamoto/crypton/pull/317) +* fix: ask the processor before using the ML-KEM and ML-DSA backends + [#317](https://github.com/kazu-yamamoto/crypton/pull/317) +* test(ct): link the processor check into the ML-KEM backend build + [#317](https://github.com/kazu-yamamoto/crypton/pull/317) + ## 2.2.0 `ProcessorOption` is no longer a sum of constructors. It is a number with diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -111,31 +111,31 @@ Throughput in MB/s, **higher is better**: -| | crypton 1.1.5 | crypton 2.1.5 | OpenSSL | 2.1.5 / OpenSSL | +| | crypton 1.1.5 | crypton 2.2.0 | OpenSSL | 2.2.0 / OpenSSL | | --- | ---: | ---: | ---: | ---: | -| AES-128-GCM | 1362 | **6038** | 4055 | 1.49 | -| AES-256-GCM | 1093 | **5462** | 3770 | 1.45 | -| ChaCha20-Poly1305 | 399 | 2211 | 2229 | 0.99 | -| SHA-1 | 727 | 1678 | 1673 | 1.00 | -| SHA-256 | 290 | 1585 | 1579 | 1.00 | -| SHA-512 | 463 | 804 | 751 | 1.07 | -| SHA3-256 | 109 | 424 | 425 | 1.00 | +| AES-128-GCM | 1335 | **6034** | 4050 | 1.49 | +| AES-256-GCM | 1093 | **5484** | 3783 | 1.45 | +| ChaCha20-Poly1305 | 399 | 2198 | 2213 | 0.99 | +| SHA-1 | 729 | 1695 | 1673 | 1.01 | +| SHA-256 | 290 | 1585 | 1571 | 1.01 | +| SHA-512 | 449 | 770 | 752 | 1.02 | +| SHA3-256 | 109 | 422 | 425 | 0.99 | Time per operation in microseconds, **lower is better**: -| | crypton 1.1.5 | crypton 2.1.5 | OpenSSL | OpenSSL / 2.1.5 | +| | crypton 1.1.5 | crypton 2.2.0 | OpenSSL | OpenSSL / 2.2.0 | | --- | ---: | ---: | ---: | ---: | -| X25519 | 45.31 | 28.41 | 36.48 | 1.28 | -| ECDH P-256 | 165.4 | 51.14 | 51.65 | 1.01 | -| ECDH P-384 | 2278 | **165.0** | 847.5 | 5.13 | -| Ed25519 sign | 30.03 | 18.62 | 33.71 | 1.81 | -| Ed25519 verify | 48.05 | 47.66 | 110.6 | 2.32 | -| ECDSA P-256 sign | 81.70 | 18.96 | 21.87 | 1.15 | -| ECDSA P-256 verify | 233.3 | 70.47 | 67.52 | 0.96 | -| ECDSA P-384 sign | 2264 | **303.4** | 890.1 | 2.93 | -| ECDSA P-384 verify | 2676 | **471.4** | 721.5 | 1.53 | -| RSA-2048 sign/decrypt | 759.4 | 612.0 | 659.4 | 1.08 | -| RSA-2048 verify/encrypt | 33.56 | 30.21 | 18.86 | 0.62 | +| X25519 | 44.63 | 28.29 | 36.52 | 1.29 | +| ECDH P-256 | 164.8 | 50.85 | 51.63 | 1.02 | +| ECDH P-384 | 2255 | **164.8** | 835.4 | 5.07 | +| Ed25519 sign | 29.27 | 18.54 | 33.63 | 1.81 | +| Ed25519 verify | 48.44 | 51.80 | 109.8 | 2.12 | +| ECDSA P-256 sign | 81.91 | 18.81 | 21.81 | 1.16 | +| ECDSA P-256 verify | 233.1 | 70.00 | 67.57 | 0.97 | +| ECDSA P-384 sign | 2287 | **300.0** | 878.7 | 2.93 | +| ECDSA P-384 verify | 2673 | **468.3** | 723.3 | 1.54 | +| RSA-2048 sign/decrypt | 759.2 | 617.8 | 659.8 | 1.07 | +| RSA-2048 verify/encrypt | 33.07 | 30.08 | 18.87 | 0.63 | ### AArch64 @@ -144,35 +144,35 @@ Throughput in MB/s, **higher is better**: -| | crypton 1.1.5 | crypton 2.1.5 | OpenSSL | 2.1.5 / OpenSSL | +| | crypton 1.1.5 | crypton 2.2.0 | OpenSSL | 2.2.0 / OpenSSL | | --- | ---: | ---: | ---: | ---: | -| AES-128-GCM | 127 | **12422** | 10846 | 1.15 | -| AES-256-GCM | 98 | **9721** | 9197 | 1.06 | -| ChaCha20-Poly1305 | 771 | 2319 | 2250 | 1.03 | -| SHA-1 | 1209 | 3389 | 3361 | 1.01 | -| SHA-256 | 474 | 3400 | 3362 | 1.01 | -| SHA-512 | 730 | 1880 | 1883 | 1.00 | -| SHA3-256 | 550 | 1075 | 1065 | 1.01 | +| AES-128-GCM | 129 | **11847** | 10770 | 1.10 | +| AES-256-GCM | 98 | **9770** | 9187 | 1.06 | +| ChaCha20-Poly1305 | 778 | 2312 | 2257 | 1.02 | +| SHA-1 | 1237 | 3393 | 3368 | 1.01 | +| SHA-256 | 485 | 3398 | 3370 | 1.01 | +| SHA-512 | 740 | 1882 | 1878 | 1.00 | +| SHA3-256 | 552 | 1081 | 1079 | 1.00 | Time per operation in microseconds, **lower is better**: -| | crypton 1.1.5 | crypton 2.1.5 | OpenSSL | OpenSSL / 2.1.5 | +| | crypton 1.1.5 | crypton 2.2.0 | OpenSSL | OpenSSL / 2.2.0 | | --- | ---: | ---: | ---: | ---: | -| X25519 | 18.27 | **12.22** | 15.53 | 1.27 | -| ECDH P-256 | 68.70 | **20.43** | 24.77 | 1.21 | -| ECDH P-384 | 3328 | **73.50** | 372.6 | 5.07 | -| Ed25519 sign | 13.58 | **7.75** | 13.23 | 1.71 | -| Ed25519 verify | 18.28 | 18.17 | 34.76 | 1.91 | -| ECDSA P-256 sign | 31.97 | **6.55** | 10.92 | 1.67 | -| ECDSA P-256 verify | 95.63 | **26.80** | 32.68 | 1.22 | -| ECDSA P-384 sign | 3219 | **124.1** | 394.2 | 3.18 | -| ECDSA P-384 verify | 3870 | **203.1** | 326.5 | 1.61 | -| RSA-2048 sign/decrypt | 447.9 | 460.1 | 319.9 | 0.70 | -| RSA-2048 verify/encrypt | 18.23 | 15.12 | 8.405 | 0.56 | +| X25519 | 17.62 | **11.92** | 15.35 | 1.29 | +| ECDH P-256 | 66.93 | **20.32** | 24.53 | 1.21 | +| ECDH P-384 | 3308 | **73.17** | 371.1 | 5.07 | +| Ed25519 sign | 13.67 | **7.967** | 13.25 | 1.66 | +| Ed25519 verify | 18.25 | 18.22 | 35.18 | 1.93 | +| ECDSA P-256 sign | 32.33 | **7.00** | 10.82 | 1.55 | +| ECDSA P-256 verify | 95.78 | **27.42** | 32.69 | 1.19 | +| ECDSA P-384 sign | 3244 | **124.3** | 394.6 | 3.17 | +| ECDSA P-384 verify | 3900 | **203.2** | 327.4 | 1.61 | +| RSA-2048 sign/decrypt | 447.6 | 458.1 | 322.0 | 0.70 | +| RSA-2048 verify/encrypt | 18.20 | 15.15 | 8.450 | 0.56 | ### What the numbers say -There are two changes behind the 1.1.5 column and the 2.1.5 one, not a +There are two changes behind the 1.1.5 column and the 2.2.0 one, not a single steady improvement. The first, in 2.0.0, was a rewrite: the bulk algorithms moved into C, the @@ -210,7 +210,7 @@ hiding the exponent is what the difference bought. On x86-64 that cost is more than repaid -- s2n-bignum's Montgomery multiplication is twice the C's, because the C cannot form the two carry chains `ADCX` and `ADOX` give, and -2.1.5 signs in less than 1.1.5 took while keeping what 2.0.0 gained. On +2.2.0 signs in less than 1.1.5 took while keeping what 2.0.0 gained. On AArch64 there is nothing to use: s2n-bignum has no generic routine for it, and the same five that help on x86-64 measure level with the C there, so the C stays and the gap with it. No portable C closes that gap either -- the @@ -236,7 +236,7 @@ takes a block at a time where crypton takes two. The same idea as theirs, one step further down the feature ladder; not a better one. -The 512-bit path arrived after 2.1.2, so it is in the 2.1.5 column -- but +The 512-bit path arrived after 2.1.2, so it is in the 2.2.0 column -- but neither machine in the tables above has AVX-512, so neither column shows it. On the runners that do, measured over 16 KiB in MB/s: an EPYC 9V45 (Zen 5) goes from 9616 to 14268 with it, a Xeon 6973P-C from 8095 to 9848, a Xeon @@ -261,8 +261,20 @@ key from the secret key every time it signs, so that a caller who passes a public key that does not match cannot be made to leak the private one. That costs a second scalar multiplication, which OpenSSL's signing does not pay -- -and the row is still 1.71 on AArch64 and 1.81 on x86-64, so the safety is had +and the row is still 1.66 on AArch64 and 1.81 on x86-64, so the safety is had for nothing here rather than paid for. + +Ed25519 verification wants one too, for a less happy reason. On x86-64 the +row is about eight per cent slower than it was, and the step is at 2.1.8: +2.1.5, 2.1.6 and 2.1.7 all measure near 48 microseconds there, and +2.1.8, 2.1.10 and 2.2.0 all measure near 52. Nothing in Ed25519 changed +across that step. No commit touches `cbits/ed25519` between those releases, +the compiled `ed25519.o` is byte for byte the same object either side of it, +and AArch64 did not move at all. What 2.1.8 did was vendor ML-KEM and +ML-DSA, some seventy thousand lines of C that Ed25519 never calls. The same +instructions, put somewhere else; signing, which is the same file and a +different routine, is where it was. The row is still better than twice +OpenSSL's, and there is nothing in Ed25519 to fix. SHA-1 is in the tables because a number of protocols and file formats still ask for it, not because it is a good choice for anything new. The algorithms diff --git a/cbits/mldsa/crypton_mldsa.h b/cbits/mldsa/crypton_mldsa.h --- a/cbits/mldsa/crypton_mldsa.h +++ b/cbits/mldsa/crypton_mldsa.h @@ -4,7 +4,8 @@ * choice crypton makes is made here instead of by editing it. * * Included first by both crypton_mldsa.c and crypton_mldsa_asm.S, so it must - * hold nothing but preprocessor directives. + * hold nothing the assembler cannot read: directives, and C behind + * __ASSEMBLER__. */ #ifndef CRYPTON_MLDSA_H #define CRYPTON_MLDSA_H @@ -36,6 +37,43 @@ #ifdef CRYPTON_MLDSA_NATIVE_BACKEND #define MLD_CONFIG_USE_NATIVE_BACKEND_ARITH #define MLD_CONFIG_USE_NATIVE_BACKEND_FIPS202 + +/* + * Which of the backend's instructions this processor actually has. + * + * Without this both packages decide at compile time: the backend is built + * because the architecture has the instructions in principle, and is then + * called with no check. Upstream says so plainly -- if the machine that + * runs the binary may differ from the one that built it, this function + * MUST be provided -- and crypton is a library whose builds are copied + * between machines and shipped by distributions. Without it a binary + * built on a host with AVX2 dies with SIGILL on one without (#316). + * + * The dispatch sites are upstream's; each falls back to the portable C + * when this answers no. The answers come from crypton's own detection, + * the same that AES-NI, PCLMUL and the s2n-bignum variants use. + */ +#define MLD_CONFIG_CUSTOM_CAPABILITY_FUNC +#ifndef __ASSEMBLER__ +#include "src/sys.h" +#include "crypton_cpu.h" + +static MLD_INLINE int mld_sys_check_capability(mld_sys_cap cap) +{ +#if defined(__x86_64__) + if (cap == MLD_SYS_CAP_X86_64_AVX2) + return (crypton_x86_simd_features() & CRYPTON_X86_AVX2) != 0; +#elif defined(__aarch64__) + /* NEON is not optional on AArch64; the Keccak instructions are. */ + if (cap == MLD_SYS_CAP_AARCH64_NEON) + return 1; + if (cap == MLD_SYS_CAP_AARCH64_SHA3) + return (crypton_arm_features() & CRYPTON_ARM_SHA3) != 0; +#endif + (void)cap; + return 0; +} +#endif /* !__ASSEMBLER__ */ #endif #endif /* CRYPTON_MLDSA_H */ diff --git a/cbits/mlkem/crypton_mlkem.h b/cbits/mlkem/crypton_mlkem.h --- a/cbits/mlkem/crypton_mlkem.h +++ b/cbits/mlkem/crypton_mlkem.h @@ -4,7 +4,8 @@ * choice crypton makes is made here instead of by editing it. * * Included first by both crypton_mlkem.c and crypton_mlkem_asm.S, so it must - * hold nothing but preprocessor directives. + * hold nothing the assembler cannot read: directives, and C behind + * __ASSEMBLER__. */ #ifndef CRYPTON_MLKEM_H #define CRYPTON_MLKEM_H @@ -36,6 +37,43 @@ #ifdef CRYPTON_MLKEM_NATIVE_BACKEND #define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH #define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + +/* + * Which of the backend's instructions this processor actually has. + * + * Without this both packages decide at compile time: the backend is built + * because the architecture has the instructions in principle, and is then + * called with no check. Upstream says so plainly -- if the machine that + * runs the binary may differ from the one that built it, this function + * MUST be provided -- and crypton is a library whose builds are copied + * between machines and shipped by distributions. Without it a binary + * built on a host with AVX2 dies with SIGILL on one without (#316). + * + * The dispatch sites are upstream's; each falls back to the portable C + * when this answers no. The answers come from crypton's own detection, + * the same that AES-NI, PCLMUL and the s2n-bignum variants use. + */ +#define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC +#ifndef __ASSEMBLER__ +#include "src/sys.h" +#include "crypton_cpu.h" + +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +{ +#if defined(__x86_64__) + if (cap == MLK_SYS_CAP_X86_64_AVX2) + return (crypton_x86_simd_features() & CRYPTON_X86_AVX2) != 0; +#elif defined(__aarch64__) + /* NEON is not optional on AArch64; the Keccak instructions are. */ + if (cap == MLK_SYS_CAP_AARCH64_NEON) + return 1; + if (cap == MLK_SYS_CAP_AARCH64_SHA3) + return (crypton_arm_features() & CRYPTON_ARM_SHA3) != 0; +#endif + (void)cap; + return 0; +} +#endif /* !__ASSEMBLER__ */ #endif #endif /* CRYPTON_MLKEM_H */ diff --git a/cbits/tests/ct/cpu_stub.c b/cbits/tests/ct/cpu_stub.c new file mode 100644 --- /dev/null +++ b/cbits/tests/ct/cpu_stub.c @@ -0,0 +1,19 @@ +/* + * One symbol, so that cbits/crypton_cpu.c can be linked on its own. + * + * The ML-KEM backend asks crypton_cpu.c whether this processor has AVX2. + * That file also carries crypton_cpu_option, which the Haskell side calls + * and which reaches into cbits/crypton_aes.c, so the linker wants + * crypton_aes_cpu_init as well -- and behind it the whole AES build, which + * has nothing to do with what is being measured here. This stands in for + * it. Nothing in an ML-KEM driver calls it; if anything ever did, zeroes + * say "no AES instructions", which is the safe answer. + */ +#include +#include "crypton_cpu.h" + +uint8_t *crypton_aes_cpu_init(void) +{ + static uint8_t none[CPU_OPTION_COUNT]; + return none; +} diff --git a/cbits/tests/ct/run.sh b/cbits/tests/ct/run.sh --- a/cbits/tests/ct/run.sh +++ b/cbits/tests/ct/run.sh @@ -71,7 +71,13 @@ # And the hand-written backend, where the architecture has one. Both builds # have to be silent: unlike the AES pair below there is no variable-time # implementation here for one of them to expose. -mlkem_native_src="$mlkem_src cbits/mlkem/crypton_mlkem_asm.S" +# +# crypton_cpu.c because the backend asks it whether this processor has the +# instructions before using them; without it the link fails on the +# capability function in cbits/mlkem/crypton_mlkem.h. cpu_stub.c is the one +# symbol that file needs from the AES build, which is not wanted here. +mlkem_native_src="$mlkem_src cbits/mlkem/crypton_mlkem_asm.S cbits/crypton_cpu.c \ + cbits/tests/ct/cpu_stub.c" mlkem_native_inc="$mlkem_inc -DCRYPTON_MLKEM_NATIVE_BACKEND" @@ -246,12 +252,22 @@ # The ML-KEM backend, on both architectures that have one. x86-64 wants the # flags its own build system passes, or the AVX2 sources do not assemble. +# +# The backend now asks the processor before it runs, so on x86-64 without +# AVX2 this build measures the portable C under a name that says otherwise. +# Say so and skip, rather than pass while checking the thing the plain +# mlkem build above has already checked. case $(uname -m) in aarch64 | arm64) run_one mlkem_native "$mlkem_native_src" "$mlkem_native_inc" mlkem ;; x86_64 | amd64) - run_one mlkem_native "$mlkem_native_src" "$mlkem_native_inc -mavx2 -mbmi2" mlkem + if grep -qw avx2 "${CRYPTON_CT_CPUINFO:-/proc/cpuinfo}" 2>/dev/null; then + run_one mlkem_native "$mlkem_native_src" "$mlkem_native_inc -mavx2 -mbmi2" mlkem + else + echo "skip mlkem_native: this processor has no AVX2, so the backend" + echo " would fall back to the C the mlkem build above measured" + fi ;; esac diff --git a/crypton.cabal b/crypton.cabal --- a/crypton.cabal +++ b/crypton.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: crypton -version: 2.2.0 +version: 2.2.1 -- crypton's own code is BSD-3-Clause. The parts of -- cbits/aes/gcm_fused_x86.c that follow picotls's fusion are MIT, and the -- vendored s2n-bignum assembly in cbits/s2n is taken under ISC, and the @@ -736,11 +736,15 @@ cbits/mldsa/crypton_mldsa_asm.S -- What the two packages' own build systems pass for the x86-64 - -- backend; without them the AVX2 sources do not assemble. AArch64 - -- needs nothing beyond the baseline for the arithmetic, and takes - -- the armv8.4 Keccak from the -march the block below already sets. + -- backend, and only for the assembly: cc-options in a conditional + -- are the component's, not the block's, so -mavx2 here would raise + -- the baseline of every C file in the library -- Ed25519, Argon2, + -- P-256 and the rest -- and gcc would vectorise code that asks for + -- no such thing, with nothing to check at runtime. The backends' + -- own AVX2 is all in the .S. AArch64 needs nothing beyond the + -- baseline for the arithmetic, and takes the armv8.4 Keccak from + -- the -march the block below already sets. if arch(x86_64) - cc-options: -mavx2 -mbmi2 asm-options: -mavx2 -mbmi2 if (flag(support_aesni) && arch(aarch64))