crypton 2.1.3 → 2.1.4
raw patch · 12 files changed
+160/−68 lines, 12 files
Files
- CHANGELOG.md +39/−0
- cbits/aes/armv8.c +18/−22
- cbits/aes/armv8_impl.c +13/−13
- cbits/crypton_armv8_target.h +40/−0
- cbits/crypton_f2m.c +2/−1
- cbits/crypton_sha256.c +19/−7
- cbits/p256/p256_verify.h +19/−0
- cbits/sha1_armv8.c +2/−6
- cbits/sha256_armv8.c +2/−6
- cbits/sha3_armv8.c +2/−6
- cbits/sha512_armv8.c +2/−6
- crypton.cabal +2/−1
CHANGELOG.md view
@@ -1,5 +1,44 @@ # CHANGELOG for crypton +## 2.1.4++2.1.3 could not be built from Hackage at all in the default configuration,+and is deprecated there. This release is that fix and three more.++* fix(cabal): the source distribution carries `cbits/p256/p256_verify.h`.+ No field named it, so it was absent from the 2.1.3 tarball, and+ `cbits/p256/p256_ec.c` includes it whenever `support_s2n_bignum` is on --+ which is every x86-64 and aarch64 machine that leaves the flag alone. The+ package built perfectly from a git checkout and not at all from Hackage.+ Reported as #270 by Laurent P. Rene de Cotret on the day 2.1.3 went out,+ fixed by gev in #271+* fix(armv8): the C builds with gcc before 13 again. A function that uses an+ AArch64 extension says so with `__attribute__((target(...)))`, and the+ spelling used -- `target("+sha3")` -- is one clang has always taken and gcc+ learned in 13. Before that the extension never reaches the function and an+ `always_inline` intrinsic that needs it cannot be inlined, which stops the+ build rather than slowing it. Naming the architecture beside the extension+ is understood by both compilers at every version, so gcc is given that+ spelling. Reported as #273 by gev, against 2.1.1, 2.1.2 and 2.1.3+* fix(sha256): SHA-256 on AArch64 is no longer five times slower than it was+ in 2.1.2 -- 644 MB/s against 3396 over 16 KiB on an Apple M4. The+ CRYPTOGAMS assembly picks its path from `crypton_armcap_P` rather than from+ a flag in the C, and the bit was set only while that flag was still+ unresolved; 2.1.3 added a constructor that resolves it before anything+ runs, so the bit was never set and the assembly took its generic path on+ every processor. SHA-1 was unaffected, its constructor setting the+ corresponding bit itself, and the SHA-512 assembly is x86 only. No test+ could have caught this: the answers were right all along, only slow+* test(ci): three jobs for the three ways the above went unnoticed. One+ builds the source distribution and then builds the library from it+ somewhere other than the checkout, since nothing had ever built a tarball+ and listing one is not building it. One installs gcc-12 and builds the C+ with it, the runner's own gcc being 13, which is why a report covering+ three releases never reproduced here. Both are verified against the bug+ they exist for: each was red before its fix and green after++# CHANGELOG for crypton+ ## 2.1.3 * fix(number): the arithmetic crypton falls back to when it is built without
cbits/aes/armv8.c view
@@ -39,11 +39,7 @@ * * "+crypto" rather than "crypto": GCC rejects the latter. */-#ifdef WITH_TARGET_ATTRIBUTES-#define TARGET_ARMV8_CRYPTO __attribute__((target("+crypto")))-#else-#define TARGET_ARMV8_CRYPTO-#endif+#include "crypton_armv8_target.h" /* forward round keys: nbr + 1 of them, written by the generic key expansion */ #define FWD(key) ((const uint8_t *) (key)->data)@@ -70,14 +66,14 @@ * instructions exist to remove -- but a key schedule is the one thing an * attacker most wants and it costs little to keep it out of the cache. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static uint32x4_t sub_word(uint32x4_t w) { return vreinterpretq_u32_u8( vaeseq_u8(vreinterpretq_u8_u32(w), vdupq_n_u8(0))); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static uint32x4_t sub_rot_word(uint32x4_t w) { const uint8x16_t s = vreinterpretq_u8_u32(sub_word(w));@@ -85,7 +81,7 @@ return vreinterpretq_u32_u8(vextq_u8(s, s, 1)); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_aes_armv8_init(aes_key *key, uint8_t *origkey, uint8_t size) { /* 2^0 .. 2^9 in GF(2^8), which is as far as any key size reaches */@@ -159,7 +155,7 @@ */ /* reverse all 16 bytes */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t bswap128(uint8x16_t v) { return vextq_u8(vrev64q_u8(v), vrev64q_u8(v), 8);@@ -173,7 +169,7 @@ #define SHL32(v, n) vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(v), (n))) #define SHR32(v, n) vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(v), (n))) -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t clmul_ll(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128(vmull_p64(@@ -181,7 +177,7 @@ (poly64_t) vgetq_lane_u64(vreinterpretq_u64_u8(b), 0))); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t clmul_lh(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128(vmull_p64(@@ -189,7 +185,7 @@ (poly64_t) vgetq_lane_u64(vreinterpretq_u64_u8(b), 1))); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t clmul_hl(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128(vmull_p64(@@ -197,7 +193,7 @@ (poly64_t) vgetq_lane_u64(vreinterpretq_u64_u8(b), 0))); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t clmul_hh(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128(vmull_high_p64(@@ -211,7 +207,7 @@ * over XOR: several products can be added together and fixed up just once, * which is what gf_mul4 below does. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline void clmul_pmull(uint8x16_t a, uint8x16_t b, uint8x16_t *lo, uint8x16_t *hi) {@@ -234,7 +230,7 @@ /* Shift the 256-bit product left by one to undo GCM's bit reflection, then * reduce modulo the GCM polynomial. This is the expensive half. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t gfred_pmull(uint8x16_t t3, uint8x16_t t6) { uint8x16_t t2, t4, t5, t7, t8, t9;@@ -273,7 +269,7 @@ return bswap128(t6); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static uint8x16_t gfmul_pmull(uint8x16_t a, const uint8_t *htable) { uint8x16_t lo, hi;@@ -291,7 +287,7 @@ * into one reduction: gf_mul4 uses the first four, the GCM loop all eight. * The table has sixteen slots, so they are free. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_aes_armv8_hinit_pmull(block128 *htable, const block128 *h) { uint8x16_t p;@@ -307,7 +303,7 @@ } } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_aes_armv8_gf_mul_pmull(block128 *a, const block128 *htable) { vst1q_u8((uint8_t *) a,@@ -320,7 +316,7 @@ * four products can be summed first and reduced once, which is where the * time goes. Aggregated reduction, from the Intel GCM paper. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_aes_armv8_gf_mul4_pmull(block128 *a, const block128 *blocks, const block128 *htable) {@@ -359,7 +355,7 @@ * the high half, and fold the bit that leaves the top back in as 0x87. The * block is little-endian, so lane 0 is the low half. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO static inline uint8x16_t gfmulx_neon(uint8x16_t v) { const uint64x2_t x = vreinterpretq_u64_u8(v);@@ -402,7 +398,7 @@ * its round count fixed, which is what lets the eight chains stay in * registers; the choice between them is made once per message here. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_aes_armv8_gcm_fused(uint8_t *out, const block128 *ht, aes_key *key, const uint8_t *nonce, const uint8_t *aad, uint32_t aadlen,@@ -429,7 +425,7 @@ } } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO int crypton_aes_armv8_gcm_fused_dec(uint8_t *out, const block128 *ht, aes_key *key, const uint8_t *nonce, const uint8_t *aad, uint32_t aadlen,
cbits/aes/armv8_impl.c view
@@ -74,7 +74,7 @@ } \ } while (0) -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_encrypt_block)(aes_block *output, aes_key *key, aes_block *input) { const uint8_t *rk = FWD(key);@@ -85,7 +85,7 @@ EACH1(STORE_OUT); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_decrypt_block)(aes_block *output, aes_key *key, aes_block *input) { const uint8_t *fwd = FWD(key);@@ -97,7 +97,7 @@ EACH1(STORE_OUT); } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_encrypt_ecb)(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) { const uint8_t *rk = FWD(key);@@ -115,7 +115,7 @@ } } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_decrypt_ecb)(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) { const uint8_t *fwd = FWD(key);@@ -136,7 +136,7 @@ /* CBC encryption chains, so there is nothing to interleave. It still gains * the round keys staying put. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_encrypt_cbc)(aes_block *output, aes_key *key, aes_block *_iv, aes_block *input, uint32_t nb_blocks) { const uint8_t *rk = FWD(key);@@ -158,7 +158,7 @@ #define CBC_KEEP(i) c[(i) + 1] = s[i]; #define CBC_XOR(i) vst1q_u8((uint8_t *) (output + (i)), veorq_u8(s[i], c[i])); -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_decrypt_cbc)(aes_block *output, aes_key *key, aes_block *_iv, aes_block *input, uint32_t nb_blocks) { const uint8_t *fwd = FWD(key);@@ -192,7 +192,7 @@ #define CTR_XOR(i) vst1q_u8(output + 16 * (i), \ veorq_u8(s[i], vld1q_u8(input + 16 * (i)))); -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_encrypt_ctr)(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) { const uint8_t *rk = FWD(key);@@ -349,7 +349,7 @@ vst1q_u8((uint8_t *) &gcm->tag, tag); \ } while (0) -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_gcm_encrypt)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { GCM_PROLOGUE;@@ -386,7 +386,7 @@ GCM_EPILOGUE; } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_gcm_decrypt)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { GCM_PROLOGUE;@@ -470,7 +470,7 @@ } while (0); #define XTS_TWEAK_ROLL(i) do { t[i] = tn[i]; } while (0); -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_encrypt_xts)(aes_block *output, aes_key *key, aes_key *key2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks) { const uint8_t *rk = FWD(key);@@ -514,7 +514,7 @@ } } -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_decrypt_xts)(aes_block *output, aes_key *key, aes_key *key2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks) { const uint8_t *fwd = FWD(key);@@ -602,7 +602,7 @@ ({ uint8_t buf_[16]; memset(buf_, 0, 16); memcpy(buf_, (p), (n)); \ vld1q_u8(buf_); }) -TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void SIZED(crypton_aes_armv8_gcm_fused)(uint8_t *out, const block128 *ht, aes_key *key, const uint8_t *nonce, const uint8_t *aad, uint32_t aadlen,@@ -696,7 +696,7 @@ * difference is the end: the tag is compared here rather than written, every * byte of it whichever way the answer goes. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO int SIZED(crypton_aes_armv8_gcm_fused_dec)(uint8_t *out, const block128 *ht, aes_key *key, const uint8_t *nonce, const uint8_t *aad, uint32_t aadlen,
+ cbits/crypton_armv8_target.h view
@@ -0,0 +1,40 @@+/*+ * Asking for an AArch64 extension on one function.+ *+ * The instructions these files use are extensions, so a translation unit+ * compiled for the baseline may not emit them, and the function that does has+ * to say which extension it needs. The two compilers spell that differently+ * and did not always:+ *+ * clang target("+crypto") for as long as it matters+ * GCC 13 and up target("+crypto") as well+ * GCC before 13 target("arch=armv8-a+crypto") -- the bare "+feature" form+ * is not understood, and the extension never reaches the+ * function, so an always_inline intrinsic that needs it+ * fails to inline and the build stops+ *+ * That last case is #273: gcc 12.2 on an aarch64 Linux could not build+ * cbits/sha3_armv8.c at all. Naming the architecture as well as the+ * extension is understood by every version of both compilers, so GCC is given+ * that spelling and clang keeps the shorter one, which leaves whatever+ * baseline the caller chose alone.+ */+#ifndef CRYPTON_ARMV8_TARGET_H+#define CRYPTON_ARMV8_TARGET_H++#ifdef WITH_TARGET_ATTRIBUTES+#if defined(__clang__)+#define CRYPTON_TARGET_ARMV8_CRYPTO __attribute__((target("+crypto")))+#define CRYPTON_TARGET_ARMV8_SHA3 __attribute__((target("+sha3")))+#else+#define CRYPTON_TARGET_ARMV8_CRYPTO \+ __attribute__((target("arch=armv8-a+crypto")))+#define CRYPTON_TARGET_ARMV8_SHA3 \+ __attribute__((target("arch=armv8.2-a+sha3")))+#endif+#else+#define CRYPTON_TARGET_ARMV8_CRYPTO+#define CRYPTON_TARGET_ARMV8_SHA3+#endif++#endif
cbits/crypton_f2m.c view
@@ -25,6 +25,7 @@ #include <stdlib.h> #include <string.h> #include <crypton_cpu.h>+#include "crypton_armv8_target.h" #include <crypton_bzero.h> #include <crypton_f2m.h> @@ -90,7 +91,7 @@ #define PMULL_ATTR #define PMULL_ALWAYS 1 #else-#define PMULL_ATTR __attribute__((target("+crypto")))+#define PMULL_ATTR CRYPTON_TARGET_ARMV8_CRYPTO #define PMULL_ALWAYS 0 #endif
cbits/crypton_sha256.c view
@@ -152,15 +152,27 @@ extern void crypton_sha256_asm_block_data_order(uint32_t state[8], const void *data, size_t blocks); +#ifdef WITH_ARMV8_SHA256_ASM+/* The assembly picks its path from crypton_armcap_P, so the answer to the+ * runtime check has to reach that word rather than the flag above. It is+ * set here, before there is a second thread, for the reason the constructor+ * in cbits/crypton_aes.c gives.+ *+ * This ran from sha256_asm_ready below until 2.1.3, guarded by the flag+ * still being unresolved -- which stopped happening when the constructor+ * above was added, so the bit was never set and the assembly took its+ * generic path. SHA-256 was 5.7 times slower on an Apple M4 for it. */+__attribute__((constructor))+static void sha256_armcap_ctor(void)+{+ if (crypton_sha256_armv8_available())+ crypton_armcap_P |= CRYPTON_ARMCAP_SHA256;+}+#endif+ static void sha256_asm_ready(void) {-#ifdef WITH_ARMV8_SHA256_ASM- if (sha256_use_armv8 < 0) {- if (crypton_sha256_armv8_available())- crypton_armcap_P |= CRYPTON_ARMCAP_SHA256;- sha256_use_armv8 = 1;- }-#else+#ifndef WITH_ARMV8_SHA256_ASM crypton_x86_ia32cap_resolve(); #endif }
+ cbits/p256/p256_verify.h view
@@ -0,0 +1,19 @@+#ifndef CRYPTON_P256_VERIFY_H+#define CRYPTON_P256_VERIFY_H++#include <stdint.h>++/*+ * n1*G + n2*Q, in variable time, as a Jacobian triple in the plain domain.+ *+ * Returns 1 when the answer is in `out`, and 0 when the walk met the one+ * case s2n-bignum's point addition does not cover -- adding a point to+ * itself -- in which case the caller works the answer out the constant-time+ * way instead. Nothing here is secret: it is all in the signature and the+ * public key.+ */+int crypton_p256_verify_mul(uint64_t out[12], const uint64_t n1[4],+ const uint64_t n2[4], const uint64_t qx[4],+ const uint64_t qy[4]);++#endif
cbits/sha1_armv8.c view
@@ -23,11 +23,7 @@ * baseline ARMv8-A may not use them; see sha256_armv8.c for the whole of that * argument. */-#ifdef WITH_TARGET_ATTRIBUTES-#define TARGET_ARMV8_CRYPTO __attribute__((target("+crypto")))-#else-#define TARGET_ARMV8_CRYPTO-#endif+#include "crypton_armv8_target.h" /* * A group of four rounds, and the schedule that goes with it.@@ -55,7 +51,7 @@ * One 64-byte block. `state` is the five words of chaining value in host * order, `buf` the block as it arrived, which SHA-1 reads big-endian. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_sha1_armv8_do_chunks(uint32_t state[5], const uint8_t *data, uint32_t blocks) {
cbits/sha256_armv8.c view
@@ -29,11 +29,7 @@ * * "+crypto" rather than "crypto": GCC rejects the latter. */-#ifdef WITH_TARGET_ATTRIBUTES-#define TARGET_ARMV8_CRYPTO __attribute__((target("+crypto")))-#else-#define TARGET_ARMV8_CRYPTO-#endif+#include "crypton_armv8_target.h" static const uint32_t K[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,@@ -58,7 +54,7 @@ * One 64-byte block. `state` is the eight words of chaining value in host * order, `buf` the block as it arrived, which SHA-256 reads big-endian. */-TARGET_ARMV8_CRYPTO+CRYPTON_TARGET_ARMV8_CRYPTO void crypton_sha256_armv8_do_chunk(uint32_t state[8], const uint8_t buf[64]) { uint32x4_t abcd, efgh, abcd_prev, efgh_prev, abcd_save, tmp;
cbits/sha3_armv8.c view
@@ -38,11 +38,7 @@ * of that argument. The flag for a build without attributes already asks for * "+sha3", which the SHA-512 path needed. */-#ifdef WITH_TARGET_ATTRIBUTES-#define TARGET_ARMV8_SHA3 __attribute__((target("+sha3")))-#else-#define TARGET_ARMV8_SHA3-#endif+#include "crypton_armv8_target.h" static const uint64_t rc[24] = { 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL,@@ -130,7 +126,7 @@ } while (0) /* the twenty-four rounds over the state, in place */-TARGET_ARMV8_SHA3+CRYPTON_TARGET_ARMV8_SHA3 void crypton_sha3_armv8_permute(uint64_t state[25]) { uint64x2_t a[25], b[25], c[5], d[5];
cbits/sha512_armv8.c view
@@ -28,11 +28,7 @@ * baseline ARMv8-A may not use them; mark the function that does. The * SHA-512 instructions live behind "+sha3" in both GCC and clang. */-#ifdef WITH_TARGET_ATTRIBUTES-#define TARGET_ARMV8_SHA3 __attribute__((target("+sha3")))-#else-#define TARGET_ARMV8_SHA3-#endif+#include "crypton_armv8_target.h" static const uint64_t K[80] = { 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,@@ -72,7 +68,7 @@ * two rounds and rotates which pair plays which part, so four steps return * to the start; a group of eight steps is one pass over the schedule. */-TARGET_ARMV8_SHA3+CRYPTON_TARGET_ARMV8_SHA3 void crypton_sha512_armv8_do_chunk(uint64_t state[8], const uint8_t buf[128]) { uint64x2_t ab, cd, ef, gh, ab0, cd0, ef0, gh0;
crypton.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.0 name: crypton-version: 2.1.3+version: 2.1.4 license: BSD-3-Clause license-file: LICENSE copyright: Vincent Hanquez <vincent@snarc.org>@@ -65,6 +65,7 @@ cbits/s2n/LICENSE cbits/s2n/README.md cbits/s2n/arm/*.S+ cbits/p256/*.h cbits/p256/gen_base_table.py cbits/s2n/import.sh cbits/s2n/include/*.h