ppad-sha512-0.2.6: cbits/sha512_native.c
#include <stdint.h>
#include <string.h>
/*
* The x86 SHA512 intrinsics arrived in GCC 14 and Clang 18; older
* compilers get the stubs below.
*/
#if defined(__x86_64__) && defined(__has_include)
#if __has_include(<sha512intrin.h>)
#define HAVE_X86_SHA512
#endif
#endif
#if defined(__aarch64__)
#include <arm_neon.h>
#if defined(__APPLE__)
#include <sys/sysctl.h>
#elif defined(__linux__)
#include <sys/auxv.h>
#ifndef HWCAP_SHA512
#define HWCAP_SHA512 (1UL << 21)
#endif
#endif
#elif defined(HAVE_X86_SHA512)
#include <cpuid.h>
#include <immintrin.h>
#ifndef bit_SHA512
#define bit_SHA512 (1 << 0)
#endif
#endif
#if defined(__aarch64__) || defined(HAVE_X86_SHA512)
static const uint64_t K[80] = {
0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL,
0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL,
0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL,
0xd807aa98a3030242ULL, 0x12835b0145706fbeULL,
0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL,
0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL,
0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL,
0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL,
0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL,
0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL,
0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
0x06ca6351e003826fULL, 0x142929670a0e6e70ULL,
0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL,
0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL,
0x81c2c92e47edaee6ULL, 0x92722c851482353bULL,
0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL,
0xd192e819d6ef5218ULL, 0xd69906245565a910ULL,
0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL,
0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL,
0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL,
0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL,
0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL,
0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL,
0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL,
0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL,
0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
0x28db77f523047d84ULL, 0x32caab7b40c72493ULL,
0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL,
0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL
};
#endif
#if defined(__aarch64__)
/*
* Process one 128-byte block using ARM SHA512 crypto instructions.
*
* state: pointer to 8 uint64_t words (a,b,c,d,e,f,g,h)
* block: pointer to 16 uint64_t words (already native endian)
*
* The state is updated in place.
*
* Compiled for the SHA512 extension only; callers must first check
* 'sha512_native_available'.
*/
__attribute__((target("+sha3")))
void sha512_block_native(uint64_t *state, const uint64_t *block) {
/* Load current hash state */
uint64x2_t ab = vld1q_u64(&state[0]);
uint64x2_t cd = vld1q_u64(&state[2]);
uint64x2_t ef = vld1q_u64(&state[4]);
uint64x2_t gh = vld1q_u64(&state[6]);
/* Save original for final addition */
uint64x2_t ab_orig = ab;
uint64x2_t cd_orig = cd;
uint64x2_t ef_orig = ef;
uint64x2_t gh_orig = gh;
/* Load message (already native endian) */
uint64x2_t m0 = vld1q_u64(&block[0]);
uint64x2_t m1 = vld1q_u64(&block[2]);
uint64x2_t m2 = vld1q_u64(&block[4]);
uint64x2_t m3 = vld1q_u64(&block[6]);
uint64x2_t m4 = vld1q_u64(&block[8]);
uint64x2_t m5 = vld1q_u64(&block[10]);
uint64x2_t m6 = vld1q_u64(&block[12]);
uint64x2_t m7 = vld1q_u64(&block[14]);
uint64x2_t tmp;
/* Rounds 0-1 */
tmp = vaddq_u64(m0, vld1q_u64(&K[0]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1));
/* Rounds 2-3 */
tmp = vaddq_u64(m1, vld1q_u64(&K[2]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1));
/* Rounds 4-5 */
tmp = vaddq_u64(m2, vld1q_u64(&K[4]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1));
/* Rounds 6-7 */
tmp = vaddq_u64(m3, vld1q_u64(&K[6]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1));
/* Rounds 8-9 */
tmp = vaddq_u64(m4, vld1q_u64(&K[8]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1));
/* Rounds 10-11 */
tmp = vaddq_u64(m5, vld1q_u64(&K[10]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1));
/* Rounds 12-13 */
tmp = vaddq_u64(m6, vld1q_u64(&K[12]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1));
/* Rounds 14-15 */
tmp = vaddq_u64(m7, vld1q_u64(&K[14]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1));
/* Rounds 16-17 */
tmp = vaddq_u64(m0, vld1q_u64(&K[16]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1));
/* Rounds 18-19 */
tmp = vaddq_u64(m1, vld1q_u64(&K[18]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1));
/* Rounds 20-21 */
tmp = vaddq_u64(m2, vld1q_u64(&K[20]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1));
/* Rounds 22-23 */
tmp = vaddq_u64(m3, vld1q_u64(&K[22]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1));
/* Rounds 24-25 */
tmp = vaddq_u64(m4, vld1q_u64(&K[24]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1));
/* Rounds 26-27 */
tmp = vaddq_u64(m5, vld1q_u64(&K[26]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1));
/* Rounds 28-29 */
tmp = vaddq_u64(m6, vld1q_u64(&K[28]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1));
/* Rounds 30-31 */
tmp = vaddq_u64(m7, vld1q_u64(&K[30]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1));
/* Rounds 32-33 */
tmp = vaddq_u64(m0, vld1q_u64(&K[32]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1));
/* Rounds 34-35 */
tmp = vaddq_u64(m1, vld1q_u64(&K[34]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1));
/* Rounds 36-37 */
tmp = vaddq_u64(m2, vld1q_u64(&K[36]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1));
/* Rounds 38-39 */
tmp = vaddq_u64(m3, vld1q_u64(&K[38]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1));
/* Rounds 40-41 */
tmp = vaddq_u64(m4, vld1q_u64(&K[40]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1));
/* Rounds 42-43 */
tmp = vaddq_u64(m5, vld1q_u64(&K[42]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1));
/* Rounds 44-45 */
tmp = vaddq_u64(m6, vld1q_u64(&K[44]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1));
/* Rounds 46-47 */
tmp = vaddq_u64(m7, vld1q_u64(&K[46]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1));
/* Rounds 48-49 */
tmp = vaddq_u64(m0, vld1q_u64(&K[48]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1));
/* Rounds 50-51 */
tmp = vaddq_u64(m1, vld1q_u64(&K[50]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1));
/* Rounds 52-53 */
tmp = vaddq_u64(m2, vld1q_u64(&K[52]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1));
/* Rounds 54-55 */
tmp = vaddq_u64(m3, vld1q_u64(&K[54]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1));
/* Rounds 56-57 */
tmp = vaddq_u64(m4, vld1q_u64(&K[56]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1));
/* Rounds 58-59 */
tmp = vaddq_u64(m5, vld1q_u64(&K[58]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1));
/* Rounds 60-61 */
tmp = vaddq_u64(m6, vld1q_u64(&K[60]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1));
/* Rounds 62-63 */
tmp = vaddq_u64(m7, vld1q_u64(&K[62]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1));
/* Rounds 64-65 */
tmp = vaddq_u64(m0, vld1q_u64(&K[64]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
/* Rounds 66-67 */
tmp = vaddq_u64(m1, vld1q_u64(&K[66]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
/* Rounds 68-69 */
tmp = vaddq_u64(m2, vld1q_u64(&K[68]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
/* Rounds 70-71 */
tmp = vaddq_u64(m3, vld1q_u64(&K[70]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
/* Rounds 72-73 */
tmp = vaddq_u64(m4, vld1q_u64(&K[72]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(gh, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
gh = vsha512h2q_u64(tmp, cd, ab);
cd = vaddq_u64(cd, tmp);
/* Rounds 74-75 */
tmp = vaddq_u64(m5, vld1q_u64(&K[74]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ef, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
ef = vsha512h2q_u64(tmp, ab, gh);
ab = vaddq_u64(ab, tmp);
/* Rounds 76-77 */
tmp = vaddq_u64(m6, vld1q_u64(&K[76]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(cd, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
cd = vsha512h2q_u64(tmp, gh, ef);
gh = vaddq_u64(gh, tmp);
/* Rounds 78-79 */
tmp = vaddq_u64(m7, vld1q_u64(&K[78]));
tmp = vextq_u64(tmp, tmp, 1);
tmp = vaddq_u64(ab, tmp);
tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
ab = vsha512h2q_u64(tmp, ef, cd);
ef = vaddq_u64(ef, tmp);
/* Add original state back */
ab = vaddq_u64(ab, ab_orig);
cd = vaddq_u64(cd, cd_orig);
ef = vaddq_u64(ef, ef_orig);
gh = vaddq_u64(gh, gh_orig);
/* Store result */
vst1q_u64(&state[0], ab);
vst1q_u64(&state[2], cd);
vst1q_u64(&state[4], ef);
vst1q_u64(&state[6], gh);
}
#if defined(__APPLE__)
static int sysctl_flag(const char *name) {
int val = 0;
size_t len = sizeof(val);
if (sysctlbyname(name, &val, &len, NULL, 0) != 0) return 0;
return val != 0;
}
#endif
/*
* Return 1 if the CPU implements the ARM SHA512 extension, 0
* otherwise. We only know how to ask Darwin and Linux, and report 0
* (selecting the pure Haskell fallback) on any other OS.
*/
int sha512_native_available(void) {
#if defined(__APPLE__)
return sysctl_flag("hw.optional.arm.FEAT_SHA512")
|| sysctl_flag("hw.optional.armv8_2_sha512");
#elif defined(__linux__)
return (getauxval(AT_HWCAP) & HWCAP_SHA512) != 0;
#else
return 0;
#endif
}
#elif defined(HAVE_X86_SHA512)
/*
* Four rounds. 'sha512rnds2' does two: it takes the (c,d,g,h) and
* (a,b,e,f) halves of the state, and two message-plus-constant words
* in its third argument, and returns the new (a,b,e,f). The old
* (a,b,e,f) is then the new (c,d,g,h), so a second call with the
* halves swapped, on the upper two words, does the next two.
*/
#define ROUNDS4(abef, cdgh, msg) \
do { \
cdgh = _mm256_sha512rnds2_epi64(cdgh, abef, \
_mm256_castsi256_si128(msg)); \
abef = _mm256_sha512rnds2_epi64(abef, cdgh, \
_mm256_extracti128_si256(msg, 1)); \
} while (0)
/*
* The four words following prev[0], i.e. (prev[1], prev[2], prev[3],
* cur[0]) from lowest lane to highest.
*/
#define ALIGNR1(cur, prev) \
_mm256_permute4x64_epi64(_mm256_blend_epi32(prev, cur, 0x03), 0x39)
/*
* Process one 128-byte block using x86 SHA512 instructions.
*
* state: pointer to 8 uint64_t words (a,b,c,d,e,f,g,h)
* block: pointer to 16 uint64_t words (already native endian)
*
* The state is updated in place.
*
* Compiled for the SHA512 and AVX2 extensions only; callers must first
* check 'sha512_native_available'.
*
* Lane orders in comments are written from highest lane to lowest.
*/
__attribute__((target("avx2,sha512")))
void sha512_block_native(uint64_t *state, const uint64_t *block) {
__m256i abcd, efgh, abef, cdgh, abef_orig, cdgh_orig;
__m256i m0, m1, m2, m3, msg;
/* Load current hash state, rearranging it for the SHA instructions */
abcd = _mm256_loadu_si256((const __m256i *)&state[0]); /* d c b a */
efgh = _mm256_loadu_si256((const __m256i *)&state[4]); /* h g f e */
abef = _mm256_permute4x64_epi64(
_mm256_permute2x128_si256(abcd, efgh, 0x20), 0x1b);
/* a b e f */
cdgh = _mm256_permute4x64_epi64(
_mm256_permute2x128_si256(abcd, efgh, 0x31), 0x1b);
/* c d g h */
/* Save original for final addition */
abef_orig = abef;
cdgh_orig = cdgh;
/* Load message (already native endian) */
m0 = _mm256_loadu_si256((const __m256i *)&block[0]);
m1 = _mm256_loadu_si256((const __m256i *)&block[4]);
m2 = _mm256_loadu_si256((const __m256i *)&block[8]);
m3 = _mm256_loadu_si256((const __m256i *)&block[12]);
/* Rounds 0-3 */
msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[0]));
ROUNDS4(abef, cdgh, msg);
/* Rounds 4-7 */
msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[4]));
ROUNDS4(abef, cdgh, msg);
m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
/* Rounds 8-11 */
msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[8]));
ROUNDS4(abef, cdgh, msg);
m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
/* Rounds 12-15 */
msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[12]));
ROUNDS4(abef, cdgh, msg);
m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
m0 = _mm256_sha512msg2_epi64(m0, m3);
m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
/* Rounds 16-19 */
msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[16]));
ROUNDS4(abef, cdgh, msg);
m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
m1 = _mm256_sha512msg2_epi64(m1, m0);
m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
/* Rounds 20-23 */
msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[20]));
ROUNDS4(abef, cdgh, msg);
m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
m2 = _mm256_sha512msg2_epi64(m2, m1);
m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
/* Rounds 24-27 */
msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[24]));
ROUNDS4(abef, cdgh, msg);
m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
m3 = _mm256_sha512msg2_epi64(m3, m2);
m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
/* Rounds 28-31 */
msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[28]));
ROUNDS4(abef, cdgh, msg);
m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
m0 = _mm256_sha512msg2_epi64(m0, m3);
m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
/* Rounds 32-35 */
msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[32]));
ROUNDS4(abef, cdgh, msg);
m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
m1 = _mm256_sha512msg2_epi64(m1, m0);
m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
/* Rounds 36-39 */
msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[36]));
ROUNDS4(abef, cdgh, msg);
m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
m2 = _mm256_sha512msg2_epi64(m2, m1);
m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
/* Rounds 40-43 */
msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[40]));
ROUNDS4(abef, cdgh, msg);
m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
m3 = _mm256_sha512msg2_epi64(m3, m2);
m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
/* Rounds 44-47 */
msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[44]));
ROUNDS4(abef, cdgh, msg);
m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
m0 = _mm256_sha512msg2_epi64(m0, m3);
m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
/* Rounds 48-51 */
msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[48]));
ROUNDS4(abef, cdgh, msg);
m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
m1 = _mm256_sha512msg2_epi64(m1, m0);
m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
/* Rounds 52-55 */
msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[52]));
ROUNDS4(abef, cdgh, msg);
m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
m2 = _mm256_sha512msg2_epi64(m2, m1);
m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
/* Rounds 56-59 */
msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[56]));
ROUNDS4(abef, cdgh, msg);
m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
m3 = _mm256_sha512msg2_epi64(m3, m2);
m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
/* Rounds 60-63 */
msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[60]));
ROUNDS4(abef, cdgh, msg);
m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
m0 = _mm256_sha512msg2_epi64(m0, m3);
m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
/* Rounds 64-67 */
msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[64]));
ROUNDS4(abef, cdgh, msg);
m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
m1 = _mm256_sha512msg2_epi64(m1, m0);
m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
/* Rounds 68-71 */
msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[68]));
ROUNDS4(abef, cdgh, msg);
m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
m2 = _mm256_sha512msg2_epi64(m2, m1);
/* Rounds 72-75 */
msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[72]));
ROUNDS4(abef, cdgh, msg);
m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
m3 = _mm256_sha512msg2_epi64(m3, m2);
/* Rounds 76-79 */
msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[76]));
ROUNDS4(abef, cdgh, msg);
/* Add original state back */
abef = _mm256_add_epi64(abef, abef_orig);
cdgh = _mm256_add_epi64(cdgh, cdgh_orig);
/* Restore the state's original order and store result */
abef = _mm256_permute4x64_epi64(abef, 0x1b); /* f e b a */
cdgh = _mm256_permute4x64_epi64(cdgh, 0x1b); /* h g d c */
_mm256_storeu_si256((__m256i *)&state[0],
_mm256_permute2x128_si256(abef, cdgh, 0x20)); /* d c b a */
_mm256_storeu_si256((__m256i *)&state[4],
_mm256_permute2x128_si256(abef, cdgh, 0x31)); /* h g f e */
}
/*
* Return 1 if the CPU implements the x86 SHA512 extension and the AVX2
* instructions used alongside it, and the OS saves the YMM registers,
* 0 otherwise.
*/
int sha512_native_available(void) {
unsigned int eax, ebx, ecx, edx, xcr0;
if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx))
return 0;
if (!(ecx & bit_OSXSAVE) || !(ecx & bit_AVX))
return 0;
/* XCR0 bits 1 and 2: the OS saves the XMM and YMM registers */
__asm__ ("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
if ((xcr0 & 0x6) != 0x6)
return 0;
if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx))
return 0;
if (eax < 1 || !(ebx & bit_AVX2))
return 0;
__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx);
return (eax & bit_SHA512) != 0;
}
#else
/* Stub implementations for builds without hardware support */
void sha512_block_native(uint64_t *state, const uint64_t *block) {
(void)state;
(void)block;
/* Should never be called - use pure Haskell fallback */
}
int sha512_native_available(void) {
return 0;
}
#endif