crypton-2.2.4: cbits/crypton_powm_ifma.h

/* Generated by cbits/crypton_powm_ifma.py; do not edit.
 *
 * Almost-Montgomery multiplication in radix 2^52, two at once: r = a b
 * 2^-52D mod m, under 2m for a and b under 2m.  Digits are 52 bits in 64-bit
 * words, each number padded with zero words to a whole number of 256-bit
 * registers.  m is read from memory; at 20 digits a is held in registers,
 * above that it is read from memory too. */
#ifndef CRYPTON_POWM_IFMA_H
#define CRYPTON_POWM_IFMA_H

#include <stdint.h>
#include <immintrin.h>

#define IFMA_MASK52 ((UINT64_C(1) << 52) - 1)
#define IFMA_TARGET __attribute__((target("avx2,avx512f,avx512vl,avx512ifma")))

/* 20 lanes to 52-bit digits: what is above 52 bits moves up a lane,
 * and the one-bit carries that can then remain are resolved together */
IFMA_TARGET static inline void ifma_norm5(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4)
{
	const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1);
	const __m256i zero = _mm256_setzero_si256();
	uint64_t g = 0, p = 0, c;
	__m256i C0 = _mm256_srli_epi64(X0, 52);
	__m256i C1 = _mm256_srli_epi64(X1, 52);
	__m256i C2 = _mm256_srli_epi64(X2, 52);
	__m256i C3 = _mm256_srli_epi64(X3, 52);
	__m256i C4 = _mm256_srli_epi64(X4, 52);
	X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3));
	X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3));
	X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3));
	X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3));
	X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3));
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16;
	c = ((g << 1) + p) ^ p;
	X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 0), X0);
	X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 4), X1);
	X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 8), X2);
	X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 12), X3);
	X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 16), X4);
}

/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping
 * one; entries are 20 words apart */
IFMA_TARGET static void ifma_sel5(uint64_t *o1, const uint64_t *t1, uint64_t k1,
                                uint64_t *o2, const uint64_t *t2, uint64_t k2)
{
	const __m256i one = _mm256_set1_epi64x(1);
	__m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2);
	__m256i kv = _mm256_setzero_si256();
	__m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv;
	int k;
	for (k = 0; k < 32; k++) {
		__mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv);
		const uint64_t *e1 = t1 + 20 * k, *e2 = t2 + 20 * k;
		P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0);
		P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4);
		P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8);
		P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12);
		P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16);
		kv = _mm256_add_epi64(kv, one);
	}
	_mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0);
	_mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1);
	_mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2);
	_mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3);
	_mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4);
}

/* 32 lanes to 52-bit digits: what is above 52 bits moves up a lane,
 * and the one-bit carries that can then remain are resolved together */
IFMA_TARGET static inline void ifma_norm8(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4, __m256i X5, __m256i X6, __m256i X7)
{
	const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1);
	const __m256i zero = _mm256_setzero_si256();
	uint64_t g = 0, p = 0, c;
	__m256i C0 = _mm256_srli_epi64(X0, 52);
	__m256i C1 = _mm256_srli_epi64(X1, 52);
	__m256i C2 = _mm256_srli_epi64(X2, 52);
	__m256i C3 = _mm256_srli_epi64(X3, 52);
	__m256i C4 = _mm256_srli_epi64(X4, 52);
	__m256i C5 = _mm256_srli_epi64(X5, 52);
	__m256i C6 = _mm256_srli_epi64(X6, 52);
	__m256i C7 = _mm256_srli_epi64(X7, 52);
	X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3));
	X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3));
	X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3));
	X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3));
	X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3));
	X5 = _mm256_add_epi64(_mm256_and_si256(X5, mk), _mm256_alignr_epi64(C5, C4, 3));
	X6 = _mm256_add_epi64(_mm256_and_si256(X6, mk), _mm256_alignr_epi64(C6, C5, 3));
	X7 = _mm256_add_epi64(_mm256_and_si256(X7, mk), _mm256_alignr_epi64(C7, C6, 3));
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X5, mk) << 20;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X5, mk) << 20;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X6, mk) << 24;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X6, mk) << 24;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X7, mk) << 28;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X7, mk) << 28;
	c = ((g << 1) + p) ^ p;
	X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 0), X0);
	X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 4), X1);
	X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 8), X2);
	X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 12), X3);
	X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 16), X4);
	X5 = _mm256_and_si256(_mm256_mask_add_epi64(X5, (__mmask8)(c >> 20 & 15), X5, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 20), X5);
	X6 = _mm256_and_si256(_mm256_mask_add_epi64(X6, (__mmask8)(c >> 24 & 15), X6, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 24), X6);
	X7 = _mm256_and_si256(_mm256_mask_add_epi64(X7, (__mmask8)(c >> 28 & 15), X7, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 28), X7);
}

/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping
 * one; entries are 32 words apart */
IFMA_TARGET static void ifma_sel8(uint64_t *o1, const uint64_t *t1, uint64_t k1,
                                uint64_t *o2, const uint64_t *t2, uint64_t k2)
{
	const __m256i one = _mm256_set1_epi64x(1);
	__m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2);
	__m256i kv = _mm256_setzero_si256();
	__m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv, P5 = kv, Q5 = kv, P6 = kv, Q6 = kv, P7 = kv, Q7 = kv;
	int k;
	for (k = 0; k < 32; k++) {
		__mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv);
		const uint64_t *e1 = t1 + 32 * k, *e2 = t2 + 32 * k;
		P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0);
		P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4);
		P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8);
		P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12);
		P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16);
		P5 = _mm256_mask_loadu_epi64(P5, s1, e1 + 20); Q5 = _mm256_mask_loadu_epi64(Q5, s2, e2 + 20);
		P6 = _mm256_mask_loadu_epi64(P6, s1, e1 + 24); Q6 = _mm256_mask_loadu_epi64(Q6, s2, e2 + 24);
		P7 = _mm256_mask_loadu_epi64(P7, s1, e1 + 28); Q7 = _mm256_mask_loadu_epi64(Q7, s2, e2 + 28);
		kv = _mm256_add_epi64(kv, one);
	}
	_mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0);
	_mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1);
	_mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2);
	_mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3);
	_mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4);
	_mm256_storeu_si256((__m256i *)(o1 + 20), P5); _mm256_storeu_si256((__m256i *)(o2 + 20), Q5);
	_mm256_storeu_si256((__m256i *)(o1 + 24), P6); _mm256_storeu_si256((__m256i *)(o2 + 24), Q6);
	_mm256_storeu_si256((__m256i *)(o1 + 28), P7); _mm256_storeu_si256((__m256i *)(o2 + 28), Q7);
}

/* 40 lanes to 52-bit digits: what is above 52 bits moves up a lane,
 * and the one-bit carries that can then remain are resolved together */
IFMA_TARGET static inline void ifma_norm10(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4, __m256i X5, __m256i X6, __m256i X7, __m256i X8, __m256i X9)
{
	const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1);
	const __m256i zero = _mm256_setzero_si256();
	uint64_t g = 0, p = 0, c;
	__m256i C0 = _mm256_srli_epi64(X0, 52);
	__m256i C1 = _mm256_srli_epi64(X1, 52);
	__m256i C2 = _mm256_srli_epi64(X2, 52);
	__m256i C3 = _mm256_srli_epi64(X3, 52);
	__m256i C4 = _mm256_srli_epi64(X4, 52);
	__m256i C5 = _mm256_srli_epi64(X5, 52);
	__m256i C6 = _mm256_srli_epi64(X6, 52);
	__m256i C7 = _mm256_srli_epi64(X7, 52);
	__m256i C8 = _mm256_srli_epi64(X8, 52);
	__m256i C9 = _mm256_srli_epi64(X9, 52);
	X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3));
	X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3));
	X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3));
	X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3));
	X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3));
	X5 = _mm256_add_epi64(_mm256_and_si256(X5, mk), _mm256_alignr_epi64(C5, C4, 3));
	X6 = _mm256_add_epi64(_mm256_and_si256(X6, mk), _mm256_alignr_epi64(C6, C5, 3));
	X7 = _mm256_add_epi64(_mm256_and_si256(X7, mk), _mm256_alignr_epi64(C7, C6, 3));
	X8 = _mm256_add_epi64(_mm256_and_si256(X8, mk), _mm256_alignr_epi64(C8, C7, 3));
	X9 = _mm256_add_epi64(_mm256_and_si256(X9, mk), _mm256_alignr_epi64(C9, C8, 3));
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X5, mk) << 20;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X5, mk) << 20;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X6, mk) << 24;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X6, mk) << 24;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X7, mk) << 28;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X7, mk) << 28;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X8, mk) << 32;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X8, mk) << 32;
	g |= (uint64_t) _mm256_cmpgt_epu64_mask(X9, mk) << 36;
	p |= (uint64_t) _mm256_cmpeq_epu64_mask(X9, mk) << 36;
	c = ((g << 1) + p) ^ p;
	X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 0), X0);
	X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 4), X1);
	X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 8), X2);
	X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 12), X3);
	X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 16), X4);
	X5 = _mm256_and_si256(_mm256_mask_add_epi64(X5, (__mmask8)(c >> 20 & 15), X5, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 20), X5);
	X6 = _mm256_and_si256(_mm256_mask_add_epi64(X6, (__mmask8)(c >> 24 & 15), X6, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 24), X6);
	X7 = _mm256_and_si256(_mm256_mask_add_epi64(X7, (__mmask8)(c >> 28 & 15), X7, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 28), X7);
	X8 = _mm256_and_si256(_mm256_mask_add_epi64(X8, (__mmask8)(c >> 32 & 15), X8, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 32), X8);
	X9 = _mm256_and_si256(_mm256_mask_add_epi64(X9, (__mmask8)(c >> 36 & 15), X9, one), mk);
	_mm256_storeu_si256((__m256i *)(r + 36), X9);
}

/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping
 * one; entries are 40 words apart */
IFMA_TARGET static void ifma_sel10(uint64_t *o1, const uint64_t *t1, uint64_t k1,
                                uint64_t *o2, const uint64_t *t2, uint64_t k2)
{
	const __m256i one = _mm256_set1_epi64x(1);
	__m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2);
	__m256i kv = _mm256_setzero_si256();
	__m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv, P5 = kv, Q5 = kv, P6 = kv, Q6 = kv, P7 = kv, Q7 = kv, P8 = kv, Q8 = kv, P9 = kv, Q9 = kv;
	int k;
	for (k = 0; k < 32; k++) {
		__mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv);
		const uint64_t *e1 = t1 + 40 * k, *e2 = t2 + 40 * k;
		P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0);
		P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4);
		P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8);
		P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12);
		P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16);
		P5 = _mm256_mask_loadu_epi64(P5, s1, e1 + 20); Q5 = _mm256_mask_loadu_epi64(Q5, s2, e2 + 20);
		P6 = _mm256_mask_loadu_epi64(P6, s1, e1 + 24); Q6 = _mm256_mask_loadu_epi64(Q6, s2, e2 + 24);
		P7 = _mm256_mask_loadu_epi64(P7, s1, e1 + 28); Q7 = _mm256_mask_loadu_epi64(Q7, s2, e2 + 28);
		P8 = _mm256_mask_loadu_epi64(P8, s1, e1 + 32); Q8 = _mm256_mask_loadu_epi64(Q8, s2, e2 + 32);
		P9 = _mm256_mask_loadu_epi64(P9, s1, e1 + 36); Q9 = _mm256_mask_loadu_epi64(Q9, s2, e2 + 36);
		kv = _mm256_add_epi64(kv, one);
	}
	_mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0);
	_mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1);
	_mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2);
	_mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3);
	_mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4);
	_mm256_storeu_si256((__m256i *)(o1 + 20), P5); _mm256_storeu_si256((__m256i *)(o2 + 20), Q5);
	_mm256_storeu_si256((__m256i *)(o1 + 24), P6); _mm256_storeu_si256((__m256i *)(o2 + 24), Q6);
	_mm256_storeu_si256((__m256i *)(o1 + 28), P7); _mm256_storeu_si256((__m256i *)(o2 + 28), Q7);
	_mm256_storeu_si256((__m256i *)(o1 + 32), P8); _mm256_storeu_si256((__m256i *)(o2 + 32), Q8);
	_mm256_storeu_si256((__m256i *)(o1 + 36), P9); _mm256_storeu_si256((__m256i *)(o2 + 36), Q9);
}

/* 20 digits, 5 registers a number */
IFMA_TARGET static void ifma_amm20_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1,
                                    const uint64_t *m1, uint64_t k1,
                                    uint64_t *r2, const uint64_t *a2, const uint64_t *b2,
                                    const uint64_t *m2, uint64_t k2)
{
	const __m256i zero = _mm256_setzero_si256();
#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k)))
	__m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero;
	__m256i A0 = L(a1, 0), A1 = L(a1, 1), A2 = L(a1, 2), A3 = L(a1, 3), A4 = L(a1, 4);
	__m256i B0 = L(a2, 0), B1 = L(a2, 1), B2 = L(a2, 2), B3 = L(a2, 3), B4 = L(a2, 4);
	uint64_t x1 = 0, x2 = 0;
	int i;

	for (i = 0; i < 20; i++) {
		uint64_t bi = b1[i], ci = b2[i], y, z;
		/* lane 0 in full: the products' high halves, which are lane 1's, too */
		unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1;
		unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2;
		__m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci);
		__m256i yv, zv;

		y = ((uint64_t) s1 * k1) & IFMA_MASK52;
		z = ((uint64_t) s2 * k2) & IFMA_MASK52;
		s1 += (unsigned __int128) m1[0] * y;
		s2 += (unsigned __int128) m2[0] * z;
		yv = _mm256_set1_epi64x((long long) y);
		zv = _mm256_set1_epi64x((long long) z);
		X0 = _mm256_madd52lo_epu64(X0, A0, bv); Y0 = _mm256_madd52lo_epu64(Y0, B0, cv);
		X1 = _mm256_madd52lo_epu64(X1, A1, bv); Y1 = _mm256_madd52lo_epu64(Y1, B1, cv);
		X2 = _mm256_madd52lo_epu64(X2, A2, bv); Y2 = _mm256_madd52lo_epu64(Y2, B2, cv);
		X3 = _mm256_madd52lo_epu64(X3, A3, bv); Y3 = _mm256_madd52lo_epu64(Y3, B3, cv);
		X4 = _mm256_madd52lo_epu64(X4, A4, bv); Y4 = _mm256_madd52lo_epu64(Y4, B4, cv);
		X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv);
		/* down a lane: lane 0 is a multiple of 2^52, and s carries it */
		X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1);
		X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1);
		X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1);
		X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1);
		X4 = _mm256_alignr_epi64(zero, X4, 1); Y4 = _mm256_alignr_epi64(zero, Y4, 1);
		x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52);
		x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52);
		X0 = _mm256_madd52hi_epu64(X0, A0, bv); Y0 = _mm256_madd52hi_epu64(Y0, B0, cv);
		X1 = _mm256_madd52hi_epu64(X1, A1, bv); Y1 = _mm256_madd52hi_epu64(Y1, B1, cv);
		X2 = _mm256_madd52hi_epu64(X2, A2, bv); Y2 = _mm256_madd52hi_epu64(Y2, B2, cv);
		X3 = _mm256_madd52hi_epu64(X3, A3, bv); Y3 = _mm256_madd52hi_epu64(Y3, B3, cv);
		X4 = _mm256_madd52hi_epu64(X4, A4, bv); Y4 = _mm256_madd52hi_epu64(Y4, B4, cv);
		X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv);
	}
#undef L
	/* lane 0, with the carry it was never given, is what the scalar holds */
	X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1);
	Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2);
	ifma_norm5(r1, X0, X1, X2, X3, X4);
	ifma_norm5(r2, Y0, Y1, Y2, Y3, Y4);
}

/* 30 digits, 8 registers a number */
IFMA_TARGET static void ifma_amm30_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1,
                                    const uint64_t *m1, uint64_t k1,
                                    uint64_t *r2, const uint64_t *a2, const uint64_t *b2,
                                    const uint64_t *m2, uint64_t k2)
{
	const __m256i zero = _mm256_setzero_si256();
#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k)))
	__m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, X5 = zero, X6 = zero, X7 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero, Y5 = zero, Y6 = zero, Y7 = zero;
	uint64_t x1 = 0, x2 = 0;
	int i;

	for (i = 0; i < 30; i++) {
		uint64_t bi = b1[i], ci = b2[i], y, z;
		/* lane 0 in full: the products' high halves, which are lane 1's, too */
		unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1;
		unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2;
		__m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci);
		__m256i yv, zv;

		y = ((uint64_t) s1 * k1) & IFMA_MASK52;
		z = ((uint64_t) s2 * k2) & IFMA_MASK52;
		s1 += (unsigned __int128) m1[0] * y;
		s2 += (unsigned __int128) m2[0] * z;
		yv = _mm256_set1_epi64x((long long) y);
		zv = _mm256_set1_epi64x((long long) z);
		X0 = _mm256_madd52lo_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52lo_epu64(Y0, L(a2, 0), cv);
		X1 = _mm256_madd52lo_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52lo_epu64(Y1, L(a2, 1), cv);
		X2 = _mm256_madd52lo_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52lo_epu64(Y2, L(a2, 2), cv);
		X3 = _mm256_madd52lo_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52lo_epu64(Y3, L(a2, 3), cv);
		X4 = _mm256_madd52lo_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52lo_epu64(Y4, L(a2, 4), cv);
		X5 = _mm256_madd52lo_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52lo_epu64(Y5, L(a2, 5), cv);
		X6 = _mm256_madd52lo_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52lo_epu64(Y6, L(a2, 6), cv);
		X7 = _mm256_madd52lo_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52lo_epu64(Y7, L(a2, 7), cv);
		X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv);
		X5 = _mm256_madd52lo_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52lo_epu64(Y5, L(m2, 5), zv);
		X6 = _mm256_madd52lo_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52lo_epu64(Y6, L(m2, 6), zv);
		X7 = _mm256_madd52lo_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52lo_epu64(Y7, L(m2, 7), zv);
		/* down a lane: lane 0 is a multiple of 2^52, and s carries it */
		X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1);
		X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1);
		X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1);
		X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1);
		X4 = _mm256_alignr_epi64(X5, X4, 1); Y4 = _mm256_alignr_epi64(Y5, Y4, 1);
		X5 = _mm256_alignr_epi64(X6, X5, 1); Y5 = _mm256_alignr_epi64(Y6, Y5, 1);
		X6 = _mm256_alignr_epi64(X7, X6, 1); Y6 = _mm256_alignr_epi64(Y7, Y6, 1);
		X7 = _mm256_alignr_epi64(zero, X7, 1); Y7 = _mm256_alignr_epi64(zero, Y7, 1);
		x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52);
		x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52);
		X0 = _mm256_madd52hi_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52hi_epu64(Y0, L(a2, 0), cv);
		X1 = _mm256_madd52hi_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52hi_epu64(Y1, L(a2, 1), cv);
		X2 = _mm256_madd52hi_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52hi_epu64(Y2, L(a2, 2), cv);
		X3 = _mm256_madd52hi_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52hi_epu64(Y3, L(a2, 3), cv);
		X4 = _mm256_madd52hi_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52hi_epu64(Y4, L(a2, 4), cv);
		X5 = _mm256_madd52hi_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52hi_epu64(Y5, L(a2, 5), cv);
		X6 = _mm256_madd52hi_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52hi_epu64(Y6, L(a2, 6), cv);
		X7 = _mm256_madd52hi_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52hi_epu64(Y7, L(a2, 7), cv);
		X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv);
		X5 = _mm256_madd52hi_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52hi_epu64(Y5, L(m2, 5), zv);
		X6 = _mm256_madd52hi_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52hi_epu64(Y6, L(m2, 6), zv);
		X7 = _mm256_madd52hi_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52hi_epu64(Y7, L(m2, 7), zv);
	}
#undef L
	/* lane 0, with the carry it was never given, is what the scalar holds */
	X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1);
	Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2);
	ifma_norm8(r1, X0, X1, X2, X3, X4, X5, X6, X7);
	ifma_norm8(r2, Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7);
}

/* 40 digits, 10 registers a number */
IFMA_TARGET static void ifma_amm40_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1,
                                    const uint64_t *m1, uint64_t k1,
                                    uint64_t *r2, const uint64_t *a2, const uint64_t *b2,
                                    const uint64_t *m2, uint64_t k2)
{
	const __m256i zero = _mm256_setzero_si256();
#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k)))
	__m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, X5 = zero, X6 = zero, X7 = zero, X8 = zero, X9 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero, Y5 = zero, Y6 = zero, Y7 = zero, Y8 = zero, Y9 = zero;
	uint64_t x1 = 0, x2 = 0;
	int i;

	for (i = 0; i < 40; i++) {
		uint64_t bi = b1[i], ci = b2[i], y, z;
		/* lane 0 in full: the products' high halves, which are lane 1's, too */
		unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1;
		unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2;
		__m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci);
		__m256i yv, zv;

		y = ((uint64_t) s1 * k1) & IFMA_MASK52;
		z = ((uint64_t) s2 * k2) & IFMA_MASK52;
		s1 += (unsigned __int128) m1[0] * y;
		s2 += (unsigned __int128) m2[0] * z;
		yv = _mm256_set1_epi64x((long long) y);
		zv = _mm256_set1_epi64x((long long) z);
		X0 = _mm256_madd52lo_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52lo_epu64(Y0, L(a2, 0), cv);
		X1 = _mm256_madd52lo_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52lo_epu64(Y1, L(a2, 1), cv);
		X2 = _mm256_madd52lo_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52lo_epu64(Y2, L(a2, 2), cv);
		X3 = _mm256_madd52lo_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52lo_epu64(Y3, L(a2, 3), cv);
		X4 = _mm256_madd52lo_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52lo_epu64(Y4, L(a2, 4), cv);
		X5 = _mm256_madd52lo_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52lo_epu64(Y5, L(a2, 5), cv);
		X6 = _mm256_madd52lo_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52lo_epu64(Y6, L(a2, 6), cv);
		X7 = _mm256_madd52lo_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52lo_epu64(Y7, L(a2, 7), cv);
		X8 = _mm256_madd52lo_epu64(X8, L(a1, 8), bv); Y8 = _mm256_madd52lo_epu64(Y8, L(a2, 8), cv);
		X9 = _mm256_madd52lo_epu64(X9, L(a1, 9), bv); Y9 = _mm256_madd52lo_epu64(Y9, L(a2, 9), cv);
		X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv);
		X5 = _mm256_madd52lo_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52lo_epu64(Y5, L(m2, 5), zv);
		X6 = _mm256_madd52lo_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52lo_epu64(Y6, L(m2, 6), zv);
		X7 = _mm256_madd52lo_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52lo_epu64(Y7, L(m2, 7), zv);
		X8 = _mm256_madd52lo_epu64(X8, L(m1, 8), yv); Y8 = _mm256_madd52lo_epu64(Y8, L(m2, 8), zv);
		X9 = _mm256_madd52lo_epu64(X9, L(m1, 9), yv); Y9 = _mm256_madd52lo_epu64(Y9, L(m2, 9), zv);
		/* down a lane: lane 0 is a multiple of 2^52, and s carries it */
		X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1);
		X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1);
		X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1);
		X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1);
		X4 = _mm256_alignr_epi64(X5, X4, 1); Y4 = _mm256_alignr_epi64(Y5, Y4, 1);
		X5 = _mm256_alignr_epi64(X6, X5, 1); Y5 = _mm256_alignr_epi64(Y6, Y5, 1);
		X6 = _mm256_alignr_epi64(X7, X6, 1); Y6 = _mm256_alignr_epi64(Y7, Y6, 1);
		X7 = _mm256_alignr_epi64(X8, X7, 1); Y7 = _mm256_alignr_epi64(Y8, Y7, 1);
		X8 = _mm256_alignr_epi64(X9, X8, 1); Y8 = _mm256_alignr_epi64(Y9, Y8, 1);
		X9 = _mm256_alignr_epi64(zero, X9, 1); Y9 = _mm256_alignr_epi64(zero, Y9, 1);
		x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52);
		x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52);
		X0 = _mm256_madd52hi_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52hi_epu64(Y0, L(a2, 0), cv);
		X1 = _mm256_madd52hi_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52hi_epu64(Y1, L(a2, 1), cv);
		X2 = _mm256_madd52hi_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52hi_epu64(Y2, L(a2, 2), cv);
		X3 = _mm256_madd52hi_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52hi_epu64(Y3, L(a2, 3), cv);
		X4 = _mm256_madd52hi_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52hi_epu64(Y4, L(a2, 4), cv);
		X5 = _mm256_madd52hi_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52hi_epu64(Y5, L(a2, 5), cv);
		X6 = _mm256_madd52hi_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52hi_epu64(Y6, L(a2, 6), cv);
		X7 = _mm256_madd52hi_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52hi_epu64(Y7, L(a2, 7), cv);
		X8 = _mm256_madd52hi_epu64(X8, L(a1, 8), bv); Y8 = _mm256_madd52hi_epu64(Y8, L(a2, 8), cv);
		X9 = _mm256_madd52hi_epu64(X9, L(a1, 9), bv); Y9 = _mm256_madd52hi_epu64(Y9, L(a2, 9), cv);
		X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv);
		X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv);
		X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv);
		X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv);
		X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv);
		X5 = _mm256_madd52hi_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52hi_epu64(Y5, L(m2, 5), zv);
		X6 = _mm256_madd52hi_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52hi_epu64(Y6, L(m2, 6), zv);
		X7 = _mm256_madd52hi_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52hi_epu64(Y7, L(m2, 7), zv);
		X8 = _mm256_madd52hi_epu64(X8, L(m1, 8), yv); Y8 = _mm256_madd52hi_epu64(Y8, L(m2, 8), zv);
		X9 = _mm256_madd52hi_epu64(X9, L(m1, 9), yv); Y9 = _mm256_madd52hi_epu64(Y9, L(m2, 9), zv);
	}
#undef L
	/* lane 0, with the carry it was never given, is what the scalar holds */
	X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1);
	Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2);
	ifma_norm10(r1, X0, X1, X2, X3, X4, X5, X6, X7, X8, X9);
	ifma_norm10(r2, Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7, Y8, Y9);
}

#endif