crypton-2.0.0: cbits/crypton_chacha.c

/*
 * Copyright (c) 2014-2015 Vincent Hanquez <vincent@snarc.org>
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the author nor the names of his contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <stdint.h>
#include <string.h>
#include "crypton_chacha.h"
#include "crypton_bitfn.h"
#include "crypton_align.h"
#include <stdio.h>

/*
 * Four blocks at a time with whichever vector unit the target has: NEON in
 * chacha_neon.c, SSE2 in chacha_sse2.c.  Both present the same two entry
 * points, so there is one path here.
 *
 * The state words are held little-endian -- the core below reads them
 * without converting -- so the vector versions, which also do not convert,
 * are left out on a big-endian machine.
 */
#if (defined(WITH_ARMV8_NEON) && !defined(__AARCH64EB__)) || defined(WITH_X86_SSE2)
#define CHACHA_SIMD 1
int crypton_chacha_simd_width(void);
void crypton_chacha_simd_combine(int rounds, uint8_t *dst, const uint8_t *src,
                                 const crypton_chacha_state *in);
void crypton_chacha_simd_generate(int rounds, uint8_t *dst,
                                  const crypton_chacha_state *in);
/* The counters in a group must not carry into d[13], which the block loop
 * below handles and the vector one does not; that is one run in 2^29. */
#define CHACHA_SIMD_OK(st, n) ((st)->d[12] <= 0xffffffffU - (uint32_t) (n))
#endif

/*
 * ChaCha20 from CRYPTOGAMS, in cbits/asm/chacha-armv8-*.S.  It keeps four
 * vector blocks and a fifth in the general registers in flight at once, or
 * six and two above 512 bytes, which is more than the intrinsics above can
 * be made to do: the vector registers hold four states and there is no room
 * for another, so the extra parallelism has to come from the integer side,
 * and that means saying which register holds what.
 *
 * Twenty rounds and the 256-bit constants are built into it, and it takes
 * the counter as 32 bits wide, so it is given only the states it fits.
 */
#if (defined(WITH_ARMV8_CHACHA_ASM) && !defined(__AARCH64EB__)) \
    || defined(WITH_X86_CHACHA_ASM)
#define CHACHA_ASM 1
#include "crypton_cpu.h"
void crypton_chacha20_asm_ctr32(uint8_t *out, const uint8_t *in, size_t len,
                                const uint32_t key[8], const uint32_t counter[4]);

/* crypton_cpu.c defines the crypton_armcap_P that the assembly reads to
 * find out whether the processor has NEON. */

/* The four words at the head of the state are the constants that go with a
 * 256-bit key, and the assembly has only those. */
static int chacha_asm_state(const crypton_chacha_state *st)
{
	return st->d[0] == 0x61707865 && st->d[1] == 0x3320646e
	    && st->d[2] == 0x79622d32 && st->d[3] == 0x6b206574;
}

/*
 * How much is worth handing over.  On AArch64 the module's vector path
 * starts at three blocks and below that its scalar path measures level with
 * the C here, so there is nothing to gain; on x86-64 it is ahead from one
 * block, the C there having no vector path until eight.
 */
#ifndef CHACHA_ASM_MIN_BLOCKS
#ifdef WITH_X86_CHACHA_ASM
#define CHACHA_ASM_MIN_BLOCKS 1
#else
#define CHACHA_ASM_MIN_BLOCKS 3
#endif
#endif
#endif

#define QR(a,b,c,d) \
	a += b; d = rol32(d ^ a,16); \
	c += d; b = rol32(b ^ c,12); \
	a += b; d = rol32(d ^ a, 8); \
	c += d; b = rol32(b ^ c, 7);

#define ALIGNED64(PTR) \
	(((uintptr_t)(const void *)(PTR)) % 8 == 0)

static const uint8_t sigma[16] = "expand 32-byte k";
static const uint8_t tau[16] = "expand 16-byte k";

static void chacha_core(int rounds, block *out, const crypton_chacha_state *in)
{
	uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
	int i;

	x0 = in->d[0]; x1 = in->d[1]; x2 = in->d[2]; x3 = in->d[3];
	x4 = in->d[4]; x5 = in->d[5]; x6 = in->d[6]; x7 = in->d[7];
	x8 = in->d[8]; x9 = in->d[9]; x10 = in->d[10]; x11 = in->d[11];
	x12 = in->d[12]; x13 = in->d[13]; x14 = in->d[14]; x15 = in->d[15];

	for (i = rounds; i > 0; i -= 2) {
		QR(x0, x4, x8, x12);
		QR(x1, x5, x9, x13);
		QR(x2, x6, x10, x14);
		QR(x3, x7, x11, x15);

		QR(x0, x5, x10, x15);
		QR(x1, x6, x11, x12);
		QR(x2, x7, x8, x13);
		QR(x3, x4, x9, x14);
	}

	x0 += in->d[0]; x1 += in->d[1]; x2 += in->d[2]; x3 += in->d[3];
	x4 += in->d[4]; x5 += in->d[5]; x6 += in->d[6]; x7 += in->d[7];
	x8 += in->d[8]; x9 += in->d[9]; x10 += in->d[10]; x11 += in->d[11];
	x12 += in->d[12]; x13 += in->d[13]; x14 += in->d[14]; x15 += in->d[15];

	out->d[0] = cpu_to_le32(x0);
	out->d[1] = cpu_to_le32(x1);
	out->d[2] = cpu_to_le32(x2);
	out->d[3] = cpu_to_le32(x3);
	out->d[4] = cpu_to_le32(x4);
	out->d[5] = cpu_to_le32(x5);
	out->d[6] = cpu_to_le32(x6);
	out->d[7] = cpu_to_le32(x7);
	out->d[8] = cpu_to_le32(x8);
	out->d[9] = cpu_to_le32(x9);
	out->d[10] = cpu_to_le32(x10);
	out->d[11] = cpu_to_le32(x11);
	out->d[12] = cpu_to_le32(x12);
	out->d[13] = cpu_to_le32(x13);
	out->d[14] = cpu_to_le32(x14);
	out->d[15] = cpu_to_le32(x15);
}

static void hchacha_core(int rounds, uint32_t *out, const crypton_chacha_state *in)
{
	uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
	int i;

	x0 = in->d[0]; x1 = in->d[1]; x2 = in->d[2]; x3 = in->d[3];
	x4 = in->d[4]; x5 = in->d[5]; x6 = in->d[6]; x7 = in->d[7];
	x8 = in->d[8]; x9 = in->d[9]; x10 = in->d[10]; x11 = in->d[11];
	x12 = in->d[12]; x13 = in->d[13]; x14 = in->d[14]; x15 = in->d[15];

	for (i = rounds; i > 0; i -= 2) {
		QR(x0, x4, x8, x12);
		QR(x1, x5, x9, x13);
		QR(x2, x6, x10, x14);
		QR(x3, x7, x11, x15);

		QR(x0, x5, x10, x15);
		QR(x1, x6, x11, x12);
		QR(x2, x7, x8, x13);
		QR(x3, x4, x9, x14);
	}

	/* HChaCha doesn't perform the final addition */

	out[0] = cpu_to_le32(x0);
	out[1] = cpu_to_le32(x1);
	out[2] = cpu_to_le32(x2);
	out[3] = cpu_to_le32(x3);
	out[4] = cpu_to_le32(x12);
	out[5] = cpu_to_le32(x13);
	out[6] = cpu_to_le32(x14);
	out[7] = cpu_to_le32(x15);
}

/* Common initialization logic for ChaCha20 and HChaCha20 */
static void chacha_init_key_state(crypton_chacha_state *st,
                                  uint32_t keylen, const uint8_t *key)
{
	const uint8_t *constants = (keylen == 32) ? sigma : tau;

	ASSERT_ALIGNMENT(constants, 4);

	st->d[0] = load_le32_aligned(constants + 0);
	st->d[1] = load_le32_aligned(constants + 4);
	st->d[2] = load_le32_aligned(constants + 8);
	st->d[3] = load_le32_aligned(constants + 12);

	st->d[4] = load_le32(key + 0);
	st->d[5] = load_le32(key + 4);
	st->d[6] = load_le32(key + 8);
	st->d[7] = load_le32(key + 12);
	/* we repeat the key on 128 bits */
	if (keylen == 32)
		key += 16;
	st->d[8] = load_le32(key + 0);
	st->d[9] = load_le32(key + 4);
	st->d[10] = load_le32(key + 8);
	st->d[11] = load_le32(key + 12);
}

/* only 2 valids values are 256 (32) and 128 (16) */
void crypton_chacha_init_core(crypton_chacha_state *st,
                              uint32_t keylen, const uint8_t *key,
                              uint32_t ivlen, const uint8_t *iv)
{
	chacha_init_key_state(st, keylen, key);
	st->d[12] = 0;
	switch (ivlen) {
	case 8:
		st->d[13] = 0;
		st->d[14] = load_le32(iv + 0);
		st->d[15] = load_le32(iv + 4);
		break;
	case 12:
		st->d[13] = load_le32(iv + 0);
		st->d[14] = load_le32(iv + 4);
		st->d[15] = load_le32(iv + 8);
	default:
		return;
	}
}

void crypton_hchacha_init_core(crypton_chacha_state *st,
                                  const uint8_t *key,
                                  const uint8_t *iv)
{
	/* keylen is always 32 here */
	chacha_init_key_state(st, 32, key);
	/* fill the last 4 uint32s with the 128-bit nonce */
	st->d[12] = load_le32(iv + 0);
	st->d[13] = load_le32(iv + 4);
	st->d[14] = load_le32(iv + 8);
	st->d[15] = load_le32(iv + 12);
}

void crypton_chacha_init(crypton_chacha_context *ctx, uint8_t nb_rounds,
                            uint32_t keylen, const uint8_t *key,
                            uint32_t ivlen, const uint8_t *iv)
{
	memset(ctx, 0, sizeof(*ctx));
	ctx->nb_rounds = nb_rounds;
	crypton_chacha_init_core(&ctx->st, keylen, key, ivlen, iv);
}

void crypton_hchacha(uint8_t nb_rounds, const uint8_t *keyin, const uint8_t *iv,
                        uint8_t *keyout)
{
	crypton_chacha_state st;

	crypton_hchacha_init_core(&st, keyin, iv);
	/* output to uint32_t* and do a memcpy to avoid
	   violating the strict aliasing rule */
	uint32_t keyout32[8];
	hchacha_core(nb_rounds, keyout32, &st);
	memset(&st, 0, sizeof(st));
	memcpy(keyout, keyout32, 32);

	memset(keyout32, 0, 32);
}

/* XChaCha: 256-bit key, 192-bit nonce version of ChaCha.*/
void crypton_xchacha_init(crypton_chacha_context *ctx, uint8_t nb_rounds,
                             const uint8_t *key, const uint8_t *iv)
{
	memset(ctx, 0, sizeof(*ctx));
	ctx->nb_rounds = nb_rounds;

	/* perform HChaCha with the key and the first 16 bytes of the nonce
	to get the subkey */
	uint8_t subkey[32];
	crypton_hchacha(nb_rounds, key, iv, subkey);
	/* perform regular ChaCha with the generated subkey and the last 8 bytes
	of the input IV */
	crypton_chacha_init_core(&ctx->st, 32, subkey, 8, iv + 16);
}


void crypton_chacha_combine(uint8_t *dst, crypton_chacha_context *ctx, const uint8_t *src, uint32_t bytes)
{
	block out;
	crypton_chacha_state *st;
	int i;

	if (!bytes)
		return;

	/* xor the previous buffer first (if any) */
	if (ctx->prev_len > 0) {
		int to_copy = (ctx->prev_len < bytes) ? ctx->prev_len : bytes;
		for (i = 0; i < to_copy; i++)
			dst[i] = src[i] ^ ctx->prev[ctx->prev_ofs+i];
		memset(ctx->prev + ctx->prev_ofs, 0, to_copy);
		ctx->prev_len -= to_copy;
		ctx->prev_ofs += to_copy;
		src += to_copy;
		dst += to_copy;
		bytes -= to_copy;
	}

	if (bytes == 0)
		return;

	st = &ctx->st;

#ifdef CHACHA_ASM
	if (ctx->nb_rounds == 20 && chacha_asm_state(st)) {
		uint32_t blocks = bytes / 64;

		/* the counter is the caller's to advance, and the assembly
		 * carries it no further than its own 32 bits */
		if (blocks > 0xffffffffU - st->d[12])
			blocks = 0xffffffffU - st->d[12];
		if (blocks >= CHACHA_ASM_MIN_BLOCKS) {
			const uint32_t done = blocks * 64;

#ifdef CRYPTON_X86_ASM
			/* what the module dispatches on, which it reads
			 * directly; resolved once */
			crypton_x86_ia32cap_resolve();
#endif
			crypton_chacha20_asm_ctr32(dst, src, done, &st->d[4],
			                           &st->d[12]);
			st->d[12] += blocks;
			bytes -= done; src += done; dst += done;
		}
	}
#endif

#ifdef CHACHA_SIMD
	{
		const uint32_t nb = (uint32_t) crypton_chacha_simd_width();
		const uint32_t step = 64 * nb;

		while (bytes >= step && CHACHA_SIMD_OK(st, nb)) {
			crypton_chacha_simd_combine(ctx->nb_rounds, dst, src, st);
			st->d[12] += nb;
			bytes -= step; src += step; dst += step;
		}
	}
#endif

	/* xor new 64-bytes chunks and store the left over if any */
	for (; bytes >= 64; bytes -= 64, src += 64, dst += 64) {
		/* generate new chunk and update state */
		chacha_core(ctx->nb_rounds, &out, st);
		uint32_t t0 = le32_to_cpu(st->d[12]);
		st->d[12] = cpu_to_le32(t0 + 1);
		if (st->d[12] == 0) {
			uint32_t t1 = le32_to_cpu(st->d[13]);
			st->d[13] = cpu_to_le32(t1 + 1);
		}

		for (i = 0; i < 64; ++i)
			dst[i] = src[i] ^ out.b[i];
	}

	if (bytes > 0) {
		/* generate new chunk and update state */
		chacha_core(ctx->nb_rounds, &out, st);
		uint32_t t0 = le32_to_cpu(st->d[12]);
		st->d[12] = cpu_to_le32(t0 + 1);
		if (st->d[12] == 0) {
			uint32_t t1 = le32_to_cpu(st->d[13]);
			st->d[13] = cpu_to_le32(t1 + 1);
		}

		/* xor as much as needed */
		for (i = 0; i < bytes; i++)
			dst[i] = src[i] ^ out.b[i];

		/* copy the left over in the buffer */
		ctx->prev_len = 64 - bytes;
		ctx->prev_ofs = i;
		for (; i < 64; i++) {
			ctx->prev[i] = out.b[i];
		}
	}
}

uint64_t crypton_chacha_counter64(crypton_chacha_state *st)
{
	uint64_t result = ((uint64_t) le32_to_cpu(st->d[12]))
		| (((uint64_t) le32_to_cpu(st->d[13])) << 32);
	return result;
}

uint32_t crypton_chacha_counter32(crypton_chacha_state *st)
{
	return le32_to_cpu(st->d[12]);
}

void crypton_chacha_set_counter64(crypton_chacha_state *st, uint64_t block_counter)
{
	uint64_t current_counter;
	current_counter = ((uint64_t) le32_to_cpu(st->d[12]))
		| (((uint64_t) le32_to_cpu(st->d[13])) << 32);

	if (current_counter == block_counter)
		return;

	st->d[12] = cpu_to_le32((uint32_t) block_counter);
	st->d[13] = cpu_to_le32((uint32_t) (block_counter >> 32));
}

void crypton_chacha_set_counter32(crypton_chacha_state *st, uint32_t block_counter)
{
	uint32_t current_counter = le32_to_cpu(st->d[12]);

	if (current_counter == block_counter)
		return;

	st->d[12] = cpu_to_le32(block_counter);
}

void crypton_chacha_generate(uint8_t *dst, crypton_chacha_context *ctx, uint32_t bytes)
{
	crypton_chacha_state *st;
	block out;
	int i;

	if (!bytes)
		return;

	/* xor the previous buffer first (if any) */
	if (ctx->prev_len > 0) {
		int to_copy = (ctx->prev_len < bytes) ? ctx->prev_len : bytes;
		for (i = 0; i < to_copy; i++)
			dst[i] = ctx->prev[ctx->prev_ofs+i];
		memset(ctx->prev + ctx->prev_ofs, 0, to_copy);
		ctx->prev_len -= to_copy;
		ctx->prev_ofs += to_copy;
		dst += to_copy;
		bytes -= to_copy;
	}

	if (bytes == 0)
		return;

	st = &ctx->st;

#ifdef CHACHA_SIMD
	{
		const uint32_t nb = (uint32_t) crypton_chacha_simd_width();
		const uint32_t step = 64 * nb;

		while (bytes >= step && CHACHA_SIMD_OK(st, nb)) {
			crypton_chacha_simd_generate(ctx->nb_rounds, dst, st);
			st->d[12] += nb;
			bytes -= step; dst += step;
		}
	}
#endif

	if (ALIGNED64(dst)) {
		/* xor new 64-bytes chunks and store the left over if any */
		for (; bytes >= 64; bytes -= 64, dst += 64) {
			/* generate new chunk and update state */
			chacha_core(ctx->nb_rounds, (block *) dst, st);
			uint32_t t0 = le32_to_cpu(st->d[12]);
			st->d[12] = cpu_to_le32(t0 + 1);
			if (st->d[12] == 0) {
				uint32_t t1 = le32_to_cpu(st->d[13]);
				st->d[13] = cpu_to_le32(t1 + 1);
			}
		}
	} else {
		/* xor new 64-bytes chunks and store the left over if any */
		for (; bytes >= 64; bytes -= 64, dst += 64) {
			/* generate new chunk and update state */
			chacha_core(ctx->nb_rounds, &out, st);
			uint32_t t0 = le32_to_cpu(st->d[12]);
			st->d[12] = cpu_to_le32(t0 + 1);
			if (st->d[12] == 0) {
				uint32_t t1 = le32_to_cpu(st->d[13]);
				st->d[13] = cpu_to_le32(t1 + 1);
			}

			for (i = 0; i < 64; ++i)
				dst[i] = out.b[i];
		}
	}

	if (bytes > 0) {
		/* generate new chunk and update state */
		chacha_core(ctx->nb_rounds, &out, st);
		uint32_t t0 = le32_to_cpu(st->d[12]);
		st->d[12] = cpu_to_le32(t0 + 1);
		if (st->d[12] == 0) {
			uint32_t t1 = le32_to_cpu(st->d[13]);
			st->d[13] = cpu_to_le32(t1 + 1);
		}

		/* xor as much as needed */
		for (i = 0; i < bytes; i++)
			dst[i] = out.b[i];

		/* copy the left over in the buffer */
		ctx->prev_len = 64 - bytes;
		ctx->prev_ofs = i;
		for (; i < 64; i++)
			ctx->prev[i] = out.b[i];
	}
}

void crypton_chacha_generate_simple_block(uint8_t *dst, crypton_chacha_state *st, uint8_t rounds)
{
	if (ALIGNED64(dst)) {
		chacha_core(rounds, (block *) dst, st);
	} else {
		block out;
		int i;
		chacha_core(rounds, &out, st);
		for (i = 0; i < 64; ++i) {
			dst[i] = out.b[i];
		}
	}

	uint32_t t0 = le32_to_cpu(st->d[12]);
	st->d[12] = cpu_to_le32(t0 + 1);
	if (st->d[12] == 0) {
		uint32_t t1 = le32_to_cpu(st->d[13]);
		st->d[13] = cpu_to_le32(t1 + 1);
	}
}

void crypton_chacha_random(uint32_t rounds, uint8_t *dst, crypton_chacha_state *st, uint32_t bytes)
{
	block out;

	if (!bytes)
		return;
	for (; bytes >= 16; bytes -= 16, dst += 16) {
		chacha_core(rounds, &out, st);
		memcpy(dst, out.b + 40, 16);
		crypton_chacha_init_core(st, 32, out.b, 8, out.b + 32);
	}
	if (bytes) {
		chacha_core(rounds, &out, st);
		memcpy(dst, out.b + 40, bytes);
		crypton_chacha_init_core(st, 32, out.b, 8, out.b + 32);
	}
}