/*
* Copyright (c) 2012 Vincent Hanquez <vincent@snarc.org>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the author nor the names of his contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "cpu.h"
#include "aes.h"
#include "aes_generic.h"
#include "bitfn.h"
#include <string.h>
#include <stdio.h>
#include "gf.h"
#include "aes_x86ni.h"
void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);
void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);
void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);
void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);
void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length);
void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks);
void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks);
void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);
void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);
enum {
/* init */
INIT_128, INIT_192, INIT_256,
/* single block */
ENCRYPT_BLOCK_128, ENCRYPT_BLOCK_192, ENCRYPT_BLOCK_256,
DECRYPT_BLOCK_128, DECRYPT_BLOCK_192, DECRYPT_BLOCK_256,
/* ecb */
ENCRYPT_ECB_128, ENCRYPT_ECB_192, ENCRYPT_ECB_256,
DECRYPT_ECB_128, DECRYPT_ECB_192, DECRYPT_ECB_256,
/* cbc */
ENCRYPT_CBC_128, ENCRYPT_CBC_192, ENCRYPT_CBC_256,
DECRYPT_CBC_128, DECRYPT_CBC_192, DECRYPT_CBC_256,
/* ctr */
ENCRYPT_CTR_128, ENCRYPT_CTR_192, ENCRYPT_CTR_256,
/* xts */
ENCRYPT_XTS_128, ENCRYPT_XTS_192, ENCRYPT_XTS_256,
DECRYPT_XTS_128, DECRYPT_XTS_192, DECRYPT_XTS_256,
/* xts */
ENCRYPT_GCM_128, ENCRYPT_GCM_192, ENCRYPT_GCM_256,
DECRYPT_GCM_128, DECRYPT_GCM_192, DECRYPT_GCM_256,
};
void *branch_table[] = {
/* INIT */
[INIT_128] = aes_generic_init,
[INIT_192] = aes_generic_init,
[INIT_256] = aes_generic_init,
/* BLOCK */
[ENCRYPT_BLOCK_128] = aes_generic_encrypt_block,
[ENCRYPT_BLOCK_192] = aes_generic_encrypt_block,
[ENCRYPT_BLOCK_256] = aes_generic_encrypt_block,
[DECRYPT_BLOCK_128] = aes_generic_decrypt_block,
[DECRYPT_BLOCK_192] = aes_generic_decrypt_block,
[DECRYPT_BLOCK_256] = aes_generic_decrypt_block,
/* ECB */
[ENCRYPT_ECB_128] = aes_generic_encrypt_ecb,
[ENCRYPT_ECB_192] = aes_generic_encrypt_ecb,
[ENCRYPT_ECB_256] = aes_generic_encrypt_ecb,
[DECRYPT_ECB_128] = aes_generic_decrypt_ecb,
[DECRYPT_ECB_192] = aes_generic_decrypt_ecb,
[DECRYPT_ECB_256] = aes_generic_decrypt_ecb,
/* CBC */
[ENCRYPT_CBC_128] = aes_generic_encrypt_cbc,
[ENCRYPT_CBC_192] = aes_generic_encrypt_cbc,
[ENCRYPT_CBC_256] = aes_generic_encrypt_cbc,
[DECRYPT_CBC_128] = aes_generic_decrypt_cbc,
[DECRYPT_CBC_192] = aes_generic_decrypt_cbc,
[DECRYPT_CBC_256] = aes_generic_decrypt_cbc,
/* CTR */
[ENCRYPT_CTR_128] = aes_generic_encrypt_ctr,
[ENCRYPT_CTR_192] = aes_generic_encrypt_ctr,
[ENCRYPT_CTR_256] = aes_generic_encrypt_ctr,
/* XTS */
[ENCRYPT_XTS_128] = aes_generic_encrypt_xts,
[ENCRYPT_XTS_192] = aes_generic_encrypt_xts,
[ENCRYPT_XTS_256] = aes_generic_encrypt_xts,
[DECRYPT_XTS_128] = aes_generic_decrypt_xts,
[DECRYPT_XTS_192] = aes_generic_decrypt_xts,
[DECRYPT_XTS_256] = aes_generic_decrypt_xts,
/* GCM */
[ENCRYPT_GCM_128] = aes_generic_gcm_encrypt,
[ENCRYPT_GCM_192] = aes_generic_gcm_encrypt,
[ENCRYPT_GCM_256] = aes_generic_gcm_encrypt,
[DECRYPT_GCM_128] = aes_generic_gcm_decrypt,
[DECRYPT_GCM_192] = aes_generic_gcm_decrypt,
[DECRYPT_GCM_256] = aes_generic_gcm_decrypt,
};
typedef void (*init_f)(aes_key *, uint8_t *, uint8_t);
typedef void (*ecb_f)(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);
typedef void (*cbc_f)(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);
typedef void (*ctr_f)(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length);
typedef void (*xts_f)(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks);
typedef void (*gcm_crypt_f)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);
typedef void (*block_f)(aes_block *output, aes_key *key, aes_block *input);
#ifdef WITH_AESNI
#define GET_INIT(strength) \
((init_f) (branch_table[INIT_128 + strength]))
#define GET_ECB_ENCRYPT(strength) \
((ecb_f) (branch_table[ENCRYPT_ECB_128 + strength]))
#define GET_ECB_DECRYPT(strength) \
((ecb_f) (branch_table[DECRYPT_ECB_128 + strength]))
#define GET_CBC_ENCRYPT(strength) \
((cbc_f) (branch_table[ENCRYPT_CBC_128 + strength]))
#define GET_CBC_DECRYPT(strength) \
((cbc_f) (branch_table[DECRYPT_CBC_128 + strength]))
#define GET_CTR_ENCRYPT(strength) \
((ctr_f) (branch_table[ENCRYPT_CTR_128 + strength]))
#define GET_XTS_ENCRYPT(strength) \
((xts_f) (branch_table[ENCRYPT_XTS_128 + strength]))
#define GET_XTS_DECRYPT(strength) \
((xts_f) (branch_table[DECRYPT_XTS_128 + strength]))
#define GET_GCM_ENCRYPT(strength) \
((gcm_crypt_f) (branch_table[ENCRYPT_GCM_128 + strength]))
#define GET_GCM_DECRYPT(strength) \
((gcm_crypt_f) (branch_table[DECRYPT_GCM_128 + strength]))
#define aes_encrypt_block(o,k,i) \
(((block_f) (branch_table[ENCRYPT_BLOCK_128 + k->strength]))(o,k,i))
#define aes_decrypt_block(o,k,i) \
(((block_f) (branch_table[DECRYPT_BLOCK_128 + k->strength]))(o,k,i))
#else
#define GET_INIT(strenght) aes_generic_init
#define GET_ECB_ENCRYPT(strength) aes_generic_encrypt_ecb
#define GET_ECB_DECRYPT(strength) aes_generic_decrypt_ecb
#define GET_CBC_ENCRYPT(strength) aes_generic_encrypt_cbc
#define GET_CBC_DECRYPT(strength) aes_generic_decrypt_cbc
#define GET_CTR_ENCRYPT(strength) aes_generic_encrypt_ctr
#define GET_XTS_ENCRYPT(strength) aes_generic_encrypt_xts
#define GET_XTS_DECRYPT(strength) aes_generic_decrypt_xts
#define GET_GCM_ENCRYPT(strength) aes_generic_gcm_encrypt
#define GET_GCM_DECRYPT(strength) aes_generic_gcm_decrypt
#define aes_encrypt_block(o,k,i) aes_generic_encrypt_block(o,k,i)
#define aes_decrypt_block(o,k,i) aes_generic_decrypt_block(o,k,i)
#endif
#if defined(ARCH_X86) && defined(WITH_AESNI)
void initialize_table_ni(int aesni, int pclmul)
{
if (!aesni)
return;
branch_table[INIT_128] = aes_ni_init;
branch_table[INIT_256] = aes_ni_init;
branch_table[ENCRYPT_BLOCK_128] = aes_ni_encrypt_block128;
branch_table[DECRYPT_BLOCK_128] = aes_ni_decrypt_block128;
branch_table[ENCRYPT_BLOCK_256] = aes_ni_encrypt_block256;
branch_table[DECRYPT_BLOCK_256] = aes_ni_decrypt_block256;
/* ECB */
branch_table[ENCRYPT_ECB_128] = aes_ni_encrypt_ecb128;
branch_table[DECRYPT_ECB_128] = aes_ni_decrypt_ecb128;
branch_table[ENCRYPT_ECB_256] = aes_ni_encrypt_ecb256;
branch_table[DECRYPT_ECB_256] = aes_ni_decrypt_ecb256;
/* CBC */
branch_table[ENCRYPT_CBC_128] = aes_ni_encrypt_cbc128;
branch_table[DECRYPT_CBC_128] = aes_ni_decrypt_cbc128;
branch_table[ENCRYPT_CBC_256] = aes_ni_encrypt_cbc256;
branch_table[DECRYPT_CBC_256] = aes_ni_decrypt_cbc256;
/* CTR */
branch_table[ENCRYPT_CTR_128] = aes_ni_encrypt_ctr128;
branch_table[ENCRYPT_CTR_256] = aes_ni_encrypt_ctr256;
/* XTS */
branch_table[ENCRYPT_XTS_128] = aes_ni_encrypt_xts128;
branch_table[ENCRYPT_XTS_256] = aes_ni_encrypt_xts256;
/* GCM */
branch_table[ENCRYPT_GCM_128] = aes_ni_gcm_encrypt128;
branch_table[ENCRYPT_GCM_256] = aes_ni_gcm_encrypt256;
}
#endif
void aes_initkey(aes_key *key, uint8_t *origkey, uint8_t size)
{
switch (size) {
case 16: key->nbr = 10; key->strength = 0; break;
case 24: key->nbr = 12; key->strength = 1; break;
case 32: key->nbr = 14; key->strength = 2; break;
}
#if defined(ARCH_X86) && defined(WITH_AESNI)
initialize_hw(initialize_table_ni);
#endif
init_f _init = GET_INIT(key->strength);
_init(key, origkey, size);
}
void aes_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)
{
ecb_f e = GET_ECB_ENCRYPT(key->strength);
e(output, key, input, nb_blocks);
}
void aes_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)
{
ecb_f d = GET_ECB_DECRYPT(key->strength);
d(output, key, input, nb_blocks);
}
void aes_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks)
{
cbc_f e = GET_CBC_ENCRYPT(key->strength);
e(output, key, iv, input, nb_blocks);
}
void aes_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks)
{
cbc_f d = GET_CBC_DECRYPT(key->strength);
d(output, key, iv, input, nb_blocks);
}
void aes_gen_ctr(aes_block *output, aes_key *key, aes_block *iv, uint32_t nb_blocks)
{
aes_block block;
/* preload IV in block */
block128_copy(&block, iv);
for ( ; nb_blocks-- > 0; output++, block128_inc_be(&block)) {
aes_encrypt_block(output, key, &block);
}
}
void aes_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len)
{
ctr_f e = GET_CTR_ENCRYPT(key->strength);
e(output, key, iv, input, len);
}
void aes_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks)
{
xts_f e = GET_XTS_ENCRYPT(k1->strength);
e(output, k1, k2, dataunit, spoint, input, nb_blocks);
}
void aes_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks)
{
aes_generic_decrypt_xts(output, k1, k2, dataunit, spoint, input, nb_blocks);
}
void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)
{
gcm_crypt_f e = GET_GCM_ENCRYPT(key->strength);
e(output, gcm, key, input, length);
}
void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)
{
gcm_crypt_f d = GET_GCM_DECRYPT(key->strength);
d(output, gcm, key, input, length);
}
static void gcm_ghash_add(aes_gcm *gcm, block128 *b)
{
block128_xor(&gcm->tag, b);
gf_mul(&gcm->tag, &gcm->h);
}
void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len)
{
gcm->length_aad = 0;
gcm->length_input = 0;
block128_zero(&gcm->h);
block128_zero(&gcm->tag);
block128_zero(&gcm->iv);
/* prepare H : encrypt_K(0^128) */
aes_encrypt_block(&gcm->h, key, &gcm->h);
if (len == 12) {
block128_copy_bytes(&gcm->iv, iv, 12);
gcm->iv.b[15] = 0x01;
} else {
uint32_t origlen = len << 3;
int i;
for (; len >= 16; len -= 16, iv += 16) {
block128_xor(&gcm->iv, (block128 *) iv);
gf_mul(&gcm->iv, &gcm->h);
}
if (len > 0) {
block128_xor_bytes(&gcm->iv, iv, len);
gf_mul(&gcm->iv, &gcm->h);
}
for (i = 15; origlen; --i, origlen >>= 8)
gcm->iv.b[i] ^= (uint8_t) origlen;
gf_mul(&gcm->iv, &gcm->h);
}
block128_copy(&gcm->civ, &gcm->iv);
}
void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length)
{
gcm->length_aad += length;
for (; length >= 16; input += 16, length -= 16) {
gcm_ghash_add(gcm, (block128 *) input);
}
if (length > 0) {
aes_block tmp;
block128_zero(&tmp);
block128_copy_bytes(&tmp, input, length);
gcm_ghash_add(gcm, &tmp);
}
}
void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm, aes_key *key)
{
aes_block lblock;
int i;
/* tag = (tag-1 xor (lenbits(a) | lenbits(c)) ) . H */
lblock.q[0] = cpu_to_be64(gcm->length_aad << 3);
lblock.q[1] = cpu_to_be64(gcm->length_input << 3);
gcm_ghash_add(gcm, &lblock);
aes_encrypt_block(&lblock, key, &gcm->iv);
block128_xor(&gcm->tag, &lblock);
for (i = 0; i < 16; i++) {
tag[i] = gcm->tag.b[i];
}
}
void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)
{
for ( ; nb_blocks-- > 0; input++, output++) {
aes_generic_encrypt_block(output, key, input);
}
}
void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)
{
for ( ; nb_blocks-- > 0; input++, output++) {
aes_generic_decrypt_block(output, key, input);
}
}
void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks)
{
aes_block block;
/* preload IV in block */
block128_copy(&block, iv);
for ( ; nb_blocks-- > 0; input++, output++) {
block128_xor(&block, (block128 *) input);
aes_generic_encrypt_block(&block, key, &block);
block128_copy((block128 *) output, &block);
}
}
void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *ivini, aes_block *input, uint32_t nb_blocks)
{
aes_block block, blocko;
aes_block iv;
/* preload IV in block */
block128_copy(&iv, ivini);
for ( ; nb_blocks-- > 0; input++, output++) {
block128_copy(&block, (block128 *) input);
aes_generic_decrypt_block(&blocko, key, &block);
block128_vxor((block128 *) output, &blocko, &iv);
block128_copy(&iv, &block);
}
}
void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len)
{
aes_block block, o;
uint32_t nb_blocks = len / 16;
int i;
/* preload IV in block */
block128_copy(&block, iv);
for ( ; nb_blocks-- > 0; block128_inc_be(&block), output += 16, input += 16) {
aes_encrypt_block(&o, key, &block);
block128_vxor((block128 *) output, &o, (block128 *) input);
}
if ((len % 16) != 0) {
aes_encrypt_block(&o, key, &block);
for (i = 0; i < (len % 16); i++) {
*output = ((uint8_t *) &o)[i] ^ *input;
output++;
input++;
}
}
}
void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks)
{
aes_block block, tweak;
/* load IV and encrypt it using k2 as the tweak */
block128_copy(&tweak, dataunit);
aes_encrypt_block(&tweak, k2, &tweak);
/* TO OPTIMISE: this is really inefficient way to do that */
while (spoint-- > 0)
gf_mulx(&tweak);
for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) {
block128_vxor(&block, input, &tweak);
aes_encrypt_block(&block, k1, &block);
block128_vxor(output, &block, &tweak);
}
}
void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,
uint32_t spoint, aes_block *input, uint32_t nb_blocks)
{
aes_block block, tweak;
/* load IV and encrypt it using k2 as the tweak */
block128_copy(&tweak, dataunit);
aes_encrypt_block(&tweak, k2, &tweak);
/* TO OPTIMISE: this is really inefficient way to do that */
while (spoint-- > 0)
gf_mulx(&tweak);
for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) {
block128_vxor(&block, input, &tweak);
aes_decrypt_block(&block, k1, &block);
block128_vxor(output, &block, &tweak);
}
}
void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)
{
aes_block out;
gcm->length_input += length;
for (; length >= 16; input += 16, output += 16, length -= 16) {
block128_inc_be(&gcm->civ);
aes_encrypt_block(&out, key, &gcm->civ);
block128_xor(&out, (block128 *) input);
gcm_ghash_add(gcm, &out);
block128_copy((block128 *) output, &out);
}
if (length > 0) {
aes_block tmp;
int i;
block128_inc_be(&gcm->civ);
/* create e(civ) in out */
aes_encrypt_block(&out, key, &gcm->civ);
/* initialize a tmp as input and xor it to e(civ) */
block128_zero(&tmp);
block128_copy_bytes(&tmp, input, length);
block128_xor_bytes(&tmp, out.b, length);
gcm_ghash_add(gcm, &tmp);
for (i = 0; i < length; i++) {
output[i] = tmp.b[i];
}
}
}
void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)
{
aes_block out;
gcm->length_input += length;
for (; length >= 16; input += 16, output += 16, length -= 16) {
block128_inc_be(&gcm->civ);
aes_encrypt_block(&out, key, &gcm->civ);
gcm_ghash_add(gcm, (block128 *) input);
block128_xor(&out, (block128 *) input);
block128_copy((block128 *) output, &out);
}
if (length > 0) {
aes_block tmp;
int i;
block128_inc_be(&gcm->civ);
block128_zero(&tmp);
block128_copy_bytes(&tmp, input, length);
gcm_ghash_add(gcm, &tmp);
aes_encrypt_block(&out, key, &gcm->civ);
block128_xor_bytes(&tmp, out.b, length);
for (i = 0; i < length; i++) {
output[i] = tmp.b[i];
}
}
}