diff --git a/Benchmarks/Benchmarks.hs b/Benchmarks/Benchmarks.hs --- a/Benchmarks/Benchmarks.hs +++ b/Benchmarks/Benchmarks.hs @@ -1,102 +1,7 @@ -{-# LANGUAGE CPP #-} -import Criterion -import Criterion.Environment -import Criterion.Config -import Criterion.Monad -import Criterion.Analysis -import Criterion.Measurement - -import Text.Printf - -import Control.Monad.Trans - -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as L - -import qualified Crypto.Cipher.AES as AES - -key128 = AES.initKey $ B.replicate 16 0 -key192 = AES.initKey $ B.replicate 24 0 -key256 = AES.initKey $ B.replicate 32 0 - -nullIV = AES.IV $ B.replicate 16 0 -nullIVGCM = AES.IV $ B.replicate 12 0 - -aesEncrypt128 = AES.encryptECB key128 -aesEncrypt128CBC = AES.encryptCBC key128 nullIV -aesEncrypt128CTR = AES.encryptCTR key128 nullIV -aesEncrypt128XTS = AES.encryptXTS (key128,key128) nullIV 0 -aesEncrypt128GCM = fst . AES.encryptGCM key128 nullIVGCM B.empty - -aesEncrypt192 = AES.encryptECB key192 -aesEncrypt192CBC = AES.encryptCBC key192 nullIV -aesEncrypt192CTR = AES.encryptCTR key192 nullIV -aesEncrypt192GCM = fst . AES.encryptGCM key192 nullIVGCM B.empty -aesEncrypt256 = AES.encryptECB key256 -aesEncrypt256CBC = AES.encryptCBC key256 nullIV -aesEncrypt256CTR = AES.encryptCTR key256 nullIV -aesEncrypt256XTS = AES.encryptXTS (key256,key256) nullIV 0 -aesEncrypt256GCM = fst . AES.encryptGCM key256 nullIVGCM B.empty - -b16 f = whnf f $ B.replicate 16 0 -b32 f = whnf f $ B.replicate 32 0 -b128 f = whnf f $ B.replicate 128 0 -b512 f = whnf f $ B.replicate 512 0 -b1024 f = whnf f $ B.replicate 1024 0 -b4096 f = whnf f $ B.replicate 4096 0 -b16384 f = whnf f $ B.replicate 16384 0 - -doCipher env f = do - mean16 <- runBenchmark env (b16 f) >>= \sample -> analyseMean sample 100 - mean32 <- runBenchmark env (b32 f) >>= \sample -> analyseMean sample 100 - mean128 <- runBenchmark env (b128 f) >>= \sample -> analyseMean sample 100 - mean512 <- runBenchmark env (b512 f) >>= \sample -> analyseMean sample 100 - mean1024 <- runBenchmark env (b1024 f) >>= \sample -> analyseMean sample 100 - mean4096 <- runBenchmark env (b4096 f) >>= \sample -> analyseMean sample 100 - mean16384 <- runBenchmark env (b16384 f) >>= \sample -> analyseMean sample 100 - return (mean16, mean32, mean128, mean512, mean1024, mean4096, mean16384) - -norm :: Int -> Double -> Double -norm n time - | n < 1024 = 1.0 / (time * (1024 / fromIntegral n)) - | n == 1024 = 1.0 / time - | n > 1024 = 1.0 / (time / (fromIntegral n / 1024)) - -pn :: Int -> Double -> String -pn n time - | val > (10 * 1024) = printf "%.1f M/s" (val / 1024) - | otherwise = printf "%.1f K/s" val - where val = norm n time - -doOne env (cipherName, f) = do - (mean16, mean32, mean128, mean512, mean1024, mean4096, mean16384) <- doCipher env f - let s = printf "%12s: %12s %12s %12s %12s %12s %12s %12s\n %12s %12s %12s %12s %12s %12s %12s" - cipherName - (secs mean16) (secs mean32) (secs mean128) - (secs mean512) (secs mean1024) (secs mean4096) (secs mean16384) - (pn 16 mean16) (pn 32 mean32) (pn 128 mean128) - (pn 512 mean512) (pn 1024 mean1024) (pn 4096 mean4096) (pn 16384 mean16384) - return s +import Crypto.Cipher.Benchmarks +import Crypto.Cipher.AES (AES128, AES192, AES256) -main = withConfig defaultConfig $ do - env <- measureEnvironment - l <- mapM (doOne env) - [ ("AES128" , aesEncrypt128) - , ("AES128-CBC" , aesEncrypt128CBC) - , ("AES128-CTR" , aesEncrypt128CTR) - , ("AES128-XTS" , aesEncrypt128XTS) - , ("AES128-GCM" , aesEncrypt128GCM) - , ("AES192" , aesEncrypt192) - , ("AES192-CBC" , aesEncrypt192CBC) - , ("AES192-CTR" , aesEncrypt192CTR) - , ("AES192-GCM" , aesEncrypt192GCM) - , ("AES256" , aesEncrypt256) - , ("AES256-CBC" , aesEncrypt256CBC) - , ("AES256-CTR" , aesEncrypt256CTR) - , ("AES256-XTS" , aesEncrypt256XTS) - , ("AES256-GCM" , aesEncrypt256GCM) - ] - liftIO $ printf "%12s| %12s %12s %12s %12s %12s %12s %12s\n" - "cipher" "16 bytes" "32 bytes" "64 bytes" "512 bytes" "1024 bytes" "4096 bytes" "16384 bytes" - liftIO $ printf "===================================================================================================\n" - mapM_ (liftIO . putStrLn) l +main = defaultMain + [GBlockCipher (undefined :: AES128) + ,GBlockCipher (undefined :: AES192) + ,GBlockCipher (undefined :: AES256)] diff --git a/Crypto/Cipher/AES.hs b/Crypto/Cipher/AES.hs --- a/Crypto/Cipher/AES.hs +++ b/Crypto/Cipher/AES.hs @@ -1,5 +1,7 @@ {-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE ViewPatterns #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE CPP #-} -- | -- Module : Crypto.Cipher.AES -- License : BSD-style @@ -10,12 +12,14 @@ module Crypto.Cipher.AES ( -- * data types - Key - , IV(..) + AES + , AES128 + , AES192 + , AES256 -- * creation + , initAES , initKey - , keyOfCtx -- * misc , genCTR @@ -35,82 +39,121 @@ , decryptGCM ) where --- import Data.ByteString (ByteString) import Data.Word import Foreign.Ptr -import Foreign.ForeignPtr -import Foreign.Storable import Foreign.C.Types import Foreign.C.String -import Foreign.Marshal.Alloc import Data.ByteString.Internal import Data.ByteString.Unsafe +import Data.Byteable import qualified Data.ByteString as B import System.IO.Unsafe (unsafePerformIO) --- | AES Key -newtype Key = Key ByteString +import Crypto.Cipher.Types +import Data.SecureMem --- | AES IV -newtype IV = IV ByteString +-- | AES Context (pre-processed key) +newtype AES = AES SecureMem --- | GCM Context -newtype GCM = GCM ByteString +newtype AES128 = AES128 AES +newtype AES192 = AES192 AES +newtype AES256 = AES256 AES -sizeGCM :: Int -sizeGCM = 540 +instance Cipher AES128 where + cipherName _ = "AES128" + cipherKeySize _ = Just 16 + cipherInit k = AES128 $ initAES k -instance Storable GCM where - sizeOf _ = sizeGCM - alignment _ = 16 - poke ptr (GCM b) = unsafeUseAsCString b (\cs -> memcpy (castPtr ptr) (castPtr cs) (fromIntegral sizeGCM)) - peek ptr = create sizeGCM (\bptr -> memcpy bptr (castPtr ptr) (fromIntegral sizeGCM)) >>= return . GCM +instance Cipher AES192 where + cipherName _ = "AES192" + cipherKeySize _ = Just 24 + cipherInit k = AES192 $ initAES k -keyToPtr :: Key -> (Ptr Key -> IO a) -> IO a -keyToPtr (Key b) f = unsafeUseAsCString b (f . castPtr) +instance Cipher AES256 where + cipherName _ = "AES256" + cipherKeySize _ = Just 32 + cipherInit k = AES256 $ initAES k -ivToPtr :: IV -> (Ptr IV -> IO a) -> IO a -ivToPtr (IV b) f = unsafeUseAsCString b (f . castPtr) +#define INSTANCE_BLOCKCIPHER(CSTR) \ +instance BlockCipher CSTR where \ + { blockSize _ = 16 \ + ; ecbEncrypt (CSTR aes) = encryptECB aes \ + ; ecbDecrypt (CSTR aes) = decryptECB aes \ + ; cbcEncrypt (CSTR aes) = encryptCBC aes \ + ; cbcDecrypt (CSTR aes) = decryptCBC aes \ + ; ctrCombine (CSTR aes) = encryptCTR aes \ + ; xtsEncrypt (CSTR aes1, CSTR aes2) = encryptXTS (aes1,aes2) \ + ; xtsDecrypt (CSTR aes1, CSTR aes2) = decryptXTS (aes1,aes2) \ + ; aeadInit AEAD_GCM cipher@(CSTR aes) iv = Just $ AEAD cipher $ AEADState $ gcmInit aes iv \ + ; aeadInit _ _ _ = Nothing \ + }; \ +\ +instance AEADModeImpl CSTR GCM where \ + { aeadStateAppendHeader (CSTR _) gcmState bs = gcmAppendAAD gcmState bs \ + ; aeadStateEncrypt (CSTR aes) gcmState input = gcmAppendEncrypt aes gcmState input \ + ; aeadStateDecrypt (CSTR aes) gcmState input = gcmAppendDecrypt aes gcmState input \ + ; aeadStateFinalize (CSTR aes) gcmState len = gcmFinish aes gcmState len \ + } -withKeyAndIV :: Key -> IV -> (Ptr Key -> Ptr IV -> IO a) -> IO a -withKeyAndIV key iv f = keyToPtr key $ \kptr -> ivToPtr iv $ \ivp -> f kptr ivp +INSTANCE_BLOCKCIPHER(AES128) +INSTANCE_BLOCKCIPHER(AES192) +INSTANCE_BLOCKCIPHER(AES256) -withKey2AndIV :: Key -> Key -> IV -> (Ptr Key -> Ptr Key -> Ptr IV -> IO a) -> IO a +-- | GCM State +newtype GCM = GCM SecureMem + +sizeGCM :: Int +sizeGCM = 80 + +keyToPtr :: AES -> (Ptr AES -> IO a) -> IO a +keyToPtr (AES b) f = withSecureMemPtr b (f . castPtr) + +ivToPtr :: Byteable iv => iv -> (Ptr Word8 -> IO a) -> IO a +ivToPtr iv f = withBytePtr iv (f . castPtr) + +withKeyAndIV :: Byteable iv => AES -> iv -> (Ptr AES -> Ptr Word8 -> IO a) -> IO a +withKeyAndIV ctx iv f = keyToPtr ctx $ \kptr -> ivToPtr iv $ \ivp -> f kptr ivp + +withKey2AndIV :: Byteable iv => AES -> AES -> iv -> (Ptr AES -> Ptr AES -> Ptr Word8 -> IO a) -> IO a withKey2AndIV key1 key2 iv f = keyToPtr key1 $ \kptr1 -> keyToPtr key2 $ \kptr2 -> ivToPtr iv $ \ivp -> f kptr1 kptr2 ivp +withGCMKeyAndCopySt :: AES -> GCM -> (Ptr GCM -> Ptr AES -> IO a) -> IO (a, GCM) +withGCMKeyAndCopySt aes (GCM gcmSt) f = + keyToPtr aes $ \aesPtr -> do + newSt <- secureMemCopy gcmSt + a <- withSecureMemPtr newSt $ \gcmStPtr -> f (castPtr gcmStPtr) aesPtr + return (a, GCM newSt) + +withNewGCMSt :: GCM -> (Ptr GCM -> IO ()) -> IO GCM +withNewGCMSt (GCM gcmSt) f = withSecureMemCopy gcmSt (f . castPtr) >>= \sm2 -> return (GCM sm2) + -- | initialize key -{-# NOINLINE initKey #-} -initKey :: ByteString -> Key -initKey b@(B.length -> len) - | len == 16 = doInit 10 - | len == 24 = doInit 12 - | len == 32 = doInit 14 - | otherwise = error "wrong key size: need to be 16, 24 or 32 bytes." - where doInit nbR = unsafePerformIO $ unsafeUseAsCString b (allocAndFill nbR) - allocAndFill nbR ikey = do - ptr <- mallocBytes (16+2*2*16*nbR) - c_aes_init ptr (castPtr ikey) (fromIntegral len) - fptr <- newForeignPtr c_free_finalizer (castPtr ptr) - return $ Key $ fromForeignPtr fptr 0 (16+2*2*16*nbR) +-- +-- rounds need to be 10 / 12 / 14. any other values will cause undefined behavior +initAES :: Byteable b => b -> AES +initAES k + | len == 16 = initWithRounds 10 + | len == 24 = initWithRounds 12 + | len == 32 = initWithRounds 14 + | otherwise = error "not a valid key length" + where len = byteableLength k + initWithRounds nbR = AES $ unsafeCreateSecureMem (16+2*2*16*nbR) aesInit + aesInit ptr = withBytePtr k $ \ikey -> + c_aes_init (castPtr ptr) (castPtr ikey) (fromIntegral len) --- | return the user key from the Key context -keyOfCtx :: Key -> ByteString -keyOfCtx (Key bs) = B.take sz (B.drop 8 bs) - where nbRound = unsafeHead $ B.take 1 bs - sz | nbRound == 10 = 16 - | nbRound == 12 = 24 - | nbRound == 14 = 32 - | otherwise = error "not a valid key" +{-# DEPRECATED initKey "use initAES" #-} +initKey :: Byteable b => b -> AES +initKey = initAES -- | encrypt using Electronic Code Book (ECB) {-# NOINLINE encryptECB #-} -encryptECB :: Key -> ByteString -> ByteString +encryptECB :: AES -> ByteString -> ByteString encryptECB = doECB c_aes_encrypt_ecb -- | encrypt using Cipher Block Chaining (CBC) {-# NOINLINE encryptCBC #-} -encryptCBC :: Key -> IV -> ByteString -> ByteString +encryptCBC :: Byteable iv => AES -> iv -> ByteString -> ByteString encryptCBC = doCBC c_aes_encrypt_cbc -- | generate a counter mode pad. this is generally xor-ed to an input @@ -120,36 +163,46 @@ -- more data will be returned, so that the returned bytestring is -- a multiple of the block cipher size. {-# NOINLINE genCTR #-} -genCTR :: Key -- ^ Cipher Key. - -> IV -- ^ usually a 128 bit integer. - -> Int -- ^ length of bytes required. +genCTR :: Byteable iv + => AES -- ^ Cipher Key. + -> iv -- ^ usually a 128 bit integer. + -> Int -- ^ length of bytes required. -> ByteString -genCTR key iv len = unsafeCreate (nbBlocks * 16) generate - where - generate o = withKeyAndIV key iv $ \k i -> c_aes_gen_ctr (castPtr o) k i (fromIntegral nbBlocks) - (nbBlocks',r) = len `divMod` 16 - nbBlocks = if r == 0 then nbBlocks' else nbBlocks' + 1 +genCTR ctx iv len + | len <= 0 = B.empty + | otherwise = unsafeCreate (nbBlocks * 16) generate + where generate o = withKeyAndIV ctx iv $ \k i -> c_aes_gen_ctr (castPtr o) k i (fromIntegral nbBlocks) + (nbBlocks',r) = len `quotRem` 16 + nbBlocks = if r == 0 then nbBlocks' else nbBlocks' + 1 -- | encrypt using Counter mode (CTR) -- -- in CTR mode encryption and decryption is the same operation. {-# NOINLINE encryptCTR #-} -encryptCTR :: Key -> IV -> ByteString -> ByteString -encryptCTR key iv input = unsafeCreate len doEncrypt - where doEncrypt o = withKeyAndIV key iv $ \k v -> unsafeUseAsCString input $ \i -> - c_aes_encrypt_ctr (castPtr o) k v i (fromIntegral len) - len = B.length input +encryptCTR :: Byteable iv + => AES + -> iv + -> ByteString + -> ByteString +encryptCTR ctx iv input + | len <= 0 = B.empty + | otherwise = unsafeCreate len doEncrypt + where doEncrypt o = withKeyAndIV ctx iv $ \k v -> unsafeUseAsCString input $ \i -> + c_aes_encrypt_ctr (castPtr o) k v i (fromIntegral len) + len = B.length input -- | encrypt using Galois counter mode (GCM) -- return the encrypted bytestring and the tag associated -- -- note: encrypted data is identical to CTR mode in GCM, however -- a tag is also computed. -encryptGCM :: Key -- ^ Key - -> IV -- ^ initial vector +{-# NOINLINE encryptGCM #-} +encryptGCM :: Byteable iv + => AES -- ^ Key + -> iv -- ^ initial vector -> ByteString -- ^ data to authenticate (AAD) -> ByteString -- ^ data to encrypt - -> (ByteString, ByteString) -- ^ ciphertext and tag + -> (ByteString, AuthTag) -- ^ ciphertext and tag encryptGCM = doGCM gcmAppendEncrypt -- | encrypt using XTS @@ -157,164 +210,173 @@ -- the first key is the normal block encryption key -- the second key is used for the initial block tweak {-# NOINLINE encryptXTS #-} -encryptXTS :: (Key,Key) -> IV -> Word32 -> ByteString -> ByteString +encryptXTS :: Byteable iv => (AES,AES) -> iv -> Word32 -> ByteString -> ByteString encryptXTS = doXTS c_aes_encrypt_xts -- | decrypt using Electronic Code Book (ECB) {-# NOINLINE decryptECB #-} -decryptECB :: Key -> ByteString -> ByteString +decryptECB :: AES -> ByteString -> ByteString decryptECB = doECB c_aes_decrypt_ecb -- | decrypt using Cipher block chaining (CBC) {-# NOINLINE decryptCBC #-} -decryptCBC :: Key -> IV -> ByteString -> ByteString +decryptCBC :: Byteable iv => AES -> iv -> ByteString -> ByteString decryptCBC = doCBC c_aes_decrypt_cbc -- | decrypt using Counter mode (CTR). -- -- in CTR mode encryption and decryption is the same operation. -decryptCTR :: Key -> IV -> ByteString -> ByteString +decryptCTR :: Byteable iv => AES -> iv -> ByteString -> ByteString decryptCTR = encryptCTR -- | decrypt using XTS {-# NOINLINE decryptXTS #-} -decryptXTS :: (Key,Key) -> IV -> Word32 -> ByteString -> ByteString +decryptXTS :: Byteable iv => (AES,AES) -> iv -> Word32 -> ByteString -> ByteString decryptXTS = doXTS c_aes_decrypt_xts -- | decrypt using Galois Counter Mode (GCM) {-# NOINLINE decryptGCM #-} -decryptGCM :: Key -> IV -> ByteString -> ByteString -> (ByteString, ByteString) +decryptGCM :: Byteable iv => AES -> iv -> ByteString -> ByteString -> (ByteString, AuthTag) decryptGCM = doGCM gcmAppendDecrypt {-# INLINE doECB #-} -doECB :: (Ptr b -> Ptr Key -> CString -> CUInt -> IO ()) - -> Key -> ByteString -> ByteString -doECB f key input +doECB :: (Ptr b -> Ptr AES -> CString -> CUInt -> IO ()) + -> AES -> ByteString -> ByteString +doECB f ctx input | r /= 0 = error "cannot use with non multiple of block size" - | otherwise = unsafeCreate len $ \o -> keyToPtr key $ \k -> unsafeUseAsCString input $ \i -> - f (castPtr o) k i (fromIntegral nbBlocks) - where (nbBlocks, r) = len `divMod` 16 - len = (B.length input) + | otherwise = unsafeCreate len $ \o -> + keyToPtr ctx $ \k -> + unsafeUseAsCString input $ \i -> + f (castPtr o) k i (fromIntegral nbBlocks) + where (nbBlocks, r) = len `quotRem` 16 + len = (B.length input) {-# INLINE doCBC #-} -doCBC :: (Ptr b -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO ()) - -> Key -> IV -> ByteString -> ByteString -doCBC f key iv input +doCBC :: Byteable iv + => (Ptr b -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO ()) + -> AES -> iv -> ByteString -> ByteString +doCBC f ctx iv input + | len == 0 = B.empty | r /= 0 = error "cannot use with non multiple of block size" - | otherwise = unsafeCreate len $ \o -> withKeyAndIV key iv $ \k v -> unsafeUseAsCString input $ \i -> - f (castPtr o) k v i (fromIntegral nbBlocks) - where (nbBlocks, r) = len `divMod` 16 - len = (B.length input) + | otherwise = unsafeCreate len $ \o -> + withKeyAndIV ctx iv $ \k v -> + unsafeUseAsCString input $ \i -> + f (castPtr o) k v i (fromIntegral nbBlocks) + where (nbBlocks, r) = len `quotRem` 16 + len = B.length input {-# INLINE doXTS #-} -doXTS :: (Ptr b -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO ()) - -> (Key, Key) -> IV -> Word32 -> ByteString -> ByteString +doXTS :: Byteable iv + => (Ptr b -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO ()) + -> (AES, AES) -> iv -> Word32 -> ByteString -> ByteString doXTS f (key1,key2) iv spoint input + | len == 0 = B.empty | r /= 0 = error "cannot use with non multiple of block size (yet)" | otherwise = unsafeCreate len $ \o -> withKey2AndIV key1 key2 iv $ \k1 k2 v -> unsafeUseAsCString input $ \i -> f (castPtr o) k1 k2 v (fromIntegral spoint) i (fromIntegral nbBlocks) - where (nbBlocks, r) = len `divMod` 16 - len = (B.length input) + where (nbBlocks, r) = len `quotRem` 16 + len = B.length input {-# INLINE doGCM #-} -doGCM :: (GCM -> ByteString -> (ByteString, GCM)) -> Key -> IV -> ByteString -> ByteString -> (ByteString, ByteString) -doGCM f key iv aad input = (cipher, tag) - where - tag = gcmFinish after 16 - (cipher, after) = f afterAAD input - afterAAD = gcmAppendAAD ini aad - ini = gcmInit key iv - -allocaFrom :: Storable a => a -> (Ptr a -> IO b) -> IO b -allocaFrom z f = alloca $ \ptr -> poke ptr z >> f ptr +doGCM :: Byteable iv => (AES -> GCM -> ByteString -> (ByteString, GCM)) -> AES -> iv -> ByteString -> ByteString -> (ByteString, AuthTag) +doGCM f ctx iv aad input = (output, tag) + where tag = gcmFinish ctx after 16 + (output, after) = f ctx afterAAD input + afterAAD = gcmAppendAAD ini aad + ini = gcmInit ctx iv -- | initialize a gcm context {-# NOINLINE gcmInit #-} -gcmInit :: Key -> IV -> GCM -gcmInit key iv@(IV b) = unsafePerformIO $ alloca doInit - where doInit gcm = withKeyAndIV key iv (\k v -> c_aes_gcm_init gcm k v (fromIntegral $ B.length b)) >> peek gcm +gcmInit :: Byteable iv => AES -> iv -> GCM +gcmInit ctx iv = unsafePerformIO $ do + sm <- createSecureMem sizeGCM $ \gcmStPtr -> + withKeyAndIV ctx iv $ \k v -> + c_aes_gcm_init (castPtr gcmStPtr) k v (fromIntegral $ byteableLength iv) + return $ GCM sm -- | append data which is going to just be authentified to the GCM context. -- -- need to happen after initialization and before appending encryption/decryption data. {-# NOINLINE gcmAppendAAD #-} gcmAppendAAD :: GCM -> ByteString -> GCM -gcmAppendAAD gcm input = unsafePerformIO $ allocaFrom gcm doAppend - where doAppend p = do - unsafeUseAsCString input $ \i -> c_aes_gcm_aad p i (fromIntegral $ B.length input) - peek p +gcmAppendAAD gcmSt input = unsafePerformIO doAppend + where doAppend = + withNewGCMSt gcmSt $ \gcmStPtr -> + unsafeUseAsCString input $ \i -> + c_aes_gcm_aad gcmStPtr i (fromIntegral $ B.length input) -- | append data to encrypt and append to the GCM context -- -- bytestring need to be multiple of AES block size, unless it's the last call to this function. -- need to happen after AAD appending, or after initialization if no AAD data. {-# NOINLINE gcmAppendEncrypt #-} -gcmAppendEncrypt :: GCM -> ByteString -> (ByteString, GCM) -gcmAppendEncrypt gcm input = unsafePerformIO $ allocaFrom gcm doEnc - where len = B.length input - doEnc p = do - output <- create len $ \o -> unsafeUseAsCString input $ \i -> c_aes_gcm_encrypt (castPtr o) p i (fromIntegral len) - ngcm <- peek p - return (output, ngcm) +gcmAppendEncrypt :: AES -> GCM -> ByteString -> (ByteString, GCM) +gcmAppendEncrypt ctx gcm input = unsafePerformIO $ withGCMKeyAndCopySt ctx gcm doEnc + where len = B.length input + doEnc gcmStPtr aesPtr = + create len $ \o -> + unsafeUseAsCString input $ \i -> + c_aes_gcm_encrypt (castPtr o) gcmStPtr aesPtr i (fromIntegral len) -- | append data to decrypt and append to the GCM context -- -- bytestring need to be multiple of AES block size, unless it's the last call to this function. -- need to happen after AAD appending, or after initialization if no AAD data. {-# NOINLINE gcmAppendDecrypt #-} -gcmAppendDecrypt :: GCM -> ByteString -> (ByteString, GCM) -gcmAppendDecrypt gcm input = unsafePerformIO $ allocaFrom gcm doDec - where len = B.length input - doDec p = do - output <- create len $ \o -> unsafeUseAsCString input $ \i -> c_aes_gcm_decrypt (castPtr o) p i (fromIntegral len) - ngcm <- peek p - return (output, ngcm) +gcmAppendDecrypt :: AES -> GCM -> ByteString -> (ByteString, GCM) +gcmAppendDecrypt ctx gcm input = unsafePerformIO $ withGCMKeyAndCopySt ctx gcm doDec + where len = B.length input + doDec gcmStPtr aesPtr = + create len $ \o -> + unsafeUseAsCString input $ \i -> + c_aes_gcm_decrypt (castPtr o) gcmStPtr aesPtr i (fromIntegral len) -- | Generate the Tag from GCM context {-# NOINLINE gcmFinish #-} -gcmFinish :: GCM -> Int -> ByteString -gcmFinish gcm taglen = B.take taglen (unsafeCreate 16 $ \t -> allocaFrom gcm (finish t)) - where finish t p = c_aes_gcm_finish (castPtr t) p +gcmFinish :: AES -> GCM -> Int -> AuthTag +gcmFinish ctx gcm taglen = AuthTag $ B.take taglen computeTag + where computeTag = unsafeCreate 16 $ \t -> + withGCMKeyAndCopySt ctx gcm (c_aes_gcm_finish (castPtr t)) >> return () foreign import ccall "aes.h aes_initkey" - c_aes_init :: Ptr Key -> CString -> CUInt -> IO () + c_aes_init :: Ptr AES -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_encrypt_ecb" - c_aes_encrypt_ecb :: CString -> Ptr Key -> CString -> CUInt -> IO () + c_aes_encrypt_ecb :: CString -> Ptr AES -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_decrypt_ecb" - c_aes_decrypt_ecb :: CString -> Ptr Key -> CString -> CUInt -> IO () + c_aes_decrypt_ecb :: CString -> Ptr AES -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_encrypt_cbc" - c_aes_encrypt_cbc :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO () + c_aes_encrypt_cbc :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_decrypt_cbc" - c_aes_decrypt_cbc :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO () + c_aes_decrypt_cbc :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_encrypt_xts" - c_aes_encrypt_xts :: CString -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO () + c_aes_encrypt_xts :: CString -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_decrypt_xts" - c_aes_decrypt_xts :: CString -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO () + c_aes_decrypt_xts :: CString -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_gen_ctr" - c_aes_gen_ctr :: CString -> Ptr Key -> Ptr IV -> CUInt -> IO () + c_aes_gen_ctr :: CString -> Ptr AES -> Ptr Word8 -> CUInt -> IO () foreign import ccall "aes.h aes_encrypt_ctr" - c_aes_encrypt_ctr :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO () + c_aes_encrypt_ctr :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_gcm_init" - c_aes_gcm_init :: Ptr GCM -> Ptr Key -> Ptr IV -> CUInt -> IO () + c_aes_gcm_init :: Ptr GCM -> Ptr AES -> Ptr Word8 -> CUInt -> IO () foreign import ccall "aes.h aes_gcm_aad" c_aes_gcm_aad :: Ptr GCM -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_gcm_encrypt" - c_aes_gcm_encrypt :: CString -> Ptr GCM -> CString -> CUInt -> IO () + c_aes_gcm_encrypt :: CString -> Ptr GCM -> Ptr AES -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_gcm_decrypt" - c_aes_gcm_decrypt :: CString -> Ptr GCM -> CString -> CUInt -> IO () + c_aes_gcm_decrypt :: CString -> Ptr GCM -> Ptr AES -> CString -> CUInt -> IO () foreign import ccall "aes.h aes_gcm_finish" - c_aes_gcm_finish :: CString -> Ptr GCM -> IO () + c_aes_gcm_finish :: CString -> Ptr GCM -> Ptr AES -> IO () diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2008-2012 Vincent Hanquez +Copyright (c) 2008-2013 Vincent Hanquez All rights reserved. diff --git a/Tests/KATCBC.hs b/Tests/KATCBC.hs --- a/Tests/KATCBC.hs +++ b/Tests/KATCBC.hs @@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} -module KATCBC (vectors_encrypt, vectors_decrypt) where +module KATCBC where import qualified Data.ByteString as B import Data.ByteString.Char8 () diff --git a/Tests/KATECB.hs b/Tests/KATECB.hs --- a/Tests/KATECB.hs +++ b/Tests/KATECB.hs @@ -1,4 +1,4 @@ -module KATECB (vectors_encrypt, vectors_decrypt) where +module KATECB where import qualified Data.ByteString as B diff --git a/Tests/KATGCM.hs b/Tests/KATGCM.hs --- a/Tests/KATGCM.hs +++ b/Tests/KATGCM.hs @@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} -module KATGCM (vectors_encrypt, vectors_decrypt) where +module KATGCM where import qualified Data.ByteString as B import Data.ByteString.Char8 () diff --git a/Tests/KATXTS.hs b/Tests/KATXTS.hs --- a/Tests/KATXTS.hs +++ b/Tests/KATXTS.hs @@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} -module KATXTS (vectors_encrypt, vectors_decrypt) where +module KATXTS where import qualified Data.ByteString as B import Data.ByteString.Char8 () diff --git a/Tests/Tests.hs b/Tests/Tests.hs --- a/Tests/Tests.hs +++ b/Tests/Tests.hs @@ -9,138 +9,52 @@ import Test.QuickCheck import Test.QuickCheck.Test -import Test.Framework.Providers.QuickCheck2 (testProperty) +import Data.Byteable import qualified Data.ByteString as B import qualified Crypto.Cipher.AES as AES +import Crypto.Cipher.Types -- (iv128, IV(..), AuthTag(..), key128, key192, key256) +import Crypto.Cipher.Tests import qualified KATECB import qualified KATCBC import qualified KATXTS import qualified KATGCM -encryptBlock initF encryptF key plaintext = - B.unpack $ encryptF (initF (B.pack key)) plaintext - -katECBTests vectors f = concatMap makeTests vectors - where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..] - where makeTest (AES.initKey -> key,plaintext,expected) = assertEq expected $ f key plaintext - -katCBCTests vectors f = concatMap makeTests vectors - where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..] - where makeTest (AES.initKey -> key,AES.IV -> iv,plaintext,expected) = assertEq expected $ f key iv plaintext - -katXTSTests vectors f = concatMap makeTests vectors - where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..] - where makeTest (AES.initKey -> key1,AES.initKey -> key2, AES.IV -> iv,plaintext,_,expected) = - (assertEq expected $ f (key1,key2) iv 0 plaintext) - -katGCMTests vectors f = concatMap makeTests vectors - where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..] - where makeTest (AES.initKey -> key, AES.IV -> iv, aad, plaintext, expectedOutput, taglen, expectedTag) = - let (output,tag) = f key iv aad plaintext in - assertEq expectedOutput output && (assertEq tag expectedTag) - - -data ECBUnit = ECBUnit B.ByteString B.ByteString - deriving (Show,Eq) -data CBCUnit = CBCUnit B.ByteString B.ByteString B.ByteString - deriving (Show,Eq) -data CTRUnit = CTRUnit B.ByteString B.ByteString B.ByteString - deriving (Show,Eq) -data XTSUnit = XTSUnit B.ByteString B.ByteString B.ByteString B.ByteString - deriving (Show,Eq) -data GCMUnit = GCMUnit B.ByteString B.ByteString B.ByteString B.ByteString - deriving (Show,Eq) -data KeyUnit = KeyUnit B.ByteString - deriving (Show,Eq) - -generateKeyOf size = B.pack <$> replicateM size arbitrary -generateKey = elements [16,24,32] >>= generateKeyOf - -generateIv = B.pack <$> replicateM 16 arbitrary -generateIvGCM = choose (12,90) >>= \sz -> (B.pack <$> replicateM sz arbitrary) - -generatePlaintextMultiple16 = choose (1,128) >>= \size -> replicateM (size*16) arbitrary >>= return . B.pack - -generatePlaintext = choose (0,324) >>= \size -> replicateM size arbitrary >>= return . B.pack - -instance Arbitrary ECBUnit where - arbitrary = ECBUnit <$> generateKey - <*> generatePlaintextMultiple16 - -instance Arbitrary CBCUnit where - arbitrary = CBCUnit <$> generateKey - <*> generateIv - <*> generatePlaintextMultiple16 - -instance Arbitrary CTRUnit where - arbitrary = CTRUnit <$> generateKey - <*> generateIv - <*> generatePlaintext - -instance Arbitrary GCMUnit where - arbitrary = GCMUnit <$> generateKey - <*> generateIvGCM - <*> generatePlaintext - <*> generatePlaintext - -instance Arbitrary XTSUnit where - arbitrary = do - size <- elements [16,32] - XTSUnit <$> generateKeyOf size - <*> generateKeyOf size - <*> generateIv - <*> generatePlaintextMultiple16 - -instance Arbitrary KeyUnit where - arbitrary = KeyUnit <$> generateKey - -idECBTests (ECBUnit (AES.initKey -> key) plaintext) = - plaintext `assertEq` AES.decryptECB key (AES.encryptECB key plaintext) - -idCBCTests (CBCUnit (AES.initKey -> key) (AES.IV -> iv) plaintext) = - plaintext `assertEq` AES.decryptCBC key iv (AES.encryptCBC key iv plaintext) - -idCTRTests (CTRUnit (AES.initKey -> key) (AES.IV -> iv) plaintext) = - plaintext `assertEq` AES.decryptCTR key iv (AES.encryptCTR key iv plaintext) - -idXTSTests (XTSUnit (AES.initKey -> key1) (AES.initKey -> key2) (AES.IV -> iv) plaintext) = - plaintext `assertEq` AES.decryptXTS (key1, key2) iv 0 (AES.encryptXTS (key1, key2) iv 0 plaintext) +toKatECB (k,p,c) = KAT_ECB { ecbKey = k, ecbPlaintext = p, ecbCiphertext = c } +toKatCBC (k,iv,p,c) = KAT_CBC { cbcKey = k, cbcIV = iv, cbcPlaintext = p, cbcCiphertext = c } +toKatXTS (k1,k2,iv,p,_,c) = KAT_XTS { xtsKey1 = k1, xtsKey2 = k2, xtsIV = iv, xtsPlaintext = p, xtsCiphertext = c } +toKatGCM (k,iv,h,p,c,taglen,tag) = + KAT_AEAD { aeadMode = AEAD_GCM + , aeadKey = k + , aeadIV = iv + , aeadHeader = h + , aeadPlaintext = p + , aeadCiphertext = c + , aeadTaglen = taglen + , aeadTag = AuthTag tag + } -idGCMTests (GCMUnit (AES.initKey -> key) (AES.IV -> iv) aad plaintext) = - let (cipherText, tag) = AES.encryptGCM key iv aad plaintext in - let (plaintext2, tag2) = AES.decryptGCM key iv aad cipherText in - (plaintext `assertEq` plaintext2) && (tag == tag2) +kats128 = defaultKATs + { kat_ECB = map toKatECB KATECB.vectors_aes128_enc + , kat_CBC = map toKatCBC KATCBC.vectors_aes128_enc + , kat_XTS = map toKatXTS KATXTS.vectors_aes128_enc + , kat_AEAD = map toKatGCM KATGCM.vectors_aes128_enc + } -idKey (KeyUnit keyBs) = keyBs == AES.keyOfCtx (AES.initKey keyBs) +kats192 = defaultKATs + { kat_ECB = map toKatECB KATECB.vectors_aes192_enc + , kat_CBC = map toKatCBC KATCBC.vectors_aes192_enc + } -assertEq expected got - | expected == got = True - | otherwise = error ("expected: " ++ showhex expected ++ " got: " ++ showhex got) - where showhex = concatMap toHex . B.unpack - toHex b = let (l,r) = b `divMod` 16 in map (toHexChar . fromIntegral) [l,r] - toHexChar c - | c >= 0 && c <= 9 = toEnum (c + fromEnum '0') - | c >= 10 && c <= 16 = toEnum (c + fromEnum 'a') - | otherwise = '_' +kats256 = defaultKATs + { kat_ECB = map toKatECB KATECB.vectors_aes256_enc + , kat_CBC = map toKatCBC KATCBC.vectors_aes256_enc + , kat_XTS = map toKatXTS KATXTS.vectors_aes256_enc + } -tests = - [ testProperty "key-id" idKey - , testGroup "KAT-ECB-Encrypt" $ katECBTests KATECB.vectors_encrypt AES.encryptECB - , testGroup "KAT-ECB-Decrypt" $ katECBTests KATECB.vectors_decrypt AES.decryptECB - , testGroup "KAT-CBC-Encrypt" $ katCBCTests KATCBC.vectors_encrypt AES.encryptCBC - , testGroup "KAT-CBC-Decrypt" $ katCBCTests KATCBC.vectors_decrypt AES.decryptCBC - , testGroup "KAT-XTS-Encrypt" $ katXTSTests KATXTS.vectors_encrypt AES.encryptXTS - , testGroup "KAT-XTS-Decrypt" $ katXTSTests KATXTS.vectors_decrypt AES.decryptXTS - , testGroup "KAT-GCM-Encrypt" $ katGCMTests KATGCM.vectors_encrypt AES.encryptGCM - , testGroup "decrypt-encrypt-is-ID" - [ testProperty "ECB" idECBTests - , testProperty "CBC" idCBCTests - , testProperty "CTR" idCTRTests - , testProperty "XTS" idXTSTests - , testProperty "GCM" idGCMTests - ] +main = defaultMain + [ testBlockCipher kats128 (undefined :: AES.AES128) + , testBlockCipher kats192 (undefined :: AES.AES192) + , testBlockCipher kats256 (undefined :: AES.AES256) ] - -main = defaultMain tests diff --git a/cbits/aes.c b/cbits/aes.c --- a/cbits/aes.c +++ b/cbits/aes.c @@ -38,134 +38,363 @@ #include "gf.h" #include "aes_x86ni.h" -void aes_encrypt_block(aes_block *output, aes_key *key, aes_block *input) -{ -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) - return aes_ni_encrypt_ecb((uint8_t *) output, key, (uint8_t *) input, 1); +void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); +void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); +void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); +void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); +void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length); +void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks); +void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks); +void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length); +void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length); + +enum { + /* init */ + INIT_128, INIT_192, INIT_256, + /* single block */ + ENCRYPT_BLOCK_128, ENCRYPT_BLOCK_192, ENCRYPT_BLOCK_256, + DECRYPT_BLOCK_128, DECRYPT_BLOCK_192, DECRYPT_BLOCK_256, + /* ecb */ + ENCRYPT_ECB_128, ENCRYPT_ECB_192, ENCRYPT_ECB_256, + DECRYPT_ECB_128, DECRYPT_ECB_192, DECRYPT_ECB_256, + /* cbc */ + ENCRYPT_CBC_128, ENCRYPT_CBC_192, ENCRYPT_CBC_256, + DECRYPT_CBC_128, DECRYPT_CBC_192, DECRYPT_CBC_256, + /* ctr */ + ENCRYPT_CTR_128, ENCRYPT_CTR_192, ENCRYPT_CTR_256, + /* xts */ + ENCRYPT_XTS_128, ENCRYPT_XTS_192, ENCRYPT_XTS_256, + DECRYPT_XTS_128, DECRYPT_XTS_192, DECRYPT_XTS_256, + /* xts */ + ENCRYPT_GCM_128, ENCRYPT_GCM_192, ENCRYPT_GCM_256, + DECRYPT_GCM_128, DECRYPT_GCM_192, DECRYPT_GCM_256, +}; + +void *branch_table[] = { + /* INIT */ + [INIT_128] = aes_generic_init, + [INIT_192] = aes_generic_init, + [INIT_256] = aes_generic_init, + /* BLOCK */ + [ENCRYPT_BLOCK_128] = aes_generic_encrypt_block, + [ENCRYPT_BLOCK_192] = aes_generic_encrypt_block, + [ENCRYPT_BLOCK_256] = aes_generic_encrypt_block, + [DECRYPT_BLOCK_128] = aes_generic_decrypt_block, + [DECRYPT_BLOCK_192] = aes_generic_decrypt_block, + [DECRYPT_BLOCK_256] = aes_generic_decrypt_block, + /* ECB */ + [ENCRYPT_ECB_128] = aes_generic_encrypt_ecb, + [ENCRYPT_ECB_192] = aes_generic_encrypt_ecb, + [ENCRYPT_ECB_256] = aes_generic_encrypt_ecb, + [DECRYPT_ECB_128] = aes_generic_decrypt_ecb, + [DECRYPT_ECB_192] = aes_generic_decrypt_ecb, + [DECRYPT_ECB_256] = aes_generic_decrypt_ecb, + /* CBC */ + [ENCRYPT_CBC_128] = aes_generic_encrypt_cbc, + [ENCRYPT_CBC_192] = aes_generic_encrypt_cbc, + [ENCRYPT_CBC_256] = aes_generic_encrypt_cbc, + [DECRYPT_CBC_128] = aes_generic_decrypt_cbc, + [DECRYPT_CBC_192] = aes_generic_decrypt_cbc, + [DECRYPT_CBC_256] = aes_generic_decrypt_cbc, + /* CTR */ + [ENCRYPT_CTR_128] = aes_generic_encrypt_ctr, + [ENCRYPT_CTR_192] = aes_generic_encrypt_ctr, + [ENCRYPT_CTR_256] = aes_generic_encrypt_ctr, + /* XTS */ + [ENCRYPT_XTS_128] = aes_generic_encrypt_xts, + [ENCRYPT_XTS_192] = aes_generic_encrypt_xts, + [ENCRYPT_XTS_256] = aes_generic_encrypt_xts, + [DECRYPT_XTS_128] = aes_generic_decrypt_xts, + [DECRYPT_XTS_192] = aes_generic_decrypt_xts, + [DECRYPT_XTS_256] = aes_generic_decrypt_xts, + /* GCM */ + [ENCRYPT_GCM_128] = aes_generic_gcm_encrypt, + [ENCRYPT_GCM_192] = aes_generic_gcm_encrypt, + [ENCRYPT_GCM_256] = aes_generic_gcm_encrypt, + [DECRYPT_GCM_128] = aes_generic_gcm_decrypt, + [DECRYPT_GCM_192] = aes_generic_gcm_decrypt, + [DECRYPT_GCM_256] = aes_generic_gcm_decrypt, +}; + +typedef void (*init_f)(aes_key *, uint8_t *, uint8_t); +typedef void (*ecb_f)(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); +typedef void (*cbc_f)(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); +typedef void (*ctr_f)(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length); +typedef void (*xts_f)(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks); +typedef void (*gcm_crypt_f)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length); +typedef void (*block_f)(aes_block *output, aes_key *key, aes_block *input); + +#ifdef WITH_AESNI +#define GET_INIT(strength) \ + ((init_f) (branch_table[INIT_128 + strength])) +#define GET_ECB_ENCRYPT(strength) \ + ((ecb_f) (branch_table[ENCRYPT_ECB_128 + strength])) +#define GET_ECB_DECRYPT(strength) \ + ((ecb_f) (branch_table[DECRYPT_ECB_128 + strength])) +#define GET_CBC_ENCRYPT(strength) \ + ((cbc_f) (branch_table[ENCRYPT_CBC_128 + strength])) +#define GET_CBC_DECRYPT(strength) \ + ((cbc_f) (branch_table[DECRYPT_CBC_128 + strength])) +#define GET_CTR_ENCRYPT(strength) \ + ((ctr_f) (branch_table[ENCRYPT_CTR_128 + strength])) +#define GET_XTS_ENCRYPT(strength) \ + ((xts_f) (branch_table[ENCRYPT_XTS_128 + strength])) +#define GET_XTS_DECRYPT(strength) \ + ((xts_f) (branch_table[DECRYPT_XTS_128 + strength])) +#define GET_GCM_ENCRYPT(strength) \ + ((gcm_crypt_f) (branch_table[ENCRYPT_GCM_128 + strength])) +#define GET_GCM_DECRYPT(strength) \ + ((gcm_crypt_f) (branch_table[DECRYPT_GCM_128 + strength])) +#define aes_encrypt_block(o,k,i) \ + (((block_f) (branch_table[ENCRYPT_BLOCK_128 + k->strength]))(o,k,i)) +#define aes_decrypt_block(o,k,i) \ + (((block_f) (branch_table[DECRYPT_BLOCK_128 + k->strength]))(o,k,i)) +#else +#define GET_INIT(strenght) aes_generic_init +#define GET_ECB_ENCRYPT(strength) aes_generic_encrypt_ecb +#define GET_ECB_DECRYPT(strength) aes_generic_decrypt_ecb +#define GET_CBC_ENCRYPT(strength) aes_generic_encrypt_cbc +#define GET_CBC_DECRYPT(strength) aes_generic_decrypt_cbc +#define GET_CTR_ENCRYPT(strength) aes_generic_encrypt_ctr +#define GET_XTS_ENCRYPT(strength) aes_generic_encrypt_xts +#define GET_XTS_DECRYPT(strength) aes_generic_decrypt_xts +#define GET_GCM_ENCRYPT(strength) aes_generic_gcm_encrypt +#define GET_GCM_DECRYPT(strength) aes_generic_gcm_decrypt +#define aes_encrypt_block(o,k,i) aes_generic_encrypt_block(o,k,i) +#define aes_decrypt_block(o,k,i) aes-generic_decrypt_block(o,k,i) #endif - aes_generic_encrypt_block(output, key, input); -} -void aes_decrypt_block(aes_block *output, aes_key *key, aes_block *input) +void initialize_table_ni(int aesni, int pclmul) { -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) - return aes_ni_decrypt_ecb((uint8_t *) output, key, (uint8_t *) input, 1); -#endif - aes_generic_decrypt_block(output, key, input); + if (!aesni) + return; + branch_table[INIT_128] = aes_ni_init; + branch_table[INIT_256] = aes_ni_init; + + branch_table[ENCRYPT_BLOCK_128] = aes_ni_encrypt_block128; + branch_table[DECRYPT_BLOCK_128] = aes_ni_decrypt_block128; + branch_table[ENCRYPT_BLOCK_256] = aes_ni_encrypt_block256; + branch_table[DECRYPT_BLOCK_256] = aes_ni_decrypt_block256; + /* ECB */ + branch_table[ENCRYPT_ECB_128] = aes_ni_encrypt_ecb128; + branch_table[DECRYPT_ECB_128] = aes_ni_decrypt_ecb128; + branch_table[ENCRYPT_ECB_256] = aes_ni_encrypt_ecb256; + branch_table[DECRYPT_ECB_256] = aes_ni_decrypt_ecb256; + /* CBC */ + branch_table[ENCRYPT_CBC_128] = aes_ni_encrypt_cbc128; + branch_table[DECRYPT_CBC_128] = aes_ni_decrypt_cbc128; + branch_table[ENCRYPT_CBC_256] = aes_ni_encrypt_cbc256; + branch_table[DECRYPT_CBC_256] = aes_ni_decrypt_cbc256; + /* CTR */ + branch_table[ENCRYPT_CTR_128] = aes_ni_encrypt_ctr128; + branch_table[ENCRYPT_CTR_256] = aes_ni_encrypt_ctr256; + /* XTS */ + branch_table[ENCRYPT_XTS_128] = aes_ni_encrypt_xts128; + branch_table[ENCRYPT_XTS_256] = aes_ni_encrypt_xts256; + /* GCM */ + branch_table[ENCRYPT_GCM_128] = aes_ni_gcm_encrypt128; + branch_table[ENCRYPT_GCM_256] = aes_ni_gcm_encrypt256; } void aes_initkey(aes_key *key, uint8_t *origkey, uint8_t size) { switch (size) { - case 16: key->nbr = 10; break; - case 24: key->nbr = 12; break; - case 32: key->nbr = 14; break; + case 16: key->nbr = 10; key->strength = 0; break; + case 24: key->nbr = 12; key->strength = 1; break; + case 32: key->nbr = 14; key->strength = 2; break; } #if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && size == 16) - return aes_ni_init(key, origkey, size); + initialize_hw(initialize_table_ni); #endif - aes_generic_init(key, origkey, size); + init_f _init = GET_INIT(key->strength); + _init(key, origkey, size); } -void aes_encrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks) +void aes_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) { - if (!nb_blocks) - return; + ecb_f e = GET_ECB_ENCRYPT(key->strength); + e(output, key, input, nb_blocks); +} -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) - return aes_ni_encrypt_ecb(output, key, input, nb_blocks); -#endif +void aes_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) +{ + ecb_f d = GET_ECB_DECRYPT(key->strength); + d(output, key, input, nb_blocks); +} - for ( ; nb_blocks-- > 0; input += 16, output += 16) { - aes_encrypt_block((block128 *) output, key, (block128 *) input); - } +void aes_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks) +{ + cbc_f e = GET_CBC_ENCRYPT(key->strength); + e(output, key, iv, input, nb_blocks); } -void aes_decrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks) +void aes_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks) { - if (!nb_blocks) - return; + cbc_f d = GET_CBC_DECRYPT(key->strength); + d(output, key, iv, input, nb_blocks); +} -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) - return aes_ni_decrypt_ecb(output, key, input, nb_blocks); -#endif +void aes_gen_ctr(aes_block *output, aes_key *key, aes_block *iv, uint32_t nb_blocks) +{ + aes_block block; - for ( ; nb_blocks-- > 0; input += 16, output += 16) { - aes_decrypt_block((block128 *) output, key, (block128 *) input); + /* preload IV in block */ + block128_copy(&block, iv); + + for ( ; nb_blocks-- > 0; output++, block128_inc_be(&block)) { + aes_encrypt_block(output, key, &block); } } -void aes_encrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks) +void aes_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) { - aes_block block; + ctr_f e = GET_CTR_ENCRYPT(key->strength); + e(output, key, iv, input, len); +} - if (!nb_blocks) - return; -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) - return aes_ni_encrypt_cbc(output, key, (uint8_t *) iv, input, nb_blocks); -#endif +void aes_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks) +{ + xts_f e = GET_XTS_ENCRYPT(k1->strength); + e(output, k1, k2, dataunit, spoint, input, nb_blocks); +} - /* preload IV in block */ - block128_copy(&block, iv); +void aes_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks) +{ + aes_generic_decrypt_xts(output, k1, k2, dataunit, spoint, input, nb_blocks); +} - for ( ; nb_blocks-- > 0; input += 16, output += 16) { - block128_xor(&block, (block128 *) input); +void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) +{ + gcm_crypt_f e = GET_GCM_ENCRYPT(key->strength); + e(output, gcm, key, input, length); +} - aes_encrypt_block(&block, key, &block); +void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) +{ + gcm_crypt_f d = GET_GCM_DECRYPT(key->strength); + d(output, gcm, key, input, length); +} - block128_copy((block128 *) output, &block); - } +static void gcm_ghash_add(aes_gcm *gcm, block128 *b) +{ + block128_xor(&gcm->tag, b); + gf_mul(&gcm->tag, &gcm->h); } -void aes_decrypt_cbc(uint8_t *output, aes_key *key, aes_block *ivini, uint8_t *input, uint32_t nb_blocks) +void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len) { - aes_block block,blocko; - aes_block iv; + gcm->length_aad = 0; + gcm->length_input = 0; - if (!nb_blocks) - return; -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && key->nbr == 10) { - return aes_ni_decrypt_cbc(output, key, (uint8_t *) ivini, input, nb_blocks); + block128_zero(&gcm->h); + block128_zero(&gcm->tag); + block128_zero(&gcm->iv); + + /* prepare H : encrypt_K(0^128) */ + aes_encrypt_block(&gcm->h, key, &gcm->h); + + if (len == 12) { + block128_copy_bytes(&gcm->iv, iv, 12); + gcm->iv.b[15] = 0x01; + } else { + uint32_t origlen = len << 3; + int i; + for (; len >= 16; len -= 16, iv += 16) { + block128_xor(&gcm->iv, (block128 *) iv); + gf_mul(&gcm->iv, &gcm->h); + } + if (len > 0) { + block128_xor_bytes(&gcm->iv, iv, len); + gf_mul(&gcm->iv, &gcm->h); + } + for (i = 15; origlen; --i, origlen >>= 8) + gcm->iv.b[i] ^= (uint8_t) origlen; + gf_mul(&gcm->iv, &gcm->h); } -#endif - /* preload IV in block */ - block128_copy(&iv, ivini); + block128_copy(&gcm->civ, &gcm->iv); +} - aes_decrypt_block(&block, key, &block); +void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length) +{ + gcm->length_aad += length; + for (; length >= 16; input += 16, length -= 16) { + gcm_ghash_add(gcm, (block128 *) input); + } + if (length > 0) { + aes_block tmp; + block128_zero(&tmp); + block128_copy_bytes(&tmp, input, length); + gcm_ghash_add(gcm, &tmp); + } - for ( ;nb_blocks-- > 0; input += 16, output += 16) { - block128_copy(&block, (block128 *) input); +} - aes_decrypt_block(&blocko, key, &block); +void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm, aes_key *key) +{ + aes_block lblock; + int i; - block128_vxor((block128 *) output, &blocko, &iv); - block128_copy(&iv, &block); + /* tag = (tag-1 xor (lenbits(a) | lenbits(c)) ) . H */ + lblock.q[0] = cpu_to_be64(gcm->length_aad << 3); + lblock.q[1] = cpu_to_be64(gcm->length_input << 3); + gcm_ghash_add(gcm, &lblock); + + aes_encrypt_block(&lblock, key, &gcm->iv); + block128_xor(&gcm->tag, &lblock); + + for (i = 0; i < 16; i++) { + tag[i] = gcm->tag.b[i]; } } -void aes_gen_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint32_t nb_blocks) +void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) { + for ( ; nb_blocks-- > 0; input++, output++) { + aes_generic_encrypt_block(output, key, input); + } +} + +void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) +{ + for ( ; nb_blocks-- > 0; input++, output++) { + aes_generic_decrypt_block(output, key, input); + } +} + +void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks) +{ aes_block block; - if (!nb_blocks) - return; /* preload IV in block */ block128_copy(&block, iv); + for ( ; nb_blocks-- > 0; input++, output++) { + block128_xor(&block, (block128 *) input); + aes_generic_encrypt_block(&block, key, &block); + block128_copy((block128 *) output, &block); + } +} - for ( ; nb_blocks-- > 0; output += 16, block128_inc_be(&block)) { - aes_encrypt_block((block128 *) output, key, &block); +void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *ivini, aes_block *input, uint32_t nb_blocks) +{ + aes_block block, blocko; + aes_block iv; + + /* preload IV in block */ + block128_copy(&iv, ivini); + for ( ; nb_blocks-- > 0; input++, output++) { + block128_copy(&block, (block128 *) input); + aes_generic_decrypt_block(&blocko, key, &block); + block128_vxor((block128 *) output, &blocko, &iv); + block128_copy(&iv, &block); } } -void aes_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) +void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) { aes_block block, o; uint32_t nb_blocks = len / 16; @@ -183,27 +412,17 @@ aes_encrypt_block(&o, key, &block); for (i = 0; i < (len % 16); i++) { *output = ((uint8_t *) &o)[i] ^ *input; - output += 1; - input += 1; + output++; + input++; } } } -void aes_encrypt_xts(uint8_t *output, aes_key *k1, aes_key *k2, aes_block *dataunit, - uint32_t spoint, uint8_t *input, uint32_t nb_blocks) +void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks) { aes_block block, tweak; - if (!nb_blocks) - return; - -#if defined(ARCH_X86) && defined(WITH_AESNI) - if (have_aesni() && k1->nbr == 10) { - aes_ni_encrypt_xts(output, k1, k2, (uint8_t *) dataunit, spoint, input, nb_blocks); - return; - } -#endif - /* load IV and encrypt it using k2 as the tweak */ block128_copy(&tweak, dataunit); aes_encrypt_block(&tweak, k2, &tweak); @@ -212,21 +431,18 @@ while (spoint-- > 0) gf_mulx(&tweak); - for ( ; nb_blocks-- > 0; input += 16, output += 16, gf_mulx(&tweak)) { - block128_vxor(&block, (block128 *) input, &tweak); + for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) { + block128_vxor(&block, input, &tweak); aes_encrypt_block(&block, k1, &block); - block128_vxor((block128 *) output, &block, &tweak); + block128_vxor(output, &block, &tweak); } } -void aes_decrypt_xts(uint8_t *output, aes_key *k1, aes_key *k2, aes_block *dataunit, - uint32_t spoint, uint8_t *input, uint32_t nb_blocks) +void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, + uint32_t spoint, aes_block *input, uint32_t nb_blocks) { aes_block block, tweak; - if (!nb_blocks) - return; - /* load IV and encrypt it using k2 as the tweak */ block128_copy(&tweak, dataunit); aes_encrypt_block(&tweak, k2, &tweak); @@ -235,71 +451,14 @@ while (spoint-- > 0) gf_mulx(&tweak); - for ( ; nb_blocks-- > 0; input += 16, output += 16, gf_mulx(&tweak)) { - block128_vxor(&block, (block128 *) input, &tweak); + for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) { + block128_vxor(&block, input, &tweak); aes_decrypt_block(&block, k1, &block); - block128_vxor((block128 *) output, &block, &tweak); + block128_vxor(output, &block, &tweak); } } -static void gcm_ghash_add(aes_gcm *gcm, block128 *b) -{ - block128_xor(&gcm->tag, b); - gf_mul(&gcm->tag, &gcm->h); -} - -void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len) -{ - gcm->length_aad = 0; - gcm->length_input = 0; - - block128_zero(&gcm->h); - block128_zero(&gcm->tag); - block128_zero(&gcm->iv); - - memcpy(&gcm->key, key, sizeof(aes_key)); - - /* prepare H : encrypt_K(0^128) */ - aes_encrypt_block(&gcm->h, key, &gcm->h); - - if (len == 12) { - block128_copy_bytes(&gcm->iv, iv, 12); - gcm->iv.b[15] = 0x01; - } else { - uint32_t origlen = len << 3; - int i; - for (; len >= 16; len -= 16, iv += 16) { - block128_xor(&gcm->iv, (block128 *) iv); - gf_mul(&gcm->iv, &gcm->h); - } - if (len > 0) { - block128_xor_bytes(&gcm->iv, iv, len); - gf_mul(&gcm->iv, &gcm->h); - } - for (i = 15; origlen; --i, origlen >>= 8) - gcm->iv.b[i] ^= (uint8_t) origlen; - gf_mul(&gcm->iv, &gcm->h); - } - - block128_copy(&gcm->civ, &gcm->iv); -} - -void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length) -{ - gcm->length_aad += length; - for (; length >= 16; input += 16, length -= 16) { - gcm_ghash_add(gcm, (block128 *) input); - } - if (length > 0) { - aes_block tmp; - block128_zero(&tmp); - block128_copy_bytes(&tmp, input, length); - gcm_ghash_add(gcm, &tmp); - } - -} - -void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length) +void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { aes_block out; @@ -307,7 +466,7 @@ for (; length >= 16; input += 16, output += 16, length -= 16) { block128_inc_be(&gcm->civ); - aes_encrypt_block(&out, &gcm->key, &gcm->civ); + aes_encrypt_block(&out, key, &gcm->civ); block128_xor(&out, (block128 *) input); gcm_ghash_add(gcm, &out); block128_copy((block128 *) output, &out); @@ -318,11 +477,11 @@ block128_inc_be(&gcm->civ); /* create e(civ) in out */ - aes_encrypt_block(&out, &gcm->key, &gcm->civ); + aes_encrypt_block(&out, key, &gcm->civ); /* initialize a tmp as input and xor it to e(civ) */ block128_zero(&tmp); block128_copy_bytes(&tmp, input, length); - block128_xor_bytes(&tmp, out.b, length); + block128_xor_bytes(&tmp, out.b, length); gcm_ghash_add(gcm, &tmp); @@ -332,7 +491,7 @@ } } -void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length) +void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { aes_block out; @@ -340,7 +499,7 @@ for (; length >= 16; input += 16, output += 16, length -= 16) { block128_inc_be(&gcm->civ); - aes_encrypt_block(&out, &gcm->key, &gcm->civ); + aes_encrypt_block(&out, key, &gcm->civ); gcm_ghash_add(gcm, (block128 *) input); block128_xor(&out, (block128 *) input); block128_copy((block128 *) output, &out); @@ -355,8 +514,8 @@ block128_copy_bytes(&tmp, input, length); gcm_ghash_add(gcm, &tmp); - aes_encrypt_block(&out, &gcm->key, &gcm->civ); - block128_xor_bytes(&tmp, out.b, length); + aes_encrypt_block(&out, key, &gcm->civ); + block128_xor_bytes(&tmp, out.b, length); for (i = 0; i < length; i++) { output[i] = tmp.b[i]; @@ -364,20 +523,3 @@ } } -void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm) -{ - aes_block lblock; - int i; - - /* tag = (tag-1 xor (lenbits(a) | lenbits(c)) ) . H */ - lblock.q[0] = cpu_to_be64(gcm->length_aad << 3); - lblock.q[1] = cpu_to_be64(gcm->length_input << 3); - gcm_ghash_add(gcm, &lblock); - - aes_encrypt_block(&lblock, &gcm->key, &gcm->iv); - block128_xor(&gcm->tag, &lblock); - - for (i = 0; i < 16; i++) { - tag[i] = gcm->tag.b[i]; - } -} diff --git a/cbits/aes.h b/cbits/aes.h --- a/cbits/aes.h +++ b/cbits/aes.h @@ -37,13 +37,15 @@ typedef block128 aes_block; +/* size = 456 */ typedef struct { uint8_t nbr; /* number of rounds: 10 (128), 12 (192), 14 (256) */ - uint8_t _padding[7]; + uint8_t strength; /* 128 = 0, 192 = 1, 256 = 2 */ + uint8_t _padding[6]; uint8_t data[16*14*2]; } aes_key; -/* size = 4*16+2*8+aes_key=456 = 536 */ +/* size = 4*16+2*8= 80 */ typedef struct { aes_block tag; aes_block h; @@ -51,7 +53,6 @@ aes_block civ; uint64_t length_aad; uint64_t length_input; - aes_key key; } aes_gcm; /* in bytes: either 16,24,32 */ @@ -60,22 +61,23 @@ void aes_encrypt(aes_block *output, aes_key *key, aes_block *input); void aes_decrypt(aes_block *output, aes_key *key, aes_block *input); -void aes_encrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks); -void aes_decrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks); +void aes_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); +void aes_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); -void aes_encrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks); -void aes_decrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks); +void aes_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); +void aes_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); -void aes_gen_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint32_t nb_blocks); +void aes_gen_ctr(aes_block *output, aes_key *key, aes_block *iv, uint32_t nb_blocks); -void aes_encrypt_xts(uint8_t *output, aes_key *key, aes_key *key2, aes_block *sector, - uint32_t spoint, uint8_t *input, uint32_t nb_blocks); -void aes_decrypt_xts(uint8_t *output, aes_key *key, aes_key *key2, aes_block *sector, - uint32_t spoint, uint8_t *input, uint32_t nb_blocks); +void aes_encrypt_xts(aes_block *output, aes_key *key, aes_key *key2, aes_block *sector, + uint32_t spoint, aes_block *input, uint32_t nb_blocks); +void aes_decrypt_xts(aes_block *output, aes_key *key, aes_key *key2, aes_block *sector, + uint32_t spoint, aes_block *input, uint32_t nb_blocks); void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len); void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length); -void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length); -void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm); +void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length); +void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length); +void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm, aes_key *key); #endif diff --git a/cbits/aes_x86ni.c b/cbits/aes_x86ni.c --- a/cbits/aes_x86ni.c +++ b/cbits/aes_x86ni.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Vincent Hanquez + * Copyright (c) 2012-2013 Vincent Hanquez * * All rights reserved. * @@ -32,70 +32,154 @@ #include #include +#include #include "aes.h" #include "aes_x86ni.h" +#include "block128.h" #include "cpu.h" #ifdef ARCH_X86 #define ALIGN_UP(addr, size) (((addr) + ((size) - 1)) & (~((size) - 1))) #define ALIGNMENT(n) __attribute__((aligned(n))) -static __m128i aes_128_key_expansion(__m128i key, __m128i keygened) +static __m128i aes_128_key_expansion(__m128i key, __m128i keygened, int shuffle) { - keygened = _mm_shuffle_epi32(keygened, _MM_SHUFFLE(3,3,3,3)); + keygened = _mm_shuffle_epi32(keygened, shuffle); key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); return _mm_xor_si128(key, keygened); } -static void aes_generate_key128(aes_key *key, uint8_t *ikey) +void aes_ni_init(aes_key *key, uint8_t *ikey, uint8_t size) { - __m128i k[20]; + __m128i k[28]; uint64_t *out = (uint64_t *) key->data; int i; - k[0] = _mm_loadu_si128((const __m128i*) ikey); + switch (size) { + case 16: + k[0] = _mm_loadu_si128((const __m128i*) ikey); -#define AES_128_key_exp(K, RCON) aes_128_key_expansion(K, _mm_aeskeygenassist_si128(K, RCON)) - k[1] = AES_128_key_exp(k[0], 0x01); - k[2] = AES_128_key_exp(k[1], 0x02); - k[3] = AES_128_key_exp(k[2], 0x04); - k[4] = AES_128_key_exp(k[3], 0x08); - k[5] = AES_128_key_exp(k[4], 0x10); - k[6] = AES_128_key_exp(k[5], 0x20); - k[7] = AES_128_key_exp(k[6], 0x40); - k[8] = AES_128_key_exp(k[7], 0x80); - k[9] = AES_128_key_exp(k[8], 0x1B); - k[10] = AES_128_key_exp(k[9], 0x36); + #define AES_128_key_exp(K, RCON) aes_128_key_expansion(K, _mm_aeskeygenassist_si128(K, RCON), 0xff) + k[1] = AES_128_key_exp(k[0], 0x01); + k[2] = AES_128_key_exp(k[1], 0x02); + k[3] = AES_128_key_exp(k[2], 0x04); + k[4] = AES_128_key_exp(k[3], 0x08); + k[5] = AES_128_key_exp(k[4], 0x10); + k[6] = AES_128_key_exp(k[5], 0x20); + k[7] = AES_128_key_exp(k[6], 0x40); + k[8] = AES_128_key_exp(k[7], 0x80); + k[9] = AES_128_key_exp(k[8], 0x1B); + k[10] = AES_128_key_exp(k[9], 0x36); - /* generate decryption keys in reverse order. - * k[10] is shared by last encryption and first decryption rounds - * k[20] is shared by first encryption round (and is the original user key) */ - k[11] = _mm_aesimc_si128(k[9]); - k[12] = _mm_aesimc_si128(k[8]); - k[13] = _mm_aesimc_si128(k[7]); - k[14] = _mm_aesimc_si128(k[6]); - k[15] = _mm_aesimc_si128(k[5]); - k[16] = _mm_aesimc_si128(k[4]); - k[17] = _mm_aesimc_si128(k[3]); - k[18] = _mm_aesimc_si128(k[2]); - k[19] = _mm_aesimc_si128(k[1]); + /* generate decryption keys in reverse order. + * k[10] is shared by last encryption and first decryption rounds + * k[20] is shared by first encryption round (and is the original user key) */ + k[11] = _mm_aesimc_si128(k[9]); + k[12] = _mm_aesimc_si128(k[8]); + k[13] = _mm_aesimc_si128(k[7]); + k[14] = _mm_aesimc_si128(k[6]); + k[15] = _mm_aesimc_si128(k[5]); + k[16] = _mm_aesimc_si128(k[4]); + k[17] = _mm_aesimc_si128(k[3]); + k[18] = _mm_aesimc_si128(k[2]); + k[19] = _mm_aesimc_si128(k[1]); - for (i = 0; i < 20; i++) - _mm_storeu_si128(((__m128i *) out) + i, k[i]); + for (i = 0; i < 20; i++) + _mm_storeu_si128(((__m128i *) out) + i, k[i]); + break; + case 32: +#define AES_256_key_exp_1(K1, K2, RCON) aes_128_key_expansion(K1, _mm_aeskeygenassist_si128(K2, RCON), 0xff) +#define AES_256_key_exp_2(K1, K2) aes_128_key_expansion(K1, _mm_aeskeygenassist_si128(K2, 0x00), 0xaa) + k[0] = _mm_loadu_si128((const __m128i*) ikey); + k[1] = _mm_loadu_si128((const __m128i*) (ikey+16)); + k[2] = AES_256_key_exp_1(k[0], k[1], 0x01); + k[3] = AES_256_key_exp_2(k[1], k[2]); + k[4] = AES_256_key_exp_1(k[2], k[3], 0x02); + k[5] = AES_256_key_exp_2(k[3], k[4]); + k[6] = AES_256_key_exp_1(k[4], k[5], 0x04); + k[7] = AES_256_key_exp_2(k[5], k[6]); + k[8] = AES_256_key_exp_1(k[6], k[7], 0x08); + k[9] = AES_256_key_exp_2(k[7], k[8]); + k[10] = AES_256_key_exp_1(k[8], k[9], 0x10); + k[11] = AES_256_key_exp_2(k[9], k[10]); + k[12] = AES_256_key_exp_1(k[10], k[11], 0x20); + k[13] = AES_256_key_exp_2(k[11], k[12]); + k[14] = AES_256_key_exp_1(k[12], k[13], 0x40); + + k[15] = _mm_aesimc_si128(k[13]); + k[16] = _mm_aesimc_si128(k[12]); + k[17] = _mm_aesimc_si128(k[11]); + k[18] = _mm_aesimc_si128(k[10]); + k[19] = _mm_aesimc_si128(k[9]); + k[20] = _mm_aesimc_si128(k[8]); + k[21] = _mm_aesimc_si128(k[7]); + k[22] = _mm_aesimc_si128(k[6]); + k[23] = _mm_aesimc_si128(k[5]); + k[24] = _mm_aesimc_si128(k[4]); + k[25] = _mm_aesimc_si128(k[3]); + k[26] = _mm_aesimc_si128(k[2]); + k[27] = _mm_aesimc_si128(k[1]); + for (i = 0; i < 28; i++) + _mm_storeu_si128(((__m128i *) out) + i, k[i]); + break; + default: + break; + } } -void aes_ni_init(aes_key *key, uint8_t *origkey, uint8_t size) +/* TO OPTIMISE: use pcmulqdq... or some faster code. + * this is the lamest way of doing it, but i'm out of time. + * this is basically a copy of gf_mulx in gf.c */ +static __m128i gfmulx(__m128i v) { - switch (size) { - case 16: aes_generate_key128(key, origkey); break; - default: break; - } + uint64_t v_[2] ALIGNMENT(16); + const uint64_t gf_mask = 0x8000000000000000; + + _mm_store_si128((__m128i *) v_, v); + uint64_t r = ((v_[1] & gf_mask) ? 0x87 : 0); + v_[1] = (v_[1] << 1) | (v_[0] & gf_mask ? 1 : 0); + v_[0] = (v_[0] << 1) ^ r; + v = _mm_load_si128((__m128i *) v_); + return v; } +static void unopt_gf_mul(block128 *a, block128 *b) +{ + uint64_t a0, a1, v0, v1; + int i, j; -#define PRELOAD_ENC_KEYS(k) \ + a0 = a1 = 0; + v0 = cpu_to_be64(a->q[0]); + v1 = cpu_to_be64(a->q[1]); + + for (i = 0; i < 16; i++) + for (j = 0x80; j != 0; j >>= 1) { + uint8_t x = b->b[i] & j; + a0 ^= x ? v0 : 0; + a1 ^= x ? v1 : 0; + x = (uint8_t) v1 & 1; + v1 = (v1 >> 1) | (v0 << 63); + v0 = (v0 >> 1) ^ (x ? (0xe1ULL << 56) : 0); + } + a->q[0] = cpu_to_be64(a0); + a->q[1] = cpu_to_be64(a1); +} + +static __m128i ghash_add(__m128i tag, __m128i h, __m128i m) +{ + aes_block _t, _h; + tag = _mm_xor_si128(tag, m); + + _mm_store_si128((__m128i *) &_t, tag); + _mm_store_si128((__m128i *) &_h, h); + unopt_gf_mul(&_t, &_h); + tag = _mm_load_si128((__m128i *) &_t); + return tag; +} + +#define PRELOAD_ENC_KEYS128(k) \ __m128i K0 = _mm_loadu_si128(((__m128i *) k)+0); \ __m128i K1 = _mm_loadu_si128(((__m128i *) k)+1); \ __m128i K2 = _mm_loadu_si128(((__m128i *) k)+2); \ @@ -108,7 +192,14 @@ __m128i K9 = _mm_loadu_si128(((__m128i *) k)+9); \ __m128i K10 = _mm_loadu_si128(((__m128i *) k)+10); -#define DO_ENC_BLOCK(m) \ +#define PRELOAD_ENC_KEYS256(k) \ + PRELOAD_ENC_KEYS128(k) \ + __m128i K11 = _mm_loadu_si128(((__m128i *) k)+11); \ + __m128i K12 = _mm_loadu_si128(((__m128i *) k)+12); \ + __m128i K13 = _mm_loadu_si128(((__m128i *) k)+13); \ + __m128i K14 = _mm_loadu_si128(((__m128i *) k)+14); + +#define DO_ENC_BLOCK128(m) \ m = _mm_xor_si128(m, K0); \ m = _mm_aesenc_si128(m, K1); \ m = _mm_aesenc_si128(m, K2); \ @@ -121,20 +212,49 @@ m = _mm_aesenc_si128(m, K9); \ m = _mm_aesenclast_si128(m, K10); -#define PRELOAD_DEC_KEYS(k) \ - __m128i K0 = _mm_loadu_si128(((__m128i *) k)+10+0); \ - __m128i K1 = _mm_loadu_si128(((__m128i *) k)+10+1); \ - __m128i K2 = _mm_loadu_si128(((__m128i *) k)+10+2); \ - __m128i K3 = _mm_loadu_si128(((__m128i *) k)+10+3); \ - __m128i K4 = _mm_loadu_si128(((__m128i *) k)+10+4); \ - __m128i K5 = _mm_loadu_si128(((__m128i *) k)+10+5); \ - __m128i K6 = _mm_loadu_si128(((__m128i *) k)+10+6); \ - __m128i K7 = _mm_loadu_si128(((__m128i *) k)+10+7); \ - __m128i K8 = _mm_loadu_si128(((__m128i *) k)+10+8); \ - __m128i K9 = _mm_loadu_si128(((__m128i *) k)+10+9); \ +#define DO_ENC_BLOCK256(m) \ + m = _mm_xor_si128(m, K0); \ + m = _mm_aesenc_si128(m, K1); \ + m = _mm_aesenc_si128(m, K2); \ + m = _mm_aesenc_si128(m, K3); \ + m = _mm_aesenc_si128(m, K4); \ + m = _mm_aesenc_si128(m, K5); \ + m = _mm_aesenc_si128(m, K6); \ + m = _mm_aesenc_si128(m, K7); \ + m = _mm_aesenc_si128(m, K8); \ + m = _mm_aesenc_si128(m, K9); \ + m = _mm_aesenc_si128(m, K10); \ + m = _mm_aesenc_si128(m, K11); \ + m = _mm_aesenc_si128(m, K12); \ + m = _mm_aesenc_si128(m, K13); \ + m = _mm_aesenclast_si128(m, K14); + +/* load K0 at K9 from index 'at' */ +#define PRELOAD_DEC_KEYS_AT(k, at) \ + __m128i K0 = _mm_loadu_si128(((__m128i *) k)+at+0); \ + __m128i K1 = _mm_loadu_si128(((__m128i *) k)+at+1); \ + __m128i K2 = _mm_loadu_si128(((__m128i *) k)+at+2); \ + __m128i K3 = _mm_loadu_si128(((__m128i *) k)+at+3); \ + __m128i K4 = _mm_loadu_si128(((__m128i *) k)+at+4); \ + __m128i K5 = _mm_loadu_si128(((__m128i *) k)+at+5); \ + __m128i K6 = _mm_loadu_si128(((__m128i *) k)+at+6); \ + __m128i K7 = _mm_loadu_si128(((__m128i *) k)+at+7); \ + __m128i K8 = _mm_loadu_si128(((__m128i *) k)+at+8); \ + __m128i K9 = _mm_loadu_si128(((__m128i *) k)+at+9); \ + +#define PRELOAD_DEC_KEYS128(k) \ + PRELOAD_DEC_KEYS_AT(k, 10) \ __m128i K10 = _mm_loadu_si128(((__m128i *) k)+0); -#define DO_DEC_BLOCK(m) \ +#define PRELOAD_DEC_KEYS256(k) \ + PRELOAD_DEC_KEYS_AT(k, 14) \ + __m128i K10 = _mm_loadu_si128(((__m128i *) k)+14+10); \ + __m128i K11 = _mm_loadu_si128(((__m128i *) k)+14+11); \ + __m128i K12 = _mm_loadu_si128(((__m128i *) k)+14+12); \ + __m128i K13 = _mm_loadu_si128(((__m128i *) k)+14+13); \ + __m128i K14 = _mm_loadu_si128(((__m128i *) k)+0); + +#define DO_DEC_BLOCK128(m) \ m = _mm_xor_si128(m, K0); \ m = _mm_aesdec_si128(m, K1); \ m = _mm_aesdec_si128(m, K2); \ @@ -147,128 +267,52 @@ m = _mm_aesdec_si128(m, K9); \ m = _mm_aesdeclast_si128(m, K10); -void aes_ni_encrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks) -{ - __m128i *k = (__m128i *) key->data; - - PRELOAD_ENC_KEYS(k); - - while (blocks-- > 0) { - __m128i m = _mm_loadu_si128((__m128i *) in); - - DO_ENC_BLOCK(m); - - _mm_storeu_si128((__m128i *) out, m); - in += 16; - out += 16; - } -} - -void aes_ni_decrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks) -{ - __m128i *k = (__m128i *) key->data; - - PRELOAD_DEC_KEYS(k); - - while (blocks-- > 0) { - __m128i m = _mm_loadu_si128((__m128i *) in); - - DO_DEC_BLOCK(m); - - _mm_storeu_si128((__m128i *) out, m); - in += 16; - out += 16; - } -} - -void aes_ni_encrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks) -{ - __m128i *k = (__m128i *) key->data; - __m128i iv = _mm_loadu_si128((__m128i *) _iv); - - PRELOAD_ENC_KEYS(k); - - while (blocks-- > 0) { - __m128i m = _mm_loadu_si128((__m128i *) in); - m = _mm_xor_si128(m, iv); - - DO_ENC_BLOCK(m); - - _mm_storeu_si128((__m128i *) out, m); - iv = m; - - in += 16; - out += 16; - } -} - -void aes_ni_decrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks) -{ - __m128i *k = (__m128i *) key->data; - __m128i iv = _mm_loadu_si128((__m128i *) _iv); - - PRELOAD_DEC_KEYS(k); - - while (blocks-- > 0) { - __m128i m = _mm_loadu_si128((__m128i *) in); - __m128i ivnext = m; - - DO_DEC_BLOCK(m); - m = _mm_xor_si128(m, iv); - - _mm_storeu_si128((__m128i *) out, m); - iv = ivnext; - - in += 16; - out += 16; - } -} - -/* TO OPTIMISE: use pcmulqdq... or some faster code. - * this is the lamest way of doing it, but i'm out of time. - * this is basically a copy of gf_mulx in gf.c */ -static __m128i gfmulx(__m128i v) -{ - uint64_t v_[2] ALIGNMENT(16); - const uint64_t gf_mask = 0x8000000000000000; - - _mm_store_si128((__m128i *) v_, v); - uint64_t r = ((v_[1] & gf_mask) ? 0x87 : 0); - v_[1] = (v_[1] << 1) | (v_[0] & gf_mask ? 1 : 0); - v_[0] = (v_[0] << 1) ^ r; - v = _mm_load_si128((__m128i *) v_); - return v; -} - -void aes_ni_encrypt_xts(uint8_t *out, aes_key *key1, aes_key *key2, - uint8_t *_tweak, uint32_t spoint, uint8_t *in, uint32_t blocks) -{ - __m128i tweak = _mm_loadu_si128((__m128i *) _tweak); - - do { - __m128i *k2 = (__m128i *) key2->data; - PRELOAD_ENC_KEYS(k2); - DO_ENC_BLOCK(tweak); - - while (spoint-- > 0) - tweak = gfmulx(tweak); - } while (0) ; +#define DO_DEC_BLOCK256(m) \ + m = _mm_xor_si128(m, K0); \ + m = _mm_aesdec_si128(m, K1); \ + m = _mm_aesdec_si128(m, K2); \ + m = _mm_aesdec_si128(m, K3); \ + m = _mm_aesdec_si128(m, K4); \ + m = _mm_aesdec_si128(m, K5); \ + m = _mm_aesdec_si128(m, K6); \ + m = _mm_aesdec_si128(m, K7); \ + m = _mm_aesdec_si128(m, K8); \ + m = _mm_aesdec_si128(m, K9); \ + m = _mm_aesdec_si128(m, K10); \ + m = _mm_aesdec_si128(m, K11); \ + m = _mm_aesdec_si128(m, K12); \ + m = _mm_aesdec_si128(m, K13); \ + m = _mm_aesdeclast_si128(m, K14); - do { - __m128i *k1 = (__m128i *) key1->data; - PRELOAD_ENC_KEYS(k1); +#define SIZE 128 +#define SIZED(m) m##128 +#define PRELOAD_ENC PRELOAD_ENC_KEYS128 +#define DO_ENC_BLOCK DO_ENC_BLOCK128 +#define PRELOAD_DEC PRELOAD_DEC_KEYS128 +#define DO_DEC_BLOCK DO_DEC_BLOCK128 +#include "aes_x86ni_impl.c" - for ( ; blocks-- > 0; in += 16, out += 16, tweak = gfmulx(tweak)) { - __m128i m = _mm_loadu_si128((__m128i *) in); +#undef SIZE +#undef SIZED +#undef PRELOAD_ENC +#undef PRELOAD_DEC +#undef DO_ENC_BLOCK +#undef DO_DEC_BLOCK - m = _mm_xor_si128(m, tweak); - DO_ENC_BLOCK(m); - m = _mm_xor_si128(m, tweak); +#define SIZED(m) m##256 +#define SIZE 256 +#define PRELOAD_ENC PRELOAD_ENC_KEYS256 +#define DO_ENC_BLOCK DO_ENC_BLOCK256 +#define PRELOAD_DEC PRELOAD_DEC_KEYS256 +#define DO_DEC_BLOCK DO_DEC_BLOCK256 +#include "aes_x86ni_impl.c" - _mm_storeu_si128((__m128i *) out, m); - } - } while (0); -} +#undef SIZE +#undef SIZED +#undef PRELOAD_ENC +#undef PRELOAD_DEC +#undef DO_ENC_BLOCK +#undef DO_DEC_BLOCK #endif diff --git a/cbits/aes_x86ni.h b/cbits/aes_x86ni.h --- a/cbits/aes_x86ni.h +++ b/cbits/aes_x86ni.h @@ -38,14 +38,39 @@ #include #include #include "aes.h" +#include "block128.h" +#ifdef IMPL_DEBUG +static void block128_sse_print(__m128i m) +{ + block128 b; + _mm_storeu_si128((__m128i *) &b.b, m); + block128_print(&b); +} +#endif + void aes_ni_init(aes_key *key, uint8_t *origkey, uint8_t size); -void aes_ni_encrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks); -void aes_ni_decrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks); -void aes_ni_encrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks); -void aes_ni_decrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks); -void aes_ni_encrypt_xts(uint8_t *out, aes_key *key1, aes_key *key2, - uint8_t *_tweak, uint32_t spoint, uint8_t *in, uint32_t blocks); +void aes_ni_encrypt_block128(aes_block *out, aes_key *key, aes_block *in); +void aes_ni_encrypt_block256(aes_block *out, aes_key *key, aes_block *in); +void aes_ni_decrypt_block128(aes_block *out, aes_key *key, aes_block *in); +void aes_ni_decrypt_block256(aes_block *out, aes_key *key, aes_block *in); +void aes_ni_encrypt_ecb128(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks); +void aes_ni_encrypt_ecb256(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks); +void aes_ni_decrypt_ecb128(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks); +void aes_ni_decrypt_ecb256(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks); +void aes_ni_encrypt_cbc128(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks); +void aes_ni_encrypt_cbc256(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks); +void aes_ni_decrypt_cbc128(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks); +void aes_ni_decrypt_cbc256(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks); +void aes_ni_encrypt_ctr128(uint8_t *out, aes_key *key, aes_block *_iv, uint8_t *in, uint32_t length); +void aes_ni_encrypt_ctr256(uint8_t *out, aes_key *key, aes_block *_iv, uint8_t *in, uint32_t length); +void aes_ni_encrypt_xts128(aes_block *out, aes_key *key1, aes_key *key2, + aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks); +void aes_ni_encrypt_xts256(aes_block *out, aes_key *key1, aes_key *key2, + aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks); + +void aes_ni_gcm_encrypt128(uint8_t *out, aes_gcm *gcm, aes_key *key, uint8_t *in, uint32_t length); +void aes_ni_gcm_encrypt256(uint8_t *out, aes_gcm *gcm, aes_key *key, uint8_t *in, uint32_t length); void gf_mul_x86ni(block128 *res, block128 *a_, block128 *b_); diff --git a/cbits/aes_x86ni_impl.c b/cbits/aes_x86ni_impl.c new file mode 100644 --- /dev/null +++ b/cbits/aes_x86ni_impl.c @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2012-2013 Vincent Hanquez + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of his contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +void SIZED(aes_ni_encrypt_block)(aes_block *out, aes_key *key, aes_block *in) +{ + __m128i *k = (__m128i *) key->data; + PRELOAD_ENC(k); + __m128i m = _mm_loadu_si128((__m128i *) in); + DO_ENC_BLOCK(m); + _mm_storeu_si128((__m128i *) out, m); +} + +void SIZED(aes_ni_decrypt_block)(aes_block *out, aes_key *key, aes_block *in) +{ + __m128i *k = (__m128i *) key->data; + PRELOAD_DEC(k); + __m128i m = _mm_loadu_si128((__m128i *) in); + DO_DEC_BLOCK(m); + _mm_storeu_si128((__m128i *) out, m); +} + +void SIZED(aes_ni_encrypt_ecb)(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks) +{ + __m128i *k = (__m128i *) key->data; + + PRELOAD_ENC(k); + for (; blocks-- > 0; in += 1, out += 1) { + __m128i m = _mm_loadu_si128((__m128i *) in); + DO_ENC_BLOCK(m); + _mm_storeu_si128((__m128i *) out, m); + } +} + +void SIZED(aes_ni_decrypt_ecb)(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks) +{ + __m128i *k = (__m128i *) key->data; + + PRELOAD_DEC(k); + + for (; blocks-- > 0; in += 1, out += 1) { + __m128i m = _mm_loadu_si128((__m128i *) in); + DO_DEC_BLOCK(m); + _mm_storeu_si128((__m128i *) out, m); + } +} + +void SIZED(aes_ni_encrypt_cbc)(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks) +{ + __m128i *k = (__m128i *) key->data; + __m128i iv = _mm_loadu_si128((__m128i *) _iv); + + PRELOAD_ENC(k); + + for (; blocks-- > 0; in += 1, out += 1) { + __m128i m = _mm_loadu_si128((__m128i *) in); + m = _mm_xor_si128(m, iv); + DO_ENC_BLOCK(m); + iv = m; + _mm_storeu_si128((__m128i *) out, m); + } +} + +void SIZED(aes_ni_decrypt_cbc)(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks) +{ + __m128i *k = (__m128i *) key->data; + __m128i iv = _mm_loadu_si128((__m128i *) _iv); + + PRELOAD_DEC(k); + + for (; blocks-- > 0; in += 1, out += 1) { + __m128i m = _mm_loadu_si128((__m128i *) in); + __m128i ivnext = m; + + DO_DEC_BLOCK(m); + m = _mm_xor_si128(m, iv); + + _mm_storeu_si128((__m128i *) out, m); + iv = ivnext; + } +} + +void SIZED(aes_ni_encrypt_ctr)(uint8_t *output, aes_key *key, aes_block *_iv, uint8_t *input, uint32_t len) +{ + __m128i *k = (__m128i *) key->data; + __m128i bswap_mask = _mm_setr_epi8(7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8); + __m128i one = _mm_set_epi32(0,1,0,0); + uint32_t nb_blocks = len / 16; + uint32_t part_block_len = len % 16; + + /* get the IV in little endian format */ + __m128i iv = _mm_loadu_si128((__m128i *) _iv); + iv = _mm_shuffle_epi8(iv, bswap_mask); + + PRELOAD_ENC(k); + + for (; nb_blocks-- > 0; output += 16, input += 16) { + /* put back the iv in big endian mode, + * encrypt it and and xor it the input block + */ + __m128i tmp = _mm_shuffle_epi8(iv, bswap_mask); + DO_ENC_BLOCK(tmp); + __m128i m = _mm_loadu_si128((__m128i *) input); + m = _mm_xor_si128(m, tmp); + + _mm_storeu_si128((__m128i *) output, m); + /* iv += 1 */ + iv = _mm_add_epi64(iv, one); + } + + if (part_block_len != 0) { + aes_block block; + memset(&block.b, 0, 16); + memcpy(&block.b, input, part_block_len); + + __m128i m = _mm_loadu_si128((__m128i *) &block); + __m128i tmp = _mm_shuffle_epi8(iv, bswap_mask); + + DO_ENC_BLOCK(tmp); + m = _mm_xor_si128(m, tmp); + _mm_storeu_si128((__m128i *) &block.b, m); + memcpy(output, &block.b, part_block_len); + } + + return ; +} + +void SIZED(aes_ni_encrypt_xts)(aes_block *out, aes_key *key1, aes_key *key2, + aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks) +{ + __m128i tweak = _mm_loadu_si128((__m128i *) _tweak); + + do { + __m128i *k2 = (__m128i *) key2->data; + PRELOAD_ENC(k2); + DO_ENC_BLOCK(tweak); + + while (spoint-- > 0) + tweak = gfmulx(tweak); + } while (0) ; + + do { + __m128i *k1 = (__m128i *) key1->data; + PRELOAD_ENC(k1); + + for ( ; blocks-- > 0; in += 1, out += 1, tweak = gfmulx(tweak)) { + __m128i m = _mm_loadu_si128((__m128i *) in); + + m = _mm_xor_si128(m, tweak); + DO_ENC_BLOCK(m); + m = _mm_xor_si128(m, tweak); + + _mm_storeu_si128((__m128i *) out, m); + } + } while (0); +} + +void SIZED(aes_ni_gcm_encrypt)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) +{ + __m128i *k = (__m128i *) key->data; + __m128i bswap_mask = _mm_setr_epi8(7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8); + __m128i one = _mm_set_epi32(0,1,0,0); + uint32_t nb_blocks = length / 16; + uint32_t part_block_len = length % 16; + + gcm->length_input += length; + + __m128i h = _mm_loadu_si128((__m128i *) &gcm->h); + __m128i tag = _mm_loadu_si128((__m128i *) &gcm->tag); + __m128i iv = _mm_loadu_si128((__m128i *) &gcm->civ); + iv = _mm_shuffle_epi8(iv, bswap_mask); + + PRELOAD_ENC(k); + + for (; nb_blocks-- > 0; output += 16, input += 16) { + /* iv += 1 */ + iv = _mm_add_epi64(iv, one); + + /* put back iv in big endian, encrypt it, + * and xor it to input */ + __m128i tmp = _mm_shuffle_epi8(iv, bswap_mask); + DO_ENC_BLOCK(tmp); + __m128i m = _mm_loadu_si128((__m128i *) input); + m = _mm_xor_si128(m, tmp); + + tag = ghash_add(tag, h, m); + + /* store it out */ + _mm_storeu_si128((__m128i *) output, m); + } + if (part_block_len > 0) { + __m128i mask; + aes_block block; + /* FIXME could do something a bit more clever (slli & sub & and maybe) ... */ + switch (part_block_len) { + case 1: mask = _mm_setr_epi8(0,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 2: mask = _mm_setr_epi8(0,1,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 3: mask = _mm_setr_epi8(0,1,2,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 4: mask = _mm_setr_epi8(0,1,2,3,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 5: mask = _mm_setr_epi8(0,1,2,3,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 6: mask = _mm_setr_epi8(0,1,2,3,4,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 7: mask = _mm_setr_epi8(0,1,2,3,4,5,6,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 8: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 9: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 10: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,0x80,0x80,0x80,0x80,0x80,0x80); break; + case 11: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,0x80,0x80,0x80,0x80,0x80); break; + case 12: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,0x80,0x80,0x80,0x80); break; + case 13: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,0x80,0x80,0x80); break; + case 14: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,0x80,0x80); break; + case 15: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0x80); break; + default: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); break; + } + memset(&block.b, 0, 16); + memcpy(&block.b, input, part_block_len); + + /* iv += 1 */ + iv = _mm_add_epi64(iv, one); + + /* put back iv in big endian mode, encrypt it and xor it with input */ + __m128i tmp = _mm_shuffle_epi8(iv, bswap_mask); + DO_ENC_BLOCK(tmp); + + __m128i m = _mm_loadu_si128((__m128i *) &block); + m = _mm_xor_si128(m, tmp); + m = _mm_shuffle_epi8(m, mask); + + tag = ghash_add(tag, h, m); + + /* make output */ + _mm_storeu_si128((__m128i *) &block.b, m); + memcpy(output, &block.b, part_block_len); + } + /* store back IV & tag */ + __m128i tmp = _mm_shuffle_epi8(iv, bswap_mask); + _mm_storeu_si128((__m128i *) &gcm->civ, tmp); + _mm_storeu_si128((__m128i *) &gcm->tag, tag); +} diff --git a/cbits/block128.h b/cbits/block128.h --- a/cbits/block128.h +++ b/cbits/block128.h @@ -84,4 +84,16 @@ b->q[1] = cpu_to_be64(v); } +#ifdef IMPL_DEBUG +#include +static inline void block128_print(block128 *b) +{ + int i; + for (i = 0; i < 16; i++) { + printf("%02x ", b->b[i]); + } + printf("\n"); +} +#endif + #endif diff --git a/cbits/cpu.c b/cbits/cpu.c --- a/cbits/cpu.c +++ b/cbits/cpu.c @@ -54,18 +54,22 @@ } #ifdef USE_AESNI -int have_aesni(void) +void initialize_hw(void (*init_table)(int, int)) { - static int v = -1; - if (v == -1) { + static int inited = 0; + if (inited == 0) { uint32_t eax, ebx, ecx, edx; + int aesni, pclmul; + + inited = 1; cpuid(1, &eax, &ebx, &ecx, &edx); - v = (ecx & 0x02000000); + aesni = (ecx & 0x02000000); + pclmul = (ecx & 0x00000001); + init_table(aesni, pclmul); } - return v; } #else -#define have_aesni() (0) +#define initialize_hw(init_table) (0) #endif #endif diff --git a/cbits/cpu.h b/cbits/cpu.h --- a/cbits/cpu.h +++ b/cbits/cpu.h @@ -37,9 +37,9 @@ #endif #ifdef USE_AESNI -int have_aesni(void); +void initialize_hw(void (*init_table)(int, int)); #else -#define have_aesni() (0) +#define initialize_hw(init_table) (0) #endif #endif diff --git a/cipher-aes.cabal b/cipher-aes.cabal --- a/cipher-aes.cabal +++ b/cipher-aes.cabal @@ -1,5 +1,5 @@ Name: cipher-aes -Version: 0.1.8 +Version: 0.2.0 Description: Fast AES cipher implementation with advanced mode of operations. . @@ -12,7 +12,7 @@ . The software implementation uses S-Boxes, which might suffer for cache timing issues. However do notes that most other known software implementations, including very popular - one (openssl, gnutls) also uses same implementation. If it matters for your + one (openssl, gnutls) also uses similar implementation. If it matters for your case, you should make sure you have AES-NI available, or you'll need to use a different implementation. . @@ -28,12 +28,16 @@ Cabal-Version: >=1.8 Extra-Source-Files: Tests/*.hs cbits/*.h + cbits/aes_x86ni_impl.c Library Build-Depends: base >= 4 && < 5 , bytestring + , byteable + , crypto-cipher-types + , securemem >= 0.1.2 Exposed-modules: Crypto.Cipher.AES - ghc-options: -Wall + ghc-options: -Wall -optc-O3 -fno-cse -fwarn-tabs C-sources: cbits/aes_generic.c cbits/aes.c cbits/gf.c @@ -48,7 +52,10 @@ Main-Is: Tests.hs Build-depends: base >= 4 && < 5 , cipher-aes + , crypto-cipher-types + , crypto-cipher-tests , bytestring + , byteable , QuickCheck >= 2 , test-framework >= 0.3.3 , test-framework-quickcheck2 >= 0.2.9 @@ -60,6 +67,8 @@ Build-depends: base >= 4 && < 5 , bytestring , cipher-aes + , crypto-cipher-types + , crypto-cipher-benchmarks , criterion , mtl