cipher-aes 0.1.8 → 0.2.0

raw patch · 17 files changed

+1128/−746 lines, 17 filesdep +byteabledep +crypto-cipher-benchmarksdep +crypto-cipher-testsPVP ok

version bump matches the API change (PVP)

Dependencies added: byteable, crypto-cipher-benchmarks, crypto-cipher-tests, crypto-cipher-types, securemem

API changes (from Hackage documentation)

Files

• Benchmarks/Benchmarks.hs +6/−101
• Crypto/Cipher/AES.hs +199/−137
• LICENSE +1/−1
• Tests/KATCBC.hs +1/−1
• Tests/KATECB.hs +1/−1
• Tests/KATGCM.hs +1/−1
• Tests/KATXTS.hs +1/−1
• Tests/Tests.hs +35/−121
• cbits/aes.c +325/−183
• cbits/aes.h +16/−14
• cbits/aes_x86ni.c +212/−168
• cbits/aes_x86ni.h +31/−6
• cbits/aes_x86ni_impl.c +263/−0
• cbits/block128.h +12/−0
• cbits/cpu.c +10/−6
• cbits/cpu.h +2/−2
• cipher-aes.cabal +12/−3

@@ -1,102 +1,7 @@-{-# LANGUAGE CPP #-}-import Criterion-import Criterion.Environment-import Criterion.Config-import Criterion.Monad-import Criterion.Analysis-import Criterion.Measurement--import Text.Printf--import Control.Monad.Trans--import qualified Data.ByteString as B-import qualified Data.ByteString.Lazy as L--import qualified Crypto.Cipher.AES as AES--key128 = AES.initKey $ B.replicate 16 0-key192 = AES.initKey $ B.replicate 24 0-key256 = AES.initKey $ B.replicate 32 0--nullIV = AES.IV $ B.replicate 16 0-nullIVGCM = AES.IV $ B.replicate 12 0--aesEncrypt128 = AES.encryptECB key128-aesEncrypt128CBC = AES.encryptCBC key128 nullIV-aesEncrypt128CTR = AES.encryptCTR key128 nullIV-aesEncrypt128XTS = AES.encryptXTS (key128,key128) nullIV 0-aesEncrypt128GCM = fst . AES.encryptGCM key128 nullIVGCM B.empty--aesEncrypt192 = AES.encryptECB key192-aesEncrypt192CBC = AES.encryptCBC key192 nullIV-aesEncrypt192CTR = AES.encryptCTR key192 nullIV-aesEncrypt192GCM = fst . AES.encryptGCM key192 nullIVGCM B.empty-aesEncrypt256 = AES.encryptECB key256-aesEncrypt256CBC = AES.encryptCBC key256 nullIV-aesEncrypt256CTR = AES.encryptCTR key256 nullIV-aesEncrypt256XTS = AES.encryptXTS (key256,key256) nullIV 0-aesEncrypt256GCM = fst . AES.encryptGCM key256 nullIVGCM B.empty--b16 f   = whnf f $ B.replicate 16 0-b32 f   = whnf f $ B.replicate 32 0-b128 f  = whnf f $ B.replicate 128 0-b512 f  = whnf f $ B.replicate 512 0-b1024 f = whnf f $ B.replicate 1024 0-b4096 f = whnf f $ B.replicate 4096 0-b16384 f = whnf f $ B.replicate 16384 0--doCipher env f = do-	mean16   <- runBenchmark env (b16 f)   >>= \sample -> analyseMean sample 100-	mean32   <- runBenchmark env (b32 f)   >>= \sample -> analyseMean sample 100-	mean128  <- runBenchmark env (b128 f)  >>= \sample -> analyseMean sample 100-	mean512  <- runBenchmark env (b512 f)  >>= \sample -> analyseMean sample 100-	mean1024 <- runBenchmark env (b1024 f) >>= \sample -> analyseMean sample 100-	mean4096 <- runBenchmark env (b4096 f) >>= \sample -> analyseMean sample 100-	mean16384 <- runBenchmark env (b16384 f) >>= \sample -> analyseMean sample 100-	return (mean16, mean32, mean128, mean512, mean1024, mean4096, mean16384)--norm :: Int -> Double -> Double-norm n time-	| n < 1024  = 1.0 / (time * (1024 / fromIntegral n))-	| n == 1024 = 1.0 / time-	| n > 1024  = 1.0 / (time / (fromIntegral n / 1024))--pn :: Int -> Double -> String-pn n time-    | val > (10 * 1024) = printf "%.1f M/s" (val / 1024)-    | otherwise         = printf "%.1f K/s" val-    where val = norm n time--doOne env (cipherName, f) = do-	(mean16, mean32, mean128, mean512, mean1024, mean4096, mean16384) <- doCipher env f-	let s = printf "%12s: %12s %12s %12s %12s %12s %12s %12s\n              %12s %12s %12s %12s %12s %12s %12s"-	               cipherName-	               (secs mean16) (secs mean32) (secs mean128)-	               (secs mean512) (secs mean1024) (secs mean4096) (secs mean16384)-	               (pn 16 mean16) (pn 32 mean32) (pn 128 mean128)-	               (pn 512 mean512) (pn 1024 mean1024) (pn 4096 mean4096) (pn 16384 mean16384)-	return s+import Crypto.Cipher.Benchmarks+import Crypto.Cipher.AES (AES128, AES192, AES256) -main = withConfig defaultConfig $ do-	env <- measureEnvironment-	l   <- mapM (doOne env)-		[ ("AES128"     , aesEncrypt128)-		, ("AES128-CBC" , aesEncrypt128CBC)-		, ("AES128-CTR" , aesEncrypt128CTR)-		, ("AES128-XTS" , aesEncrypt128XTS)-		, ("AES128-GCM" , aesEncrypt128GCM)-		, ("AES192"     , aesEncrypt192)-		, ("AES192-CBC" , aesEncrypt192CBC)-		, ("AES192-CTR" , aesEncrypt192CTR)-		, ("AES192-GCM" , aesEncrypt192GCM)-		, ("AES256"     , aesEncrypt256)-		, ("AES256-CBC" , aesEncrypt256CBC)-		, ("AES256-CTR" , aesEncrypt256CTR)-		, ("AES256-XTS" , aesEncrypt256XTS)-		, ("AES256-GCM" , aesEncrypt256GCM)-		]-	liftIO $ printf "%12s| %12s %12s %12s %12s %12s %12s %12s\n"-	                "cipher" "16 bytes" "32 bytes" "64 bytes" "512 bytes" "1024 bytes" "4096 bytes" "16384 bytes"-	liftIO $ printf "===================================================================================================\n"-	mapM_ (liftIO . putStrLn) l+main = defaultMain+    [GBlockCipher (undefined :: AES128)+    ,GBlockCipher (undefined :: AES192)+    ,GBlockCipher (undefined :: AES256)]

@@ -1,5 +1,7 @@ {-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE ViewPatterns #-}+{-# LANGUAGE MultiParamTypeClasses #-}+{-# LANGUAGE CPP #-} -- | -- Module      : Crypto.Cipher.AES -- License     : BSD-style@@ -10,12 +12,14 @@ module Crypto.Cipher.AES     (     -- * data types-      Key-    , IV(..)+      AES+    , AES128+    , AES192+    , AES256      -- * creation+    , initAES     , initKey-    , keyOfCtx      -- * misc     , genCTR@@ -35,82 +39,121 @@     , decryptGCM     ) where --- import Data.ByteString (ByteString) import Data.Word import Foreign.Ptr-import Foreign.ForeignPtr-import Foreign.Storable import Foreign.C.Types import Foreign.C.String-import Foreign.Marshal.Alloc import Data.ByteString.Internal import Data.ByteString.Unsafe+import Data.Byteable import qualified Data.ByteString as B import System.IO.Unsafe (unsafePerformIO) --- | AES Key-newtype Key = Key ByteString+import Crypto.Cipher.Types+import Data.SecureMem --- | AES IV-newtype IV = IV ByteString+-- | AES Context (pre-processed key)+newtype AES = AES SecureMem --- | GCM Context-newtype GCM = GCM ByteString+newtype AES128 = AES128 AES+newtype AES192 = AES192 AES+newtype AES256 = AES256 AES -sizeGCM :: Int-sizeGCM = 540+instance Cipher AES128 where+    cipherName    _ = "AES128"+    cipherKeySize _ = Just 16+    cipherInit k    = AES128 $ initAES k -instance Storable GCM where-    sizeOf _    = sizeGCM-    alignment _ = 16-    poke ptr (GCM b) = unsafeUseAsCString b (\cs -> memcpy (castPtr ptr) (castPtr cs) (fromIntegral sizeGCM))-    peek ptr         = create sizeGCM (\bptr -> memcpy bptr (castPtr ptr) (fromIntegral sizeGCM)) >>= return . GCM+instance Cipher AES192 where+    cipherName    _ = "AES192"+    cipherKeySize _ = Just 24+    cipherInit k    = AES192 $ initAES k -keyToPtr :: Key -> (Ptr Key -> IO a) -> IO a-keyToPtr (Key b) f = unsafeUseAsCString b (f . castPtr)+instance Cipher AES256 where+    cipherName    _ = "AES256"+    cipherKeySize _ = Just 32+    cipherInit k    = AES256 $ initAES k -ivToPtr :: IV -> (Ptr IV -> IO a) -> IO a-ivToPtr (IV b) f = unsafeUseAsCString b (f . castPtr)+#define INSTANCE_BLOCKCIPHER(CSTR) \+instance BlockCipher CSTR where \+    { blockSize _ = 16 \+    ; ecbEncrypt (CSTR aes) = encryptECB aes \+    ; ecbDecrypt (CSTR aes) = decryptECB aes \+    ; cbcEncrypt (CSTR aes) = encryptCBC aes \+    ; cbcDecrypt (CSTR aes) = decryptCBC aes \+    ; ctrCombine (CSTR aes) = encryptCTR aes \+    ; xtsEncrypt (CSTR aes1, CSTR aes2) = encryptXTS (aes1,aes2) \+    ; xtsDecrypt (CSTR aes1, CSTR aes2) = decryptXTS (aes1,aes2) \+    ; aeadInit AEAD_GCM cipher@(CSTR aes) iv = Just $ AEAD cipher $ AEADState $ gcmInit aes iv \+    ; aeadInit _        _                  _ = Nothing \+    }; \+\+instance AEADModeImpl CSTR GCM where \+    { aeadStateAppendHeader (CSTR _) gcmState bs = gcmAppendAAD gcmState bs \+    ; aeadStateEncrypt (CSTR aes) gcmState input = gcmAppendEncrypt aes gcmState input \+    ; aeadStateDecrypt (CSTR aes) gcmState input = gcmAppendDecrypt aes gcmState input \+    ; aeadStateFinalize (CSTR aes) gcmState len  = gcmFinish aes gcmState len \+    } -withKeyAndIV :: Key -> IV -> (Ptr Key -> Ptr IV -> IO a) -> IO a-withKeyAndIV key iv f = keyToPtr key $ \kptr -> ivToPtr iv $ \ivp -> f kptr ivp+INSTANCE_BLOCKCIPHER(AES128)+INSTANCE_BLOCKCIPHER(AES192)+INSTANCE_BLOCKCIPHER(AES256) -withKey2AndIV :: Key -> Key -> IV -> (Ptr Key -> Ptr Key -> Ptr IV -> IO a) -> IO a+-- | GCM State+newtype GCM = GCM SecureMem++sizeGCM :: Int+sizeGCM = 80++keyToPtr :: AES -> (Ptr AES -> IO a) -> IO a+keyToPtr (AES b) f = withSecureMemPtr b (f . castPtr)++ivToPtr :: Byteable iv => iv -> (Ptr Word8 -> IO a) -> IO a+ivToPtr iv f = withBytePtr iv (f . castPtr)++withKeyAndIV :: Byteable iv => AES -> iv -> (Ptr AES -> Ptr Word8 -> IO a) -> IO a+withKeyAndIV ctx iv f = keyToPtr ctx $ \kptr -> ivToPtr iv $ \ivp -> f kptr ivp++withKey2AndIV :: Byteable iv => AES -> AES -> iv -> (Ptr AES -> Ptr AES -> Ptr Word8 -> IO a) -> IO a withKey2AndIV key1 key2 iv f =     keyToPtr key1 $ \kptr1 -> keyToPtr key2 $ \kptr2 -> ivToPtr iv $ \ivp -> f kptr1 kptr2 ivp +withGCMKeyAndCopySt :: AES -> GCM -> (Ptr GCM -> Ptr AES -> IO a) -> IO (a, GCM)+withGCMKeyAndCopySt aes (GCM gcmSt) f =+    keyToPtr aes $ \aesPtr -> do+        newSt <- secureMemCopy gcmSt+        a     <- withSecureMemPtr newSt $ \gcmStPtr -> f (castPtr gcmStPtr) aesPtr+        return (a, GCM newSt)++withNewGCMSt :: GCM -> (Ptr GCM -> IO ()) -> IO GCM+withNewGCMSt (GCM gcmSt) f = withSecureMemCopy gcmSt (f . castPtr) >>= \sm2 -> return (GCM sm2)+ -- | initialize key-{-# NOINLINE initKey #-}-initKey :: ByteString -> Key-initKey b@(B.length -> len)-    | len == 16 = doInit 10-    | len == 24 = doInit 12-    | len == 32 = doInit 14-    | otherwise = error "wrong key size: need to be 16, 24 or 32 bytes."-      where doInit nbR = unsafePerformIO $ unsafeUseAsCString b (allocAndFill nbR)-            allocAndFill nbR ikey = do-                ptr <- mallocBytes (16+2*2*16*nbR)-                c_aes_init ptr (castPtr ikey) (fromIntegral len)-                fptr <- newForeignPtr c_free_finalizer (castPtr ptr)-                return $ Key $ fromForeignPtr fptr 0 (16+2*2*16*nbR)+--+-- rounds need to be 10 / 12 / 14. any other values will cause undefined behavior+initAES :: Byteable b => b -> AES+initAES k+    | len == 16 = initWithRounds 10+    | len == 24 = initWithRounds 12+    | len == 32 = initWithRounds 14+    | otherwise = error "not a valid key length"+  where len = byteableLength k+        initWithRounds nbR = AES $ unsafeCreateSecureMem (16+2*2*16*nbR) aesInit+        aesInit ptr = withBytePtr k $ \ikey ->+            c_aes_init (castPtr ptr) (castPtr ikey) (fromIntegral len) --- | return the user key from the Key context-keyOfCtx :: Key -> ByteString-keyOfCtx (Key bs) = B.take sz (B.drop 8 bs)-    where nbRound            = unsafeHead $ B.take 1 bs-          sz | nbRound == 10 = 16-             | nbRound == 12 = 24-             | nbRound == 14 = 32-             | otherwise     = error "not a valid key"+{-# DEPRECATED initKey "use initAES" #-}+initKey :: Byteable b => b -> AES+initKey = initAES  -- | encrypt using Electronic Code Book (ECB) {-# NOINLINE encryptECB #-}-encryptECB :: Key -> ByteString -> ByteString+encryptECB :: AES -> ByteString -> ByteString encryptECB = doECB c_aes_encrypt_ecb  -- | encrypt using Cipher Block Chaining (CBC) {-# NOINLINE encryptCBC #-}-encryptCBC :: Key -> IV -> ByteString -> ByteString+encryptCBC :: Byteable iv => AES -> iv -> ByteString -> ByteString encryptCBC = doCBC c_aes_encrypt_cbc  -- | generate a counter mode pad. this is generally xor-ed to an input@@ -120,36 +163,46 @@ -- more data will be returned, so that the returned bytestring is -- a multiple of the block cipher size. {-# NOINLINE genCTR #-}-genCTR :: Key        -- ^ Cipher Key.-       -> IV         -- ^ usually a 128 bit integer.-       -> Int        -- ^ length of bytes required.+genCTR :: Byteable iv+       => AES -- ^ Cipher Key.+       -> iv  -- ^ usually a 128 bit integer.+       -> Int -- ^ length of bytes required.        -> ByteString-genCTR key iv len = unsafeCreate (nbBlocks * 16) generate-    where-          generate o = withKeyAndIV key iv $ \k i -> c_aes_gen_ctr (castPtr o) k i (fromIntegral nbBlocks)-          (nbBlocks',r) = len `divMod` 16-          nbBlocks = if r == 0 then nbBlocks' else nbBlocks' + 1+genCTR ctx iv len+    | len <= 0  = B.empty+    | otherwise = unsafeCreate (nbBlocks * 16) generate+  where generate o = withKeyAndIV ctx iv $ \k i -> c_aes_gen_ctr (castPtr o) k i (fromIntegral nbBlocks)+        (nbBlocks',r) = len `quotRem` 16+        nbBlocks = if r == 0 then nbBlocks' else nbBlocks' + 1  -- | encrypt using Counter mode (CTR) -- -- in CTR mode encryption and decryption is the same operation. {-# NOINLINE encryptCTR #-}-encryptCTR :: Key -> IV -> ByteString -> ByteString-encryptCTR key iv input = unsafeCreate len doEncrypt-    where doEncrypt o = withKeyAndIV key iv $ \k v -> unsafeUseAsCString input $ \i ->-                            c_aes_encrypt_ctr (castPtr o) k v i (fromIntegral len)-          len = B.length input+encryptCTR :: Byteable iv+           => AES+           -> iv+           -> ByteString+           -> ByteString+encryptCTR ctx iv input+    | len <= 0  = B.empty+    | otherwise = unsafeCreate len doEncrypt+  where doEncrypt o = withKeyAndIV ctx iv $ \k v -> unsafeUseAsCString input $ \i ->+                      c_aes_encrypt_ctr (castPtr o) k v i (fromIntegral len)+        len = B.length input  -- | encrypt using Galois counter mode (GCM) -- return the encrypted bytestring and the tag associated -- -- note: encrypted data is identical to CTR mode in GCM, however -- a tag is also computed.-encryptGCM :: Key        -- ^ Key-           -> IV         -- ^ initial vector+{-# NOINLINE encryptGCM #-}+encryptGCM :: Byteable iv+           => AES        -- ^ Key+           -> iv         -- ^ initial vector            -> ByteString -- ^ data to authenticate (AAD)            -> ByteString -- ^ data to encrypt-           -> (ByteString, ByteString) -- ^ ciphertext and tag+           -> (ByteString, AuthTag) -- ^ ciphertext and tag encryptGCM = doGCM gcmAppendEncrypt  -- | encrypt using XTS@@ -157,164 +210,173 @@ -- the first key is the normal block encryption key -- the second key is used for the initial block tweak {-# NOINLINE encryptXTS #-}-encryptXTS :: (Key,Key) -> IV -> Word32 -> ByteString -> ByteString+encryptXTS :: Byteable iv => (AES,AES) -> iv -> Word32 -> ByteString -> ByteString encryptXTS = doXTS c_aes_encrypt_xts  -- | decrypt using Electronic Code Book (ECB) {-# NOINLINE decryptECB #-}-decryptECB :: Key -> ByteString -> ByteString+decryptECB :: AES -> ByteString -> ByteString decryptECB = doECB c_aes_decrypt_ecb  -- | decrypt using Cipher block chaining (CBC) {-# NOINLINE decryptCBC #-}-decryptCBC :: Key -> IV -> ByteString -> ByteString+decryptCBC :: Byteable iv => AES -> iv -> ByteString -> ByteString decryptCBC = doCBC c_aes_decrypt_cbc  -- | decrypt using Counter mode (CTR). -- -- in CTR mode encryption and decryption is the same operation.-decryptCTR :: Key -> IV -> ByteString -> ByteString+decryptCTR :: Byteable iv => AES -> iv -> ByteString -> ByteString decryptCTR = encryptCTR  -- | decrypt using XTS {-# NOINLINE decryptXTS #-}-decryptXTS :: (Key,Key) -> IV -> Word32 -> ByteString -> ByteString+decryptXTS :: Byteable iv => (AES,AES) -> iv -> Word32 -> ByteString -> ByteString decryptXTS = doXTS c_aes_decrypt_xts  -- | decrypt using Galois Counter Mode (GCM) {-# NOINLINE decryptGCM #-}-decryptGCM :: Key -> IV -> ByteString -> ByteString -> (ByteString, ByteString)+decryptGCM :: Byteable iv => AES -> iv -> ByteString -> ByteString -> (ByteString, AuthTag) decryptGCM = doGCM gcmAppendDecrypt  {-# INLINE doECB #-}-doECB :: (Ptr b -> Ptr Key -> CString -> CUInt -> IO ())-      -> Key -> ByteString -> ByteString-doECB f key input+doECB :: (Ptr b -> Ptr AES -> CString -> CUInt -> IO ())+      -> AES -> ByteString -> ByteString+doECB f ctx input     | r /= 0    = error "cannot use with non multiple of block size"-    | otherwise = unsafeCreate len $ \o -> keyToPtr key $ \k -> unsafeUseAsCString input $ \i ->-            f (castPtr o) k i (fromIntegral nbBlocks)-    where (nbBlocks, r) = len `divMod` 16-          len           = (B.length input)+    | otherwise = unsafeCreate len $ \o ->+                  keyToPtr ctx $ \k ->+                  unsafeUseAsCString input $ \i ->+                  f (castPtr o) k i (fromIntegral nbBlocks)+  where (nbBlocks, r) = len `quotRem` 16+        len           = (B.length input)   {-# INLINE doCBC #-}-doCBC :: (Ptr b -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO ())-      -> Key -> IV -> ByteString -> ByteString-doCBC f key iv input+doCBC :: Byteable iv+      => (Ptr b -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO ())+      -> AES -> iv -> ByteString -> ByteString+doCBC f ctx iv input+    | len == 0  = B.empty     | r /= 0    = error "cannot use with non multiple of block size"-    | otherwise = unsafeCreate len $ \o -> withKeyAndIV key iv $ \k v -> unsafeUseAsCString input $ \i ->-            f (castPtr o) k v i (fromIntegral nbBlocks)-    where (nbBlocks, r) = len `divMod` 16-          len           = (B.length input)+    | otherwise = unsafeCreate len $ \o ->+                  withKeyAndIV ctx iv $ \k v ->+                  unsafeUseAsCString input $ \i ->+                  f (castPtr o) k v i (fromIntegral nbBlocks)+  where (nbBlocks, r) = len `quotRem` 16+        len           = B.length input  {-# INLINE doXTS #-}-doXTS :: (Ptr b -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO ())-      -> (Key, Key) -> IV -> Word32 -> ByteString -> ByteString+doXTS :: Byteable iv+      => (Ptr b -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO ())+      -> (AES, AES) -> iv -> Word32 -> ByteString -> ByteString doXTS f (key1,key2) iv spoint input+    | len == 0  = B.empty     | r /= 0    = error "cannot use with non multiple of block size (yet)"     | otherwise = unsafeCreate len $ \o -> withKey2AndIV key1 key2 iv $ \k1 k2 v -> unsafeUseAsCString input $ \i ->             f (castPtr o) k1 k2 v (fromIntegral spoint) i (fromIntegral nbBlocks)-    where (nbBlocks, r) = len `divMod` 16-          len           = (B.length input)+  where (nbBlocks, r) = len `quotRem` 16+        len           = B.length input  {-# INLINE doGCM #-}-doGCM :: (GCM -> ByteString -> (ByteString, GCM)) -> Key -> IV -> ByteString -> ByteString -> (ByteString, ByteString)-doGCM f key iv aad input = (cipher, tag)-    where-          tag             = gcmFinish after 16-          (cipher, after) = f afterAAD input-          afterAAD        = gcmAppendAAD ini aad-          ini             = gcmInit key iv--allocaFrom :: Storable a => a -> (Ptr a -> IO b) -> IO b-allocaFrom z f = alloca $ \ptr -> poke ptr z >> f ptr+doGCM :: Byteable iv => (AES -> GCM -> ByteString -> (ByteString, GCM)) -> AES -> iv -> ByteString -> ByteString -> (ByteString, AuthTag)+doGCM f ctx iv aad input = (output, tag)+  where tag             = gcmFinish ctx after 16+        (output, after) = f ctx afterAAD input+        afterAAD        = gcmAppendAAD ini aad+        ini             = gcmInit ctx iv  -- | initialize a gcm context {-# NOINLINE gcmInit #-}-gcmInit :: Key -> IV -> GCM-gcmInit key iv@(IV b) = unsafePerformIO $ alloca doInit-    where doInit gcm = withKeyAndIV key iv (\k v -> c_aes_gcm_init gcm k v (fromIntegral $ B.length b)) >> peek gcm+gcmInit :: Byteable iv => AES -> iv -> GCM+gcmInit ctx iv = unsafePerformIO $ do+    sm <- createSecureMem sizeGCM $ \gcmStPtr ->+            withKeyAndIV ctx iv $ \k v ->+            c_aes_gcm_init (castPtr gcmStPtr) k v (fromIntegral $ byteableLength iv)+    return $ GCM sm  -- | append data which is going to just be authentified to the GCM context. -- -- need to happen after initialization and before appending encryption/decryption data. {-# NOINLINE gcmAppendAAD #-} gcmAppendAAD :: GCM -> ByteString -> GCM-gcmAppendAAD gcm input = unsafePerformIO $ allocaFrom gcm doAppend-    where doAppend p = do-                unsafeUseAsCString input $ \i -> c_aes_gcm_aad p i (fromIntegral $ B.length input) -                peek p+gcmAppendAAD gcmSt input = unsafePerformIO doAppend+  where doAppend =+            withNewGCMSt gcmSt $ \gcmStPtr ->+            unsafeUseAsCString input $ \i ->+            c_aes_gcm_aad gcmStPtr i (fromIntegral $ B.length input)  -- | append data to encrypt and append to the GCM context -- -- bytestring need to be multiple of AES block size, unless it's the last call to this function. -- need to happen after AAD appending, or after initialization if no AAD data. {-# NOINLINE gcmAppendEncrypt #-}-gcmAppendEncrypt :: GCM -> ByteString -> (ByteString, GCM)-gcmAppendEncrypt gcm input = unsafePerformIO $ allocaFrom gcm doEnc-    where len = B.length input-          doEnc p = do-                output <- create len $ \o -> unsafeUseAsCString input $ \i -> c_aes_gcm_encrypt (castPtr o) p i (fromIntegral len)-                ngcm   <- peek p-                return (output, ngcm)+gcmAppendEncrypt :: AES -> GCM -> ByteString -> (ByteString, GCM)+gcmAppendEncrypt ctx gcm input = unsafePerformIO $ withGCMKeyAndCopySt ctx gcm doEnc+  where len = B.length input+        doEnc gcmStPtr aesPtr =+            create len $ \o ->+            unsafeUseAsCString input $ \i ->+            c_aes_gcm_encrypt (castPtr o) gcmStPtr aesPtr i (fromIntegral len)  -- | append data to decrypt and append to the GCM context -- -- bytestring need to be multiple of AES block size, unless it's the last call to this function. -- need to happen after AAD appending, or after initialization if no AAD data. {-# NOINLINE gcmAppendDecrypt #-}-gcmAppendDecrypt :: GCM -> ByteString -> (ByteString, GCM)-gcmAppendDecrypt gcm input = unsafePerformIO $ allocaFrom gcm doDec-    where len = B.length input-          doDec p = do-                output <- create len $ \o -> unsafeUseAsCString input $ \i -> c_aes_gcm_decrypt (castPtr o) p i (fromIntegral len)-                ngcm   <- peek p-                return (output, ngcm)+gcmAppendDecrypt :: AES -> GCM -> ByteString -> (ByteString, GCM)+gcmAppendDecrypt ctx gcm input = unsafePerformIO $ withGCMKeyAndCopySt ctx gcm doDec+  where len = B.length input+        doDec gcmStPtr aesPtr =+            create len $ \o ->+            unsafeUseAsCString input $ \i ->+            c_aes_gcm_decrypt (castPtr o) gcmStPtr aesPtr i (fromIntegral len)  -- | Generate the Tag from GCM context {-# NOINLINE gcmFinish #-}-gcmFinish :: GCM -> Int -> ByteString-gcmFinish gcm taglen = B.take taglen (unsafeCreate 16 $ \t -> allocaFrom gcm (finish t))-    where finish t p = c_aes_gcm_finish (castPtr t) p+gcmFinish :: AES -> GCM -> Int -> AuthTag+gcmFinish ctx gcm taglen = AuthTag $ B.take taglen computeTag+  where computeTag = unsafeCreate 16 $ \t ->+                        withGCMKeyAndCopySt ctx gcm (c_aes_gcm_finish (castPtr t)) >> return ()  foreign import ccall "aes.h aes_initkey"-    c_aes_init :: Ptr Key -> CString -> CUInt -> IO ()+    c_aes_init :: Ptr AES -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_encrypt_ecb"-    c_aes_encrypt_ecb :: CString -> Ptr Key -> CString -> CUInt -> IO ()+    c_aes_encrypt_ecb :: CString -> Ptr AES -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_decrypt_ecb"-    c_aes_decrypt_ecb :: CString -> Ptr Key -> CString -> CUInt -> IO ()+    c_aes_decrypt_ecb :: CString -> Ptr AES -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_encrypt_cbc"-    c_aes_encrypt_cbc :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO ()+    c_aes_encrypt_cbc :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_decrypt_cbc"-    c_aes_decrypt_cbc :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO ()+    c_aes_decrypt_cbc :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_encrypt_xts"-    c_aes_encrypt_xts :: CString -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO ()+    c_aes_encrypt_xts :: CString -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_decrypt_xts"-    c_aes_decrypt_xts :: CString -> Ptr Key -> Ptr Key -> Ptr IV -> CUInt -> CString -> CUInt -> IO ()+    c_aes_decrypt_xts :: CString -> Ptr AES -> Ptr AES -> Ptr Word8 -> CUInt -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_gen_ctr"-    c_aes_gen_ctr :: CString -> Ptr Key -> Ptr IV -> CUInt -> IO ()+    c_aes_gen_ctr :: CString -> Ptr AES -> Ptr Word8 -> CUInt -> IO ()  foreign import ccall "aes.h aes_encrypt_ctr"-    c_aes_encrypt_ctr :: CString -> Ptr Key -> Ptr IV -> CString -> CUInt -> IO ()+    c_aes_encrypt_ctr :: CString -> Ptr AES -> Ptr Word8 -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_gcm_init"-    c_aes_gcm_init :: Ptr GCM -> Ptr Key -> Ptr IV -> CUInt -> IO ()+    c_aes_gcm_init :: Ptr GCM -> Ptr AES -> Ptr Word8 -> CUInt -> IO ()  foreign import ccall "aes.h aes_gcm_aad"     c_aes_gcm_aad :: Ptr GCM -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_gcm_encrypt"-    c_aes_gcm_encrypt :: CString -> Ptr GCM -> CString -> CUInt -> IO ()+    c_aes_gcm_encrypt :: CString -> Ptr GCM -> Ptr AES -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_gcm_decrypt"-    c_aes_gcm_decrypt :: CString -> Ptr GCM -> CString -> CUInt -> IO ()+    c_aes_gcm_decrypt :: CString -> Ptr GCM -> Ptr AES -> CString -> CUInt -> IO ()  foreign import ccall "aes.h aes_gcm_finish"-    c_aes_gcm_finish :: CString -> Ptr GCM -> IO ()+    c_aes_gcm_finish :: CString -> Ptr GCM -> Ptr AES -> IO ()

@@ -1,4 +1,4 @@-Copyright (c) 2008-2012 Vincent Hanquez <vincent@snarc.org>+Copyright (c) 2008-2013 Vincent Hanquez <vincent@snarc.org>  All rights reserved. 

@@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}-module KATCBC (vectors_encrypt, vectors_decrypt) where+module KATCBC where  import qualified Data.ByteString as B import Data.ByteString.Char8 ()

@@ -1,4 +1,4 @@-module KATECB (vectors_encrypt, vectors_decrypt) where+module KATECB where  import qualified Data.ByteString as B 

@@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}-module KATGCM (vectors_encrypt, vectors_decrypt) where+module KATGCM where  import qualified Data.ByteString as B import Data.ByteString.Char8 ()

@@ -1,5 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}-module KATXTS (vectors_encrypt, vectors_decrypt) where+module KATXTS where  import qualified Data.ByteString as B import Data.ByteString.Char8 ()

@@ -9,138 +9,52 @@  import Test.QuickCheck import Test.QuickCheck.Test-import Test.Framework.Providers.QuickCheck2 (testProperty) +import Data.Byteable import qualified Data.ByteString as B import qualified Crypto.Cipher.AES as AES+import Crypto.Cipher.Types -- (iv128, IV(..), AuthTag(..), key128, key192, key256)+import Crypto.Cipher.Tests  import qualified KATECB import qualified KATCBC import qualified KATXTS import qualified KATGCM -encryptBlock initF encryptF key plaintext =-    B.unpack $ encryptF (initF (B.pack key)) plaintext--katECBTests vectors f = concatMap makeTests vectors-    where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..]-               where makeTest (AES.initKey -> key,plaintext,expected) = assertEq expected $ f key plaintext--katCBCTests vectors f = concatMap makeTests vectors-    where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..]-            where makeTest (AES.initKey -> key,AES.IV -> iv,plaintext,expected) = assertEq expected $ f key iv plaintext--katXTSTests vectors f = concatMap makeTests vectors-    where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..]-              where makeTest (AES.initKey -> key1,AES.initKey -> key2, AES.IV -> iv,plaintext,_,expected) =-                        (assertEq expected $ f (key1,key2) iv 0 plaintext)--katGCMTests vectors f = concatMap makeTests vectors-    where makeTests (name, v) = map (\(z,i) -> testProperty (name ++ " " ++ show i) $ makeTest z) $ zip v [0..]-            where makeTest (AES.initKey -> key, AES.IV -> iv, aad, plaintext, expectedOutput, taglen, expectedTag) =-                        let (output,tag) = f key iv aad plaintext in-                        assertEq expectedOutput output && (assertEq tag expectedTag)---data ECBUnit = ECBUnit B.ByteString B.ByteString-    deriving (Show,Eq)-data CBCUnit = CBCUnit B.ByteString B.ByteString B.ByteString-    deriving (Show,Eq)-data CTRUnit = CTRUnit B.ByteString B.ByteString B.ByteString-    deriving (Show,Eq)-data XTSUnit = XTSUnit B.ByteString B.ByteString B.ByteString B.ByteString-    deriving (Show,Eq)-data GCMUnit = GCMUnit B.ByteString B.ByteString B.ByteString B.ByteString-    deriving (Show,Eq)-data KeyUnit = KeyUnit B.ByteString-    deriving (Show,Eq)--generateKeyOf size = B.pack <$> replicateM size arbitrary-generateKey = elements [16,24,32] >>= generateKeyOf--generateIv = B.pack <$> replicateM 16 arbitrary-generateIvGCM = choose (12,90) >>= \sz -> (B.pack <$> replicateM sz arbitrary)--generatePlaintextMultiple16 = choose (1,128) >>= \size -> replicateM (size*16) arbitrary >>= return . B.pack--generatePlaintext = choose (0,324) >>= \size -> replicateM size arbitrary >>= return . B.pack--instance Arbitrary ECBUnit where-    arbitrary = ECBUnit <$> generateKey-                        <*> generatePlaintextMultiple16--instance Arbitrary CBCUnit where-    arbitrary = CBCUnit <$> generateKey-                        <*> generateIv-                        <*> generatePlaintextMultiple16--instance Arbitrary CTRUnit where-    arbitrary = CTRUnit <$> generateKey-                        <*> generateIv-                        <*> generatePlaintext--instance Arbitrary GCMUnit where-    arbitrary = GCMUnit <$> generateKey-                        <*> generateIvGCM-                        <*> generatePlaintext-                        <*> generatePlaintext--instance Arbitrary XTSUnit where-    arbitrary = do-        size <- elements [16,32]-        XTSUnit <$> generateKeyOf size-                <*> generateKeyOf size-                <*> generateIv-                <*> generatePlaintextMultiple16--instance Arbitrary KeyUnit where-    arbitrary = KeyUnit <$> generateKey--idECBTests (ECBUnit (AES.initKey -> key) plaintext) =-    plaintext `assertEq` AES.decryptECB key (AES.encryptECB key plaintext)--idCBCTests (CBCUnit (AES.initKey -> key) (AES.IV -> iv) plaintext) =-    plaintext `assertEq` AES.decryptCBC key iv (AES.encryptCBC key iv plaintext)--idCTRTests (CTRUnit (AES.initKey -> key) (AES.IV -> iv) plaintext) =-    plaintext `assertEq` AES.decryptCTR key iv (AES.encryptCTR key iv plaintext)--idXTSTests (XTSUnit (AES.initKey -> key1) (AES.initKey -> key2) (AES.IV -> iv) plaintext) =-    plaintext `assertEq` AES.decryptXTS (key1, key2) iv 0 (AES.encryptXTS (key1, key2) iv 0 plaintext)+toKatECB (k,p,c) = KAT_ECB { ecbKey = k, ecbPlaintext = p, ecbCiphertext = c }+toKatCBC (k,iv,p,c) = KAT_CBC { cbcKey = k, cbcIV = iv, cbcPlaintext = p, cbcCiphertext = c }+toKatXTS (k1,k2,iv,p,_,c) = KAT_XTS { xtsKey1 = k1, xtsKey2 = k2, xtsIV = iv, xtsPlaintext = p, xtsCiphertext = c }+toKatGCM (k,iv,h,p,c,taglen,tag) =+    KAT_AEAD { aeadMode       = AEAD_GCM+             , aeadKey        = k+             , aeadIV         = iv+             , aeadHeader     = h+             , aeadPlaintext  = p+             , aeadCiphertext = c+             , aeadTaglen     = taglen+             , aeadTag        = AuthTag tag+             } -idGCMTests (GCMUnit (AES.initKey -> key) (AES.IV -> iv) aad plaintext) =-    let (cipherText, tag) = AES.encryptGCM key iv aad plaintext in-    let (plaintext2, tag2) = AES.decryptGCM key iv aad cipherText in-    (plaintext `assertEq` plaintext2) && (tag == tag2)+kats128 = defaultKATs+    { kat_ECB  = map toKatECB KATECB.vectors_aes128_enc+    , kat_CBC  = map toKatCBC KATCBC.vectors_aes128_enc+    , kat_XTS  = map toKatXTS KATXTS.vectors_aes128_enc+    , kat_AEAD = map toKatGCM KATGCM.vectors_aes128_enc+    } -idKey (KeyUnit keyBs) = keyBs == AES.keyOfCtx (AES.initKey keyBs)+kats192 = defaultKATs+    { kat_ECB  = map toKatECB KATECB.vectors_aes192_enc+    , kat_CBC  = map toKatCBC KATCBC.vectors_aes192_enc+    } -assertEq expected got-	| expected == got = True-	| otherwise       = error ("expected: " ++ showhex expected ++ " got: " ++ showhex got)-    where showhex = concatMap toHex . B.unpack-          toHex b = let (l,r) = b `divMod` 16 in map (toHexChar . fromIntegral) [l,r]-          toHexChar c-                  | c >= 0 && c <= 9   = toEnum (c + fromEnum '0')-                  | c >= 10 && c <= 16 = toEnum (c + fromEnum 'a')-                  | otherwise          = '_'+kats256 = defaultKATs+    { kat_ECB  = map toKatECB KATECB.vectors_aes256_enc+    , kat_CBC  = map toKatCBC KATCBC.vectors_aes256_enc+    , kat_XTS  = map toKatXTS KATXTS.vectors_aes256_enc+    } -tests =-    [ testProperty "key-id" idKey-    , testGroup "KAT-ECB-Encrypt" $ katECBTests KATECB.vectors_encrypt AES.encryptECB-    , testGroup "KAT-ECB-Decrypt" $ katECBTests KATECB.vectors_decrypt AES.decryptECB-    , testGroup "KAT-CBC-Encrypt" $ katCBCTests KATCBC.vectors_encrypt AES.encryptCBC-    , testGroup "KAT-CBC-Decrypt" $ katCBCTests KATCBC.vectors_decrypt AES.decryptCBC-    , testGroup "KAT-XTS-Encrypt" $ katXTSTests KATXTS.vectors_encrypt AES.encryptXTS-    , testGroup "KAT-XTS-Decrypt" $ katXTSTests KATXTS.vectors_decrypt AES.decryptXTS-    , testGroup "KAT-GCM-Encrypt" $ katGCMTests KATGCM.vectors_encrypt AES.encryptGCM-    , testGroup "decrypt-encrypt-is-ID"-        [ testProperty "ECB" idECBTests-        , testProperty "CBC" idCBCTests-        , testProperty "CTR" idCTRTests-        , testProperty "XTS" idXTSTests-        , testProperty "GCM" idGCMTests-        ]+main = defaultMain+    [ testBlockCipher kats128 (undefined :: AES.AES128)+    , testBlockCipher kats192 (undefined :: AES.AES192)+    , testBlockCipher kats256 (undefined :: AES.AES256)     ]--main = defaultMain tests

@@ -38,134 +38,363 @@ #include "gf.h" #include "aes_x86ni.h" -void aes_encrypt_block(aes_block *output, aes_key *key, aes_block *input)-{-#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10)-		return aes_ni_encrypt_ecb((uint8_t *) output, key, (uint8_t *) input, 1);+void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length);+void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                             uint32_t spoint, aes_block *input, uint32_t nb_blocks);+void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                             uint32_t spoint, aes_block *input, uint32_t nb_blocks);+void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);+void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);++enum {+	/* init */+	INIT_128, INIT_192, INIT_256,+	/* single block */+	ENCRYPT_BLOCK_128, ENCRYPT_BLOCK_192, ENCRYPT_BLOCK_256,+	DECRYPT_BLOCK_128, DECRYPT_BLOCK_192, DECRYPT_BLOCK_256,+	/* ecb */+	ENCRYPT_ECB_128, ENCRYPT_ECB_192, ENCRYPT_ECB_256,+	DECRYPT_ECB_128, DECRYPT_ECB_192, DECRYPT_ECB_256,+	/* cbc */+	ENCRYPT_CBC_128, ENCRYPT_CBC_192, ENCRYPT_CBC_256,+	DECRYPT_CBC_128, DECRYPT_CBC_192, DECRYPT_CBC_256,+	/* ctr */+	ENCRYPT_CTR_128, ENCRYPT_CTR_192, ENCRYPT_CTR_256,+	/* xts */+	ENCRYPT_XTS_128, ENCRYPT_XTS_192, ENCRYPT_XTS_256,+	DECRYPT_XTS_128, DECRYPT_XTS_192, DECRYPT_XTS_256,+	/* xts */+	ENCRYPT_GCM_128, ENCRYPT_GCM_192, ENCRYPT_GCM_256,+	DECRYPT_GCM_128, DECRYPT_GCM_192, DECRYPT_GCM_256,+};++void *branch_table[] = {+	/* INIT */+	[INIT_128]          = aes_generic_init,+	[INIT_192]          = aes_generic_init,+	[INIT_256]          = aes_generic_init,+	/* BLOCK */+	[ENCRYPT_BLOCK_128] = aes_generic_encrypt_block,+	[ENCRYPT_BLOCK_192] = aes_generic_encrypt_block,+	[ENCRYPT_BLOCK_256] = aes_generic_encrypt_block,+	[DECRYPT_BLOCK_128] = aes_generic_decrypt_block,+	[DECRYPT_BLOCK_192] = aes_generic_decrypt_block,+	[DECRYPT_BLOCK_256] = aes_generic_decrypt_block,+	/* ECB */+	[ENCRYPT_ECB_128]   = aes_generic_encrypt_ecb,+	[ENCRYPT_ECB_192]   = aes_generic_encrypt_ecb,+	[ENCRYPT_ECB_256]   = aes_generic_encrypt_ecb,+	[DECRYPT_ECB_128]   = aes_generic_decrypt_ecb,+	[DECRYPT_ECB_192]   = aes_generic_decrypt_ecb,+	[DECRYPT_ECB_256]   = aes_generic_decrypt_ecb,+	/* CBC */+	[ENCRYPT_CBC_128]   = aes_generic_encrypt_cbc,+	[ENCRYPT_CBC_192]   = aes_generic_encrypt_cbc,+	[ENCRYPT_CBC_256]   = aes_generic_encrypt_cbc,+	[DECRYPT_CBC_128]   = aes_generic_decrypt_cbc,+	[DECRYPT_CBC_192]   = aes_generic_decrypt_cbc,+	[DECRYPT_CBC_256]   = aes_generic_decrypt_cbc,+	/* CTR */+	[ENCRYPT_CTR_128]   = aes_generic_encrypt_ctr,+	[ENCRYPT_CTR_192]   = aes_generic_encrypt_ctr,+	[ENCRYPT_CTR_256]   = aes_generic_encrypt_ctr,+	/* XTS */+	[ENCRYPT_XTS_128]   = aes_generic_encrypt_xts,+	[ENCRYPT_XTS_192]   = aes_generic_encrypt_xts,+	[ENCRYPT_XTS_256]   = aes_generic_encrypt_xts,+	[DECRYPT_XTS_128]   = aes_generic_decrypt_xts,+	[DECRYPT_XTS_192]   = aes_generic_decrypt_xts,+	[DECRYPT_XTS_256]   = aes_generic_decrypt_xts,+	/* GCM */+	[ENCRYPT_GCM_128]   = aes_generic_gcm_encrypt,+	[ENCRYPT_GCM_192]   = aes_generic_gcm_encrypt,+	[ENCRYPT_GCM_256]   = aes_generic_gcm_encrypt,+	[DECRYPT_GCM_128]   = aes_generic_gcm_decrypt,+	[DECRYPT_GCM_192]   = aes_generic_gcm_decrypt,+	[DECRYPT_GCM_256]   = aes_generic_gcm_decrypt,+};++typedef void (*init_f)(aes_key *, uint8_t *, uint8_t);+typedef void (*ecb_f)(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+typedef void (*cbc_f)(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+typedef void (*ctr_f)(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t length);+typedef void (*xts_f)(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks);+typedef void (*gcm_crypt_f)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);+typedef void (*block_f)(aes_block *output, aes_key *key, aes_block *input);++#ifdef WITH_AESNI+#define GET_INIT(strength) \+	((init_f) (branch_table[INIT_128 + strength]))+#define GET_ECB_ENCRYPT(strength) \+	((ecb_f) (branch_table[ENCRYPT_ECB_128 + strength]))+#define GET_ECB_DECRYPT(strength) \+	((ecb_f) (branch_table[DECRYPT_ECB_128 + strength]))+#define GET_CBC_ENCRYPT(strength) \+	((cbc_f) (branch_table[ENCRYPT_CBC_128 + strength]))+#define GET_CBC_DECRYPT(strength) \+	((cbc_f) (branch_table[DECRYPT_CBC_128 + strength]))+#define GET_CTR_ENCRYPT(strength) \+	((ctr_f) (branch_table[ENCRYPT_CTR_128 + strength]))+#define GET_XTS_ENCRYPT(strength) \+	((xts_f) (branch_table[ENCRYPT_XTS_128 + strength]))+#define GET_XTS_DECRYPT(strength) \+	((xts_f) (branch_table[DECRYPT_XTS_128 + strength]))+#define GET_GCM_ENCRYPT(strength) \+	((gcm_crypt_f) (branch_table[ENCRYPT_GCM_128 + strength]))+#define GET_GCM_DECRYPT(strength) \+	((gcm_crypt_f) (branch_table[DECRYPT_GCM_128 + strength]))+#define aes_encrypt_block(o,k,i) \+	(((block_f) (branch_table[ENCRYPT_BLOCK_128 + k->strength]))(o,k,i))+#define aes_decrypt_block(o,k,i) \+	(((block_f) (branch_table[DECRYPT_BLOCK_128 + k->strength]))(o,k,i))+#else+#define GET_INIT(strenght) aes_generic_init+#define GET_ECB_ENCRYPT(strength) aes_generic_encrypt_ecb+#define GET_ECB_DECRYPT(strength) aes_generic_decrypt_ecb+#define GET_CBC_ENCRYPT(strength) aes_generic_encrypt_cbc+#define GET_CBC_DECRYPT(strength) aes_generic_decrypt_cbc+#define GET_CTR_ENCRYPT(strength) aes_generic_encrypt_ctr+#define GET_XTS_ENCRYPT(strength) aes_generic_encrypt_xts+#define GET_XTS_DECRYPT(strength) aes_generic_decrypt_xts+#define GET_GCM_ENCRYPT(strength) aes_generic_gcm_encrypt+#define GET_GCM_DECRYPT(strength) aes_generic_gcm_decrypt+#define aes_encrypt_block(o,k,i) aes_generic_encrypt_block(o,k,i)+#define aes_decrypt_block(o,k,i) aes-generic_decrypt_block(o,k,i) #endif-	aes_generic_encrypt_block(output, key, input);-} -void aes_decrypt_block(aes_block *output, aes_key *key, aes_block *input)+void initialize_table_ni(int aesni, int pclmul) {-#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10)-		return aes_ni_decrypt_ecb((uint8_t *) output, key, (uint8_t *) input, 1);-#endif-	aes_generic_decrypt_block(output, key, input);+	if (!aesni)+		return;+	branch_table[INIT_128] = aes_ni_init;+	branch_table[INIT_256] = aes_ni_init;++	branch_table[ENCRYPT_BLOCK_128] = aes_ni_encrypt_block128;+	branch_table[DECRYPT_BLOCK_128] = aes_ni_decrypt_block128;+	branch_table[ENCRYPT_BLOCK_256] = aes_ni_encrypt_block256;+	branch_table[DECRYPT_BLOCK_256] = aes_ni_decrypt_block256;+	/* ECB */+	branch_table[ENCRYPT_ECB_128] = aes_ni_encrypt_ecb128;+	branch_table[DECRYPT_ECB_128] = aes_ni_decrypt_ecb128;+	branch_table[ENCRYPT_ECB_256] = aes_ni_encrypt_ecb256;+	branch_table[DECRYPT_ECB_256] = aes_ni_decrypt_ecb256;+	/* CBC */+	branch_table[ENCRYPT_CBC_128] = aes_ni_encrypt_cbc128;+	branch_table[DECRYPT_CBC_128] = aes_ni_decrypt_cbc128;+	branch_table[ENCRYPT_CBC_256] = aes_ni_encrypt_cbc256;+	branch_table[DECRYPT_CBC_256] = aes_ni_decrypt_cbc256;+	/* CTR */+	branch_table[ENCRYPT_CTR_128] = aes_ni_encrypt_ctr128;+	branch_table[ENCRYPT_CTR_256] = aes_ni_encrypt_ctr256;+	/* XTS */+	branch_table[ENCRYPT_XTS_128] = aes_ni_encrypt_xts128;+	branch_table[ENCRYPT_XTS_256] = aes_ni_encrypt_xts256;+	/* GCM */+	branch_table[ENCRYPT_GCM_128] = aes_ni_gcm_encrypt128;+	branch_table[ENCRYPT_GCM_256] = aes_ni_gcm_encrypt256; }  void aes_initkey(aes_key *key, uint8_t *origkey, uint8_t size) { 	switch (size) {-	case 16: key->nbr = 10; break;-	case 24: key->nbr = 12; break;-	case 32: key->nbr = 14; break;+	case 16: key->nbr = 10; key->strength = 0; break;+	case 24: key->nbr = 12; key->strength = 1; break;+	case 32: key->nbr = 14; key->strength = 2; break; 	} #if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && size == 16)-		return aes_ni_init(key, origkey, size);+	initialize_hw(initialize_table_ni); #endif-	aes_generic_init(key, origkey, size);+	init_f _init = GET_INIT(key->strength);+	_init(key, origkey, size); } -void aes_encrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks)+void aes_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) {-	if (!nb_blocks)-		return;+	ecb_f e = GET_ECB_ENCRYPT(key->strength);+	e(output, key, input, nb_blocks);+} -#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10)-		return aes_ni_encrypt_ecb(output, key, input, nb_blocks);-#endif+void aes_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)+{+	ecb_f d = GET_ECB_DECRYPT(key->strength);+	d(output, key, input, nb_blocks);+} -	for ( ; nb_blocks-- > 0; input += 16, output += 16) {-		aes_encrypt_block((block128 *) output, key, (block128 *) input);-	}+void aes_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks)+{+	cbc_f e = GET_CBC_ENCRYPT(key->strength);+	e(output, key, iv, input, nb_blocks); } -void aes_decrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks)+void aes_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks) {-	if (!nb_blocks)-		return;+	cbc_f d = GET_CBC_DECRYPT(key->strength);+	d(output, key, iv, input, nb_blocks);+} -#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10)-		return aes_ni_decrypt_ecb(output, key, input, nb_blocks);-#endif+void aes_gen_ctr(aes_block *output, aes_key *key, aes_block *iv, uint32_t nb_blocks)+{+	aes_block block; -	for ( ; nb_blocks-- > 0; input += 16, output += 16) {-		aes_decrypt_block((block128 *) output, key, (block128 *) input);+	/* preload IV in block */+	block128_copy(&block, iv);++	for ( ; nb_blocks-- > 0; output++, block128_inc_be(&block)) {+		aes_encrypt_block(output, key, &block); 	} } -void aes_encrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks)+void aes_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) {-	aes_block block;+	ctr_f e = GET_CTR_ENCRYPT(key->strength);+	e(output, key, iv, input, len);+} -	if (!nb_blocks)-		return;-#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10)-		return aes_ni_encrypt_cbc(output, key, (uint8_t *) iv, input, nb_blocks);-#endif+void aes_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                     uint32_t spoint, aes_block *input, uint32_t nb_blocks)+{+	xts_f e = GET_XTS_ENCRYPT(k1->strength);+	e(output, k1, k2, dataunit, spoint, input, nb_blocks);+} -	/* preload IV in block */-	block128_copy(&block, iv);+void aes_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                     uint32_t spoint, aes_block *input, uint32_t nb_blocks)+{+	aes_generic_decrypt_xts(output, k1, k2, dataunit, spoint, input, nb_blocks);+} -	for ( ; nb_blocks-- > 0; input += 16, output += 16) {-		block128_xor(&block, (block128 *) input);+void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)+{+	gcm_crypt_f e = GET_GCM_ENCRYPT(key->strength);+	e(output, gcm, key, input, length);+} -		aes_encrypt_block(&block, key, &block);+void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)+{+	gcm_crypt_f d = GET_GCM_DECRYPT(key->strength);+	d(output, gcm, key, input, length);+} -		block128_copy((block128 *) output, &block);-	}+static void gcm_ghash_add(aes_gcm *gcm, block128 *b)+{+	block128_xor(&gcm->tag, b);+	gf_mul(&gcm->tag, &gcm->h); } -void aes_decrypt_cbc(uint8_t *output, aes_key *key, aes_block *ivini, uint8_t *input, uint32_t nb_blocks)+void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len) {-	aes_block block,blocko;-	aes_block iv;+	gcm->length_aad = 0;+	gcm->length_input = 0; -	if (!nb_blocks)-		return;-#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && key->nbr == 10) {-		return aes_ni_decrypt_cbc(output, key, (uint8_t *) ivini, input, nb_blocks);+	block128_zero(&gcm->h);+	block128_zero(&gcm->tag);+	block128_zero(&gcm->iv);++	/* prepare H : encrypt_K(0^128) */+	aes_encrypt_block(&gcm->h, key, &gcm->h);++	if (len == 12) {+		block128_copy_bytes(&gcm->iv, iv, 12);+		gcm->iv.b[15] = 0x01;+	} else {+		uint32_t origlen = len << 3;+		int i;+		for (; len >= 16; len -= 16, iv += 16) {+			block128_xor(&gcm->iv, (block128 *) iv);+			gf_mul(&gcm->iv, &gcm->h);+		}+		if (len > 0) {+			block128_xor_bytes(&gcm->iv, iv, len);+			gf_mul(&gcm->iv, &gcm->h);+		}+		for (i = 15; origlen; --i, origlen >>= 8)+			gcm->iv.b[i] ^= (uint8_t) origlen;+		gf_mul(&gcm->iv, &gcm->h); 	}-#endif -	/* preload IV in block */-	block128_copy(&iv, ivini);+	block128_copy(&gcm->civ, &gcm->iv);+} -	aes_decrypt_block(&block, key, &block);+void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length)+{+	gcm->length_aad += length;+	for (; length >= 16; input += 16, length -= 16) {+		gcm_ghash_add(gcm, (block128 *) input);+	}+	if (length > 0) {+		aes_block tmp;+		block128_zero(&tmp);+		block128_copy_bytes(&tmp, input, length);+		gcm_ghash_add(gcm, &tmp);+	} -	for ( ;nb_blocks-- > 0; input += 16, output += 16) {-		block128_copy(&block, (block128 *) input);+} -		aes_decrypt_block(&blocko, key, &block);+void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm, aes_key *key)+{+	aes_block lblock;+	int i; -		block128_vxor((block128 *) output, &blocko, &iv);-		block128_copy(&iv, &block);+	/* tag = (tag-1 xor (lenbits(a) | lenbits(c)) ) . H */+	lblock.q[0] = cpu_to_be64(gcm->length_aad << 3);+	lblock.q[1] = cpu_to_be64(gcm->length_input << 3);+	gcm_ghash_add(gcm, &lblock);++	aes_encrypt_block(&lblock, key, &gcm->iv);+	block128_xor(&gcm->tag, &lblock);++	for (i = 0; i < 16; i++) {+		tag[i] = gcm->tag.b[i]; 	} } -void aes_gen_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint32_t nb_blocks)+void aes_generic_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks) {+	for ( ; nb_blocks-- > 0; input++, output++) {+		aes_generic_encrypt_block(output, key, input);+	}+}++void aes_generic_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks)+{+	for ( ; nb_blocks-- > 0; input++, output++) {+		aes_generic_decrypt_block(output, key, input);+	}+}++void aes_generic_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks)+{ 	aes_block block; -	if (!nb_blocks)-		return; 	/* preload IV in block */ 	block128_copy(&block, iv);+	for ( ; nb_blocks-- > 0; input++, output++) {+		block128_xor(&block, (block128 *) input);+		aes_generic_encrypt_block(&block, key, &block);+		block128_copy((block128 *) output, &block);+	}+} -	for ( ; nb_blocks-- > 0; output += 16, block128_inc_be(&block)) {-		aes_encrypt_block((block128 *) output, key, &block);+void aes_generic_decrypt_cbc(aes_block *output, aes_key *key, aes_block *ivini, aes_block *input, uint32_t nb_blocks)+{+	aes_block block, blocko;+	aes_block iv;++	/* preload IV in block */+	block128_copy(&iv, ivini);+	for ( ; nb_blocks-- > 0; input++, output++) {+		block128_copy(&block, (block128 *) input);+		aes_generic_decrypt_block(&blocko, key, &block);+		block128_vxor((block128 *) output, &blocko, &iv);+		block128_copy(&iv, &block); 	} } -void aes_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len)+void aes_generic_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len) { 	aes_block block, o; 	uint32_t nb_blocks = len / 16;@@ -183,27 +412,17 @@ 		aes_encrypt_block(&o, key, &block); 		for (i = 0; i < (len % 16); i++) { 			*output = ((uint8_t *) &o)[i] ^ *input;-			output += 1;-			input += 1;+			output++;+			input++; 		} 	} } -void aes_encrypt_xts(uint8_t *output, aes_key *k1, aes_key *k2, aes_block *dataunit,-                     uint32_t spoint, uint8_t *input, uint32_t nb_blocks)+void aes_generic_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                             uint32_t spoint, aes_block *input, uint32_t nb_blocks) { 	aes_block block, tweak; -	if (!nb_blocks)-		return;--#if defined(ARCH_X86) && defined(WITH_AESNI)-	if (have_aesni() && k1->nbr == 10) {-		aes_ni_encrypt_xts(output, k1, k2, (uint8_t *) dataunit, spoint, input, nb_blocks);-		return;-	}-#endif- 	/* load IV and encrypt it using k2 as the tweak */ 	block128_copy(&tweak, dataunit); 	aes_encrypt_block(&tweak, k2, &tweak);@@ -212,21 +431,18 @@ 	while (spoint-- > 0) 		gf_mulx(&tweak); -	for ( ; nb_blocks-- > 0; input += 16, output += 16, gf_mulx(&tweak)) {-		block128_vxor(&block, (block128 *) input, &tweak);+	for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) {+		block128_vxor(&block, input, &tweak); 		aes_encrypt_block(&block, k1, &block);-		block128_vxor((block128 *) output, &block, &tweak);+		block128_vxor(output, &block, &tweak); 	} } -void aes_decrypt_xts(uint8_t *output, aes_key *k1, aes_key *k2, aes_block *dataunit,-                     uint32_t spoint, uint8_t *input, uint32_t nb_blocks)+void aes_generic_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit,+                             uint32_t spoint, aes_block *input, uint32_t nb_blocks) { 	aes_block block, tweak; -	if (!nb_blocks)-		return;- 	/* load IV and encrypt it using k2 as the tweak */ 	block128_copy(&tweak, dataunit); 	aes_encrypt_block(&tweak, k2, &tweak);@@ -235,71 +451,14 @@ 	while (spoint-- > 0) 		gf_mulx(&tweak); -	for ( ; nb_blocks-- > 0; input += 16, output += 16, gf_mulx(&tweak)) {-		block128_vxor(&block, (block128 *) input, &tweak);+	for ( ; nb_blocks-- > 0; input++, output++, gf_mulx(&tweak)) {+		block128_vxor(&block, input, &tweak); 		aes_decrypt_block(&block, k1, &block);-		block128_vxor((block128 *) output, &block, &tweak);+		block128_vxor(output, &block, &tweak); 	} } -static void gcm_ghash_add(aes_gcm *gcm, block128 *b)-{-	block128_xor(&gcm->tag, b);-	gf_mul(&gcm->tag, &gcm->h);-}--void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len)-{-	gcm->length_aad = 0;-	gcm->length_input = 0;--	block128_zero(&gcm->h);-	block128_zero(&gcm->tag);-	block128_zero(&gcm->iv);--	memcpy(&gcm->key, key, sizeof(aes_key));--	/* prepare H : encrypt_K(0^128) */-	aes_encrypt_block(&gcm->h, key, &gcm->h);--	if (len == 12) {-		block128_copy_bytes(&gcm->iv, iv, 12);-		gcm->iv.b[15] = 0x01;-	} else {-		uint32_t origlen = len << 3;-		int i;-		for (; len >= 16; len -= 16, iv += 16) {-			block128_xor(&gcm->iv, (block128 *) iv);-			gf_mul(&gcm->iv, &gcm->h);-		}-		if (len > 0) {-			block128_xor_bytes(&gcm->iv, iv, len);-			gf_mul(&gcm->iv, &gcm->h);-		}-		for (i = 15; origlen; --i, origlen >>= 8)-			gcm->iv.b[i] ^= (uint8_t) origlen;-		gf_mul(&gcm->iv, &gcm->h);-	}--	block128_copy(&gcm->civ, &gcm->iv);-}--void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length)-{-	gcm->length_aad += length;-	for (; length >= 16; input += 16, length -= 16) {-		gcm_ghash_add(gcm, (block128 *) input);-	}-	if (length > 0) {-		aes_block tmp;-		block128_zero(&tmp);-		block128_copy_bytes(&tmp, input, length);-		gcm_ghash_add(gcm, &tmp);-	}--}--void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length)+void aes_generic_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { 	aes_block out; @@ -307,7 +466,7 @@ 	for (; length >= 16; input += 16, output += 16, length -= 16) { 		block128_inc_be(&gcm->civ); -		aes_encrypt_block(&out, &gcm->key, &gcm->civ);+		aes_encrypt_block(&out, key, &gcm->civ); 		block128_xor(&out, (block128 *) input); 		gcm_ghash_add(gcm, &out); 		block128_copy((block128 *) output, &out);@@ -318,11 +477,11 @@  		block128_inc_be(&gcm->civ); 		/* create e(civ) in out */-		aes_encrypt_block(&out, &gcm->key, &gcm->civ);+		aes_encrypt_block(&out, key, &gcm->civ); 		/* initialize a tmp as input and xor it to e(civ) */ 		block128_zero(&tmp); 		block128_copy_bytes(&tmp, input, length);-		block128_xor_bytes(&tmp, out.b, length); +		block128_xor_bytes(&tmp, out.b, length);  		gcm_ghash_add(gcm, &tmp); @@ -332,7 +491,7 @@ 	} } -void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length)+void aes_generic_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length) { 	aes_block out; @@ -340,7 +499,7 @@ 	for (; length >= 16; input += 16, output += 16, length -= 16) { 		block128_inc_be(&gcm->civ); -		aes_encrypt_block(&out, &gcm->key, &gcm->civ);+		aes_encrypt_block(&out, key, &gcm->civ); 		gcm_ghash_add(gcm, (block128 *) input); 		block128_xor(&out, (block128 *) input); 		block128_copy((block128 *) output, &out);@@ -355,8 +514,8 @@ 		block128_copy_bytes(&tmp, input, length); 		gcm_ghash_add(gcm, &tmp); -		aes_encrypt_block(&out, &gcm->key, &gcm->civ);-		block128_xor_bytes(&tmp, out.b, length); +		aes_encrypt_block(&out, key, &gcm->civ);+		block128_xor_bytes(&tmp, out.b, length);  		for (i = 0; i < length; i++) { 			output[i] = tmp.b[i];@@ -364,20 +523,3 @@ 	} } -void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm)-{-	aes_block lblock;-	int i;--	/* tag = (tag-1 xor (lenbits(a) | lenbits(c)) ) . H */-	lblock.q[0] = cpu_to_be64(gcm->length_aad << 3);-	lblock.q[1] = cpu_to_be64(gcm->length_input << 3);-	gcm_ghash_add(gcm, &lblock);--	aes_encrypt_block(&lblock, &gcm->key, &gcm->iv);-	block128_xor(&gcm->tag, &lblock);--	for (i = 0; i < 16; i++) {-		tag[i] = gcm->tag.b[i];-	}-}

@@ -37,13 +37,15 @@  typedef block128 aes_block; +/* size = 456 */ typedef struct { 	uint8_t nbr; /* number of rounds: 10 (128), 12 (192), 14 (256) */-	uint8_t _padding[7];+	uint8_t strength; /* 128 = 0, 192 = 1, 256 = 2 */+	uint8_t _padding[6]; 	uint8_t data[16*14*2]; } aes_key; -/* size = 4*16+2*8+aes_key=456 = 536 */+/* size = 4*16+2*8= 80 */ typedef struct { 	aes_block tag; 	aes_block h;@@ -51,7 +53,6 @@ 	aes_block civ; 	uint64_t length_aad; 	uint64_t length_input;-	aes_key key; } aes_gcm;  /* in bytes: either 16,24,32 */@@ -60,22 +61,23 @@ void aes_encrypt(aes_block *output, aes_key *key, aes_block *input); void aes_decrypt(aes_block *output, aes_key *key, aes_block *input); -void aes_encrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks);-void aes_decrypt_ecb(uint8_t *output, aes_key *key, uint8_t *input, uint32_t nb_blocks);+void aes_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+void aes_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks); -void aes_encrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks);-void aes_decrypt_cbc(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t nb_blocks);+void aes_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+void aes_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks); -void aes_gen_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint32_t nb_blocks);+void aes_gen_ctr(aes_block *output, aes_key *key, aes_block *iv, uint32_t nb_blocks); -void aes_encrypt_xts(uint8_t *output, aes_key *key, aes_key *key2, aes_block *sector,-                     uint32_t spoint, uint8_t *input, uint32_t nb_blocks);-void aes_decrypt_xts(uint8_t *output, aes_key *key, aes_key *key2, aes_block *sector,-                     uint32_t spoint, uint8_t *input, uint32_t nb_blocks);+void aes_encrypt_xts(aes_block *output, aes_key *key, aes_key *key2, aes_block *sector,+                     uint32_t spoint, aes_block *input, uint32_t nb_blocks);+void aes_decrypt_xts(aes_block *output, aes_key *key, aes_key *key2, aes_block *sector,+                     uint32_t spoint, aes_block *input, uint32_t nb_blocks);  void aes_gcm_init(aes_gcm *gcm, aes_key *key, uint8_t *iv, uint32_t len); void aes_gcm_aad(aes_gcm *gcm, uint8_t *input, uint32_t length);-void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, uint8_t *input, uint32_t length);-void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm);+void aes_gcm_encrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);+void aes_gcm_decrypt(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length);+void aes_gcm_finish(uint8_t *tag, aes_gcm *gcm, aes_key *key);  #endif

@@ -1,5 +1,5 @@ /*- * Copyright (c) 2012 Vincent Hanquez <vincent@snarc.org>+ * Copyright (c) 2012-2013 Vincent Hanquez <vincent@snarc.org>  *   * All rights reserved.  * @@ -32,70 +32,154 @@  #include <wmmintrin.h> #include <tmmintrin.h>+#include <string.h> #include "aes.h" #include "aes_x86ni.h"+#include "block128.h" #include "cpu.h"  #ifdef ARCH_X86 #define ALIGN_UP(addr, size) (((addr) + ((size) - 1)) & (~((size) - 1))) #define ALIGNMENT(n) __attribute__((aligned(n))) -static __m128i aes_128_key_expansion(__m128i key, __m128i keygened)+static __m128i aes_128_key_expansion(__m128i key, __m128i keygened, int shuffle) {-	keygened = _mm_shuffle_epi32(keygened, _MM_SHUFFLE(3,3,3,3));+	keygened = _mm_shuffle_epi32(keygened, shuffle); 	key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); 	key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); 	key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); 	return _mm_xor_si128(key, keygened); } -static void aes_generate_key128(aes_key *key, uint8_t *ikey)+void aes_ni_init(aes_key *key, uint8_t *ikey, uint8_t size) {-	__m128i k[20];+	__m128i k[28]; 	uint64_t *out = (uint64_t *) key->data; 	int i; -	k[0] = _mm_loadu_si128((const __m128i*) ikey);+	switch (size) {+	case 16:+		k[0] = _mm_loadu_si128((const __m128i*) ikey); -#define AES_128_key_exp(K, RCON) aes_128_key_expansion(K, _mm_aeskeygenassist_si128(K, RCON))-	k[1]  = AES_128_key_exp(k[0], 0x01);-	k[2]  = AES_128_key_exp(k[1], 0x02);-	k[3]  = AES_128_key_exp(k[2], 0x04);-	k[4]  = AES_128_key_exp(k[3], 0x08);-	k[5]  = AES_128_key_exp(k[4], 0x10);-	k[6]  = AES_128_key_exp(k[5], 0x20);-	k[7]  = AES_128_key_exp(k[6], 0x40);-	k[8]  = AES_128_key_exp(k[7], 0x80);-	k[9]  = AES_128_key_exp(k[8], 0x1B);-	k[10] = AES_128_key_exp(k[9], 0x36);+		#define AES_128_key_exp(K, RCON) aes_128_key_expansion(K, _mm_aeskeygenassist_si128(K, RCON), 0xff)+		k[1]  = AES_128_key_exp(k[0], 0x01);+		k[2]  = AES_128_key_exp(k[1], 0x02);+		k[3]  = AES_128_key_exp(k[2], 0x04);+		k[4]  = AES_128_key_exp(k[3], 0x08);+		k[5]  = AES_128_key_exp(k[4], 0x10);+		k[6]  = AES_128_key_exp(k[5], 0x20);+		k[7]  = AES_128_key_exp(k[6], 0x40);+		k[8]  = AES_128_key_exp(k[7], 0x80);+		k[9]  = AES_128_key_exp(k[8], 0x1B);+		k[10] = AES_128_key_exp(k[9], 0x36); -	/* generate decryption keys in reverse order.-	 * k[10] is shared by last encryption and first decryption rounds-	 * k[20] is shared by first encryption round (and is the original user key) */-	k[11] = _mm_aesimc_si128(k[9]);-	k[12] = _mm_aesimc_si128(k[8]);-	k[13] = _mm_aesimc_si128(k[7]);-	k[14] = _mm_aesimc_si128(k[6]);-	k[15] = _mm_aesimc_si128(k[5]);-	k[16] = _mm_aesimc_si128(k[4]);-	k[17] = _mm_aesimc_si128(k[3]);-	k[18] = _mm_aesimc_si128(k[2]);-	k[19] = _mm_aesimc_si128(k[1]);+		/* generate decryption keys in reverse order.+		 * k[10] is shared by last encryption and first decryption rounds+		 * k[20] is shared by first encryption round (and is the original user key) */+		k[11] = _mm_aesimc_si128(k[9]);+		k[12] = _mm_aesimc_si128(k[8]);+		k[13] = _mm_aesimc_si128(k[7]);+		k[14] = _mm_aesimc_si128(k[6]);+		k[15] = _mm_aesimc_si128(k[5]);+		k[16] = _mm_aesimc_si128(k[4]);+		k[17] = _mm_aesimc_si128(k[3]);+		k[18] = _mm_aesimc_si128(k[2]);+		k[19] = _mm_aesimc_si128(k[1]); -	for (i = 0; i < 20; i++)-		_mm_storeu_si128(((__m128i *) out) + i, k[i]);+		for (i = 0; i < 20; i++)+			_mm_storeu_si128(((__m128i *) out) + i, k[i]);+		break;+	case 32:+#define AES_256_key_exp_1(K1, K2, RCON) aes_128_key_expansion(K1, _mm_aeskeygenassist_si128(K2, RCON), 0xff)+#define AES_256_key_exp_2(K1, K2)       aes_128_key_expansion(K1, _mm_aeskeygenassist_si128(K2, 0x00), 0xaa)+		k[0]  = _mm_loadu_si128((const __m128i*) ikey);+		k[1]  = _mm_loadu_si128((const __m128i*) (ikey+16));+		k[2]  = AES_256_key_exp_1(k[0], k[1], 0x01);+		k[3]  = AES_256_key_exp_2(k[1], k[2]);+		k[4]  = AES_256_key_exp_1(k[2], k[3], 0x02);+		k[5]  = AES_256_key_exp_2(k[3], k[4]);+		k[6]  = AES_256_key_exp_1(k[4], k[5], 0x04);+		k[7]  = AES_256_key_exp_2(k[5], k[6]);+		k[8]  = AES_256_key_exp_1(k[6], k[7], 0x08);+		k[9]  = AES_256_key_exp_2(k[7], k[8]);+		k[10] = AES_256_key_exp_1(k[8], k[9], 0x10);+		k[11] = AES_256_key_exp_2(k[9], k[10]);+		k[12] = AES_256_key_exp_1(k[10], k[11], 0x20);+		k[13] = AES_256_key_exp_2(k[11], k[12]);+		k[14] = AES_256_key_exp_1(k[12], k[13], 0x40);++		k[15] = _mm_aesimc_si128(k[13]);+		k[16] = _mm_aesimc_si128(k[12]);+		k[17] = _mm_aesimc_si128(k[11]);+		k[18] = _mm_aesimc_si128(k[10]);+		k[19] = _mm_aesimc_si128(k[9]);+		k[20] = _mm_aesimc_si128(k[8]);+		k[21] = _mm_aesimc_si128(k[7]);+		k[22] = _mm_aesimc_si128(k[6]);+		k[23] = _mm_aesimc_si128(k[5]);+		k[24] = _mm_aesimc_si128(k[4]);+		k[25] = _mm_aesimc_si128(k[3]);+		k[26] = _mm_aesimc_si128(k[2]);+		k[27] = _mm_aesimc_si128(k[1]);+		for (i = 0; i < 28; i++)+			_mm_storeu_si128(((__m128i *) out) + i, k[i]);+		break;+	default:+		break;+	} } -void aes_ni_init(aes_key *key, uint8_t *origkey, uint8_t size)+/* TO OPTIMISE: use pcmulqdq... or some faster code.+ * this is the lamest way of doing it, but i'm out of time.+ * this is basically a copy of gf_mulx in gf.c */+static __m128i gfmulx(__m128i v) {-	switch (size) {-	case 16: aes_generate_key128(key, origkey); break;-	default: break;-	}+	uint64_t v_[2] ALIGNMENT(16);+	const uint64_t gf_mask = 0x8000000000000000;++	_mm_store_si128((__m128i *) v_, v);+	uint64_t r = ((v_[1] & gf_mask) ? 0x87 : 0);+	v_[1] = (v_[1] << 1) | (v_[0] & gf_mask ? 1 : 0);+	v_[0] = (v_[0] << 1) ^ r;+	v = _mm_load_si128((__m128i *) v_);+	return v; } +static void unopt_gf_mul(block128 *a, block128 *b)+{+	uint64_t a0, a1, v0, v1;+	int i, j; -#define PRELOAD_ENC_KEYS(k) \+	a0 = a1 = 0;+	v0 = cpu_to_be64(a->q[0]);+	v1 = cpu_to_be64(a->q[1]);++	for (i = 0; i < 16; i++)+		for (j = 0x80; j != 0; j >>= 1) {+			uint8_t x = b->b[i] & j;+			a0 ^= x ? v0 : 0;+			a1 ^= x ? v1 : 0;+			x = (uint8_t) v1 & 1;+			v1 = (v1 >> 1) | (v0 << 63);+			v0 = (v0 >> 1) ^ (x ? (0xe1ULL << 56) : 0);+		}+	a->q[0] = cpu_to_be64(a0);+	a->q[1] = cpu_to_be64(a1);+}++static __m128i ghash_add(__m128i tag, __m128i h, __m128i m)+{+	aes_block _t, _h;+	tag = _mm_xor_si128(tag, m);++	_mm_store_si128((__m128i *) &_t, tag);+	_mm_store_si128((__m128i *) &_h, h);+	unopt_gf_mul(&_t, &_h);+	tag = _mm_load_si128((__m128i *) &_t);+	return tag;+}++#define PRELOAD_ENC_KEYS128(k) \ 	__m128i K0  = _mm_loadu_si128(((__m128i *) k)+0); \ 	__m128i K1  = _mm_loadu_si128(((__m128i *) k)+1); \ 	__m128i K2  = _mm_loadu_si128(((__m128i *) k)+2); \@@ -108,7 +192,14 @@ 	__m128i K9  = _mm_loadu_si128(((__m128i *) k)+9); \ 	__m128i K10 = _mm_loadu_si128(((__m128i *) k)+10); -#define DO_ENC_BLOCK(m) \+#define PRELOAD_ENC_KEYS256(k) \+	PRELOAD_ENC_KEYS128(k) \+	__m128i K11 = _mm_loadu_si128(((__m128i *) k)+11); \+	__m128i K12 = _mm_loadu_si128(((__m128i *) k)+12); \+	__m128i K13 = _mm_loadu_si128(((__m128i *) k)+13); \+	__m128i K14 = _mm_loadu_si128(((__m128i *) k)+14);++#define DO_ENC_BLOCK128(m) \ 	m = _mm_xor_si128(m, K0); \ 	m = _mm_aesenc_si128(m, K1); \ 	m = _mm_aesenc_si128(m, K2); \@@ -121,20 +212,49 @@ 	m = _mm_aesenc_si128(m, K9); \ 	m = _mm_aesenclast_si128(m, K10); -#define PRELOAD_DEC_KEYS(k) \-	__m128i K0  = _mm_loadu_si128(((__m128i *) k)+10+0); \-	__m128i K1  = _mm_loadu_si128(((__m128i *) k)+10+1); \-	__m128i K2  = _mm_loadu_si128(((__m128i *) k)+10+2); \-	__m128i K3  = _mm_loadu_si128(((__m128i *) k)+10+3); \-	__m128i K4  = _mm_loadu_si128(((__m128i *) k)+10+4); \-	__m128i K5  = _mm_loadu_si128(((__m128i *) k)+10+5); \-	__m128i K6  = _mm_loadu_si128(((__m128i *) k)+10+6); \-	__m128i K7  = _mm_loadu_si128(((__m128i *) k)+10+7); \-	__m128i K8  = _mm_loadu_si128(((__m128i *) k)+10+8); \-	__m128i K9  = _mm_loadu_si128(((__m128i *) k)+10+9); \+#define DO_ENC_BLOCK256(m) \+	m = _mm_xor_si128(m, K0); \+	m = _mm_aesenc_si128(m, K1); \+	m = _mm_aesenc_si128(m, K2); \+	m = _mm_aesenc_si128(m, K3); \+	m = _mm_aesenc_si128(m, K4); \+	m = _mm_aesenc_si128(m, K5); \+	m = _mm_aesenc_si128(m, K6); \+	m = _mm_aesenc_si128(m, K7); \+	m = _mm_aesenc_si128(m, K8); \+	m = _mm_aesenc_si128(m, K9); \+	m = _mm_aesenc_si128(m, K10); \+	m = _mm_aesenc_si128(m, K11); \+	m = _mm_aesenc_si128(m, K12); \+	m = _mm_aesenc_si128(m, K13); \+	m = _mm_aesenclast_si128(m, K14);++/* load K0 at K9 from index 'at' */+#define PRELOAD_DEC_KEYS_AT(k, at) \+	__m128i K0  = _mm_loadu_si128(((__m128i *) k)+at+0); \+	__m128i K1  = _mm_loadu_si128(((__m128i *) k)+at+1); \+	__m128i K2  = _mm_loadu_si128(((__m128i *) k)+at+2); \+	__m128i K3  = _mm_loadu_si128(((__m128i *) k)+at+3); \+	__m128i K4  = _mm_loadu_si128(((__m128i *) k)+at+4); \+	__m128i K5  = _mm_loadu_si128(((__m128i *) k)+at+5); \+	__m128i K6  = _mm_loadu_si128(((__m128i *) k)+at+6); \+	__m128i K7  = _mm_loadu_si128(((__m128i *) k)+at+7); \+	__m128i K8  = _mm_loadu_si128(((__m128i *) k)+at+8); \+	__m128i K9  = _mm_loadu_si128(((__m128i *) k)+at+9); \++#define PRELOAD_DEC_KEYS128(k) \+	PRELOAD_DEC_KEYS_AT(k, 10) \ 	__m128i K10 = _mm_loadu_si128(((__m128i *) k)+0); -#define DO_DEC_BLOCK(m) \+#define PRELOAD_DEC_KEYS256(k) \+	PRELOAD_DEC_KEYS_AT(k, 14) \+	__m128i K10 = _mm_loadu_si128(((__m128i *) k)+14+10); \+	__m128i K11 = _mm_loadu_si128(((__m128i *) k)+14+11); \+	__m128i K12 = _mm_loadu_si128(((__m128i *) k)+14+12); \+	__m128i K13 = _mm_loadu_si128(((__m128i *) k)+14+13); \+	__m128i K14 = _mm_loadu_si128(((__m128i *) k)+0);++#define DO_DEC_BLOCK128(m) \ 	m = _mm_xor_si128(m, K0); \ 	m = _mm_aesdec_si128(m, K1); \ 	m = _mm_aesdec_si128(m, K2); \@@ -147,128 +267,52 @@ 	m = _mm_aesdec_si128(m, K9); \ 	m = _mm_aesdeclast_si128(m, K10); -void aes_ni_encrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks)-{-	__m128i *k = (__m128i *) key->data;--	PRELOAD_ENC_KEYS(k);--	while (blocks-- > 0) {-		__m128i m = _mm_loadu_si128((__m128i *) in);--		DO_ENC_BLOCK(m);--		_mm_storeu_si128((__m128i *) out, m);-		in += 16;-		out += 16;-	}-}--void aes_ni_decrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks)-{-	__m128i *k = (__m128i *) key->data;--	PRELOAD_DEC_KEYS(k);--	while (blocks-- > 0) {-		__m128i m = _mm_loadu_si128((__m128i *) in);--		DO_DEC_BLOCK(m);--		_mm_storeu_si128((__m128i *) out, m);-		in += 16;-		out += 16;-	}-}--void aes_ni_encrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks)-{-	__m128i *k = (__m128i *) key->data;-	__m128i iv = _mm_loadu_si128((__m128i *) _iv);--	PRELOAD_ENC_KEYS(k);--	while (blocks-- > 0) {-		__m128i m = _mm_loadu_si128((__m128i *) in);-		m = _mm_xor_si128(m, iv);--		DO_ENC_BLOCK(m);--		_mm_storeu_si128((__m128i *) out, m);-		iv = m;--		in += 16;-		out += 16;-	}-}--void aes_ni_decrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks)-{-	__m128i *k = (__m128i *) key->data;-	__m128i iv = _mm_loadu_si128((__m128i *) _iv);--	PRELOAD_DEC_KEYS(k);--	while (blocks-- > 0) {-		__m128i m = _mm_loadu_si128((__m128i *) in);-		__m128i ivnext = m;--		DO_DEC_BLOCK(m);-		m = _mm_xor_si128(m, iv);--		_mm_storeu_si128((__m128i *) out, m);-		iv = ivnext;--		in += 16;-		out += 16;-	}-}--/* TO OPTIMISE: use pcmulqdq... or some faster code.- * this is the lamest way of doing it, but i'm out of time.- * this is basically a copy of gf_mulx in gf.c */-static __m128i gfmulx(__m128i v)-{-	uint64_t v_[2] ALIGNMENT(16);-	const uint64_t gf_mask = 0x8000000000000000;--	_mm_store_si128((__m128i *) v_, v);-	uint64_t r = ((v_[1] & gf_mask) ? 0x87 : 0);-	v_[1] = (v_[1] << 1) | (v_[0] & gf_mask ? 1 : 0);-	v_[0] = (v_[0] << 1) ^ r;-	v = _mm_load_si128((__m128i *) v_);-	return v;-}--void aes_ni_encrypt_xts(uint8_t *out, aes_key *key1, aes_key *key2,-                        uint8_t *_tweak, uint32_t spoint, uint8_t *in, uint32_t blocks)-{-	__m128i tweak = _mm_loadu_si128((__m128i *) _tweak);--	do {-		__m128i *k2 = (__m128i *) key2->data;-		PRELOAD_ENC_KEYS(k2);-		DO_ENC_BLOCK(tweak);--		while (spoint-- > 0)-			tweak = gfmulx(tweak);-	} while (0) ;+#define DO_DEC_BLOCK256(m) \+	m = _mm_xor_si128(m, K0); \+	m = _mm_aesdec_si128(m, K1); \+	m = _mm_aesdec_si128(m, K2); \+	m = _mm_aesdec_si128(m, K3); \+	m = _mm_aesdec_si128(m, K4); \+	m = _mm_aesdec_si128(m, K5); \+	m = _mm_aesdec_si128(m, K6); \+	m = _mm_aesdec_si128(m, K7); \+	m = _mm_aesdec_si128(m, K8); \+	m = _mm_aesdec_si128(m, K9); \+	m = _mm_aesdec_si128(m, K10); \+	m = _mm_aesdec_si128(m, K11); \+	m = _mm_aesdec_si128(m, K12); \+	m = _mm_aesdec_si128(m, K13); \+	m = _mm_aesdeclast_si128(m, K14); -	do {-		__m128i *k1 = (__m128i *) key1->data;-		PRELOAD_ENC_KEYS(k1);+#define SIZE 128+#define SIZED(m) m##128+#define PRELOAD_ENC PRELOAD_ENC_KEYS128+#define DO_ENC_BLOCK DO_ENC_BLOCK128+#define PRELOAD_DEC PRELOAD_DEC_KEYS128+#define DO_DEC_BLOCK DO_DEC_BLOCK128+#include "aes_x86ni_impl.c" -		for ( ; blocks-- > 0; in += 16, out += 16, tweak = gfmulx(tweak)) {-			__m128i m = _mm_loadu_si128((__m128i *) in);+#undef SIZE+#undef SIZED+#undef PRELOAD_ENC+#undef PRELOAD_DEC+#undef DO_ENC_BLOCK+#undef DO_DEC_BLOCK -			m = _mm_xor_si128(m, tweak);-			DO_ENC_BLOCK(m);-			m = _mm_xor_si128(m, tweak);+#define SIZED(m) m##256+#define SIZE 256+#define PRELOAD_ENC PRELOAD_ENC_KEYS256+#define DO_ENC_BLOCK DO_ENC_BLOCK256+#define PRELOAD_DEC PRELOAD_DEC_KEYS256+#define DO_DEC_BLOCK DO_DEC_BLOCK256+#include "aes_x86ni_impl.c" -			_mm_storeu_si128((__m128i *) out, m);-		}-	} while (0);-}+#undef SIZE+#undef SIZED+#undef PRELOAD_ENC+#undef PRELOAD_DEC+#undef DO_ENC_BLOCK+#undef DO_DEC_BLOCK  #endif 

@@ -38,14 +38,39 @@ #include <wmmintrin.h> #include <tmmintrin.h> #include "aes.h"+#include "block128.h" +#ifdef IMPL_DEBUG+static void block128_sse_print(__m128i m)+{+	block128 b;+	_mm_storeu_si128((__m128i *) &b.b, m);+	block128_print(&b);+}+#endif+ void aes_ni_init(aes_key *key, uint8_t *origkey, uint8_t size);-void aes_ni_encrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks);-void aes_ni_decrypt_ecb(uint8_t *out, aes_key *key, uint8_t *in, uint32_t blocks);-void aes_ni_encrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks);-void aes_ni_decrypt_cbc(uint8_t *out, aes_key *key, uint8_t *_iv, uint8_t *in, uint32_t blocks);-void aes_ni_encrypt_xts(uint8_t *out, aes_key *key1, aes_key *key2,-                        uint8_t *_tweak, uint32_t spoint, uint8_t *in, uint32_t blocks);+void aes_ni_encrypt_block128(aes_block *out, aes_key *key, aes_block *in);+void aes_ni_encrypt_block256(aes_block *out, aes_key *key, aes_block *in);+void aes_ni_decrypt_block128(aes_block *out, aes_key *key, aes_block *in);+void aes_ni_decrypt_block256(aes_block *out, aes_key *key, aes_block *in);+void aes_ni_encrypt_ecb128(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks);+void aes_ni_encrypt_ecb256(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks);+void aes_ni_decrypt_ecb128(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks);+void aes_ni_decrypt_ecb256(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks);+void aes_ni_encrypt_cbc128(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks);+void aes_ni_encrypt_cbc256(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks);+void aes_ni_decrypt_cbc128(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks);+void aes_ni_decrypt_cbc256(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks);+void aes_ni_encrypt_ctr128(uint8_t *out, aes_key *key, aes_block *_iv, uint8_t *in, uint32_t length);+void aes_ni_encrypt_ctr256(uint8_t *out, aes_key *key, aes_block *_iv, uint8_t *in, uint32_t length);+void aes_ni_encrypt_xts128(aes_block *out, aes_key *key1, aes_key *key2,+                           aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks);+void aes_ni_encrypt_xts256(aes_block *out, aes_key *key1, aes_key *key2,+                           aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks);++void aes_ni_gcm_encrypt128(uint8_t *out, aes_gcm *gcm, aes_key *key, uint8_t *in, uint32_t length);+void aes_ni_gcm_encrypt256(uint8_t *out, aes_gcm *gcm, aes_key *key, uint8_t *in, uint32_t length);  void gf_mul_x86ni(block128 *res, block128 *a_, block128 *b_); 

@@ -0,0 +1,263 @@+/*+ * Copyright (c) 2012-2013 Vincent Hanquez <vincent@snarc.org>+ *+ * All rights reserved.+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions+ * are met:+ * 1. Redistributions of source code must retain the above copyright+ *    notice, this list of conditions and the following disclaimer.+ * 2. Redistributions in binary form must reproduce the above copyright+ *    notice, this list of conditions and the following disclaimer in the+ *    documentation and/or other materials provided with the distribution.+ * 3. Neither the name of the author nor the names of his contributors+ *    may be used to endorse or promote products derived from this software+ *    without specific prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+ * SUCH DAMAGE.+ */++void SIZED(aes_ni_encrypt_block)(aes_block *out, aes_key *key, aes_block *in)+{+	__m128i *k = (__m128i *) key->data;+	PRELOAD_ENC(k);+	__m128i m = _mm_loadu_si128((__m128i *) in);+	DO_ENC_BLOCK(m);+	_mm_storeu_si128((__m128i *) out, m);+}++void SIZED(aes_ni_decrypt_block)(aes_block *out, aes_key *key, aes_block *in)+{+	__m128i *k = (__m128i *) key->data;+	PRELOAD_DEC(k);+	__m128i m = _mm_loadu_si128((__m128i *) in);+	DO_DEC_BLOCK(m);+	_mm_storeu_si128((__m128i *) out, m);+}++void SIZED(aes_ni_encrypt_ecb)(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks)+{+	__m128i *k = (__m128i *) key->data;++	PRELOAD_ENC(k);+	for (; blocks-- > 0; in += 1, out += 1) {+		__m128i m = _mm_loadu_si128((__m128i *) in);+		DO_ENC_BLOCK(m);+		_mm_storeu_si128((__m128i *) out, m);+	}+}++void SIZED(aes_ni_decrypt_ecb)(aes_block *out, aes_key *key, aes_block *in, uint32_t blocks)+{+	__m128i *k = (__m128i *) key->data;++	PRELOAD_DEC(k);++	for (; blocks-- > 0; in += 1, out += 1) {+		__m128i m = _mm_loadu_si128((__m128i *) in);+		DO_DEC_BLOCK(m);+		_mm_storeu_si128((__m128i *) out, m);+	}+}++void SIZED(aes_ni_encrypt_cbc)(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks)+{+	__m128i *k = (__m128i *) key->data;+	__m128i iv = _mm_loadu_si128((__m128i *) _iv);++	PRELOAD_ENC(k);++	for (; blocks-- > 0; in += 1, out += 1) {+		__m128i m = _mm_loadu_si128((__m128i *) in);+		m = _mm_xor_si128(m, iv);+		DO_ENC_BLOCK(m);+		iv = m;+		_mm_storeu_si128((__m128i *) out, m);+	}+}++void SIZED(aes_ni_decrypt_cbc)(aes_block *out, aes_key *key, aes_block *_iv, aes_block *in, uint32_t blocks)+{+	__m128i *k = (__m128i *) key->data;+	__m128i iv = _mm_loadu_si128((__m128i *) _iv);++	PRELOAD_DEC(k);++	for (; blocks-- > 0; in += 1, out += 1) {+		__m128i m = _mm_loadu_si128((__m128i *) in);+		__m128i ivnext = m;++		DO_DEC_BLOCK(m);+		m = _mm_xor_si128(m, iv);++		_mm_storeu_si128((__m128i *) out, m);+		iv = ivnext;+	}+}++void SIZED(aes_ni_encrypt_ctr)(uint8_t *output, aes_key *key, aes_block *_iv, uint8_t *input, uint32_t len)+{+	__m128i *k = (__m128i *) key->data;+	__m128i bswap_mask = _mm_setr_epi8(7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8);+	__m128i one        = _mm_set_epi32(0,1,0,0);+	uint32_t nb_blocks = len / 16;+	uint32_t part_block_len = len % 16;++	/* get the IV in little endian format */+	__m128i iv = _mm_loadu_si128((__m128i *) _iv);+	iv = _mm_shuffle_epi8(iv, bswap_mask);++	PRELOAD_ENC(k);++	for (; nb_blocks-- > 0; output += 16, input += 16) {+		/* put back the iv in big endian mode,+		 * encrypt it and and xor it the input block+		 */+		__m128i tmp = _mm_shuffle_epi8(iv, bswap_mask);+		DO_ENC_BLOCK(tmp);+		__m128i m = _mm_loadu_si128((__m128i *) input);+		m = _mm_xor_si128(m, tmp);++		_mm_storeu_si128((__m128i *) output, m);+		/* iv += 1 */+		iv = _mm_add_epi64(iv, one);+	}++	if (part_block_len != 0) {+		aes_block block;+		memset(&block.b, 0, 16);+		memcpy(&block.b, input, part_block_len);++		__m128i m = _mm_loadu_si128((__m128i *) &block);+		__m128i tmp = _mm_shuffle_epi8(iv, bswap_mask);++		DO_ENC_BLOCK(tmp);+		m = _mm_xor_si128(m, tmp);+		_mm_storeu_si128((__m128i *) &block.b, m);+		memcpy(output, &block.b, part_block_len);+	}++	return ;+}++void SIZED(aes_ni_encrypt_xts)(aes_block *out, aes_key *key1, aes_key *key2,+                               aes_block *_tweak, uint32_t spoint, aes_block *in, uint32_t blocks)+{+	__m128i tweak = _mm_loadu_si128((__m128i *) _tweak);++	do {+		__m128i *k2 = (__m128i *) key2->data;+		PRELOAD_ENC(k2);+		DO_ENC_BLOCK(tweak);++		while (spoint-- > 0)+			tweak = gfmulx(tweak);+	} while (0) ;++	do {+		__m128i *k1 = (__m128i *) key1->data;+		PRELOAD_ENC(k1);++		for ( ; blocks-- > 0; in += 1, out += 1, tweak = gfmulx(tweak)) {+			__m128i m = _mm_loadu_si128((__m128i *) in);++			m = _mm_xor_si128(m, tweak);+			DO_ENC_BLOCK(m);+			m = _mm_xor_si128(m, tweak);++			_mm_storeu_si128((__m128i *) out, m);+		}+	} while (0);+}++void SIZED(aes_ni_gcm_encrypt)(uint8_t *output, aes_gcm *gcm, aes_key *key, uint8_t *input, uint32_t length)+{+	__m128i *k = (__m128i *) key->data;+	__m128i bswap_mask = _mm_setr_epi8(7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8);+	__m128i one        = _mm_set_epi32(0,1,0,0);+	uint32_t nb_blocks = length / 16;+	uint32_t part_block_len = length % 16;++	gcm->length_input += length;++	__m128i h  = _mm_loadu_si128((__m128i *) &gcm->h);+	__m128i tag = _mm_loadu_si128((__m128i *) &gcm->tag);+	__m128i iv = _mm_loadu_si128((__m128i *) &gcm->civ);+	iv = _mm_shuffle_epi8(iv, bswap_mask);++	PRELOAD_ENC(k);++	for (; nb_blocks-- > 0; output += 16, input += 16) {+		/* iv += 1 */+		iv = _mm_add_epi64(iv, one);++		/* put back iv in big endian, encrypt it,+		 * and xor it to input */+		__m128i tmp = _mm_shuffle_epi8(iv, bswap_mask);+		DO_ENC_BLOCK(tmp);+		__m128i m = _mm_loadu_si128((__m128i *) input);+		m = _mm_xor_si128(m, tmp);++		tag = ghash_add(tag, h, m);++		/* store it out */+		_mm_storeu_si128((__m128i *) output, m);+	}+	if (part_block_len > 0) {+		__m128i mask;+		aes_block block;+		/* FIXME could do something a bit more clever (slli & sub & and maybe) ... */+		switch (part_block_len) {+		case 1: mask = _mm_setr_epi8(0,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 2: mask = _mm_setr_epi8(0,1,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 3: mask = _mm_setr_epi8(0,1,2,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 4: mask = _mm_setr_epi8(0,1,2,3,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 5: mask = _mm_setr_epi8(0,1,2,3,4,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 6: mask = _mm_setr_epi8(0,1,2,3,4,5,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 7: mask = _mm_setr_epi8(0,1,2,3,4,5,6,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 8: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,0x80,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 9: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,0x80,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 10: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,0x80,0x80,0x80,0x80,0x80,0x80); break;+		case 11: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,0x80,0x80,0x80,0x80,0x80); break;+		case 12: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,0x80,0x80,0x80,0x80); break;+		case 13: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,0x80,0x80,0x80); break;+		case 14: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,0x80,0x80); break;+		case 15: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0x80); break;+		default: mask = _mm_setr_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); break;+		}+		memset(&block.b, 0, 16);+		memcpy(&block.b, input, part_block_len);++		/* iv += 1 */+		iv = _mm_add_epi64(iv, one);++		/* put back iv in big endian mode, encrypt it and xor it with input */+		__m128i tmp = _mm_shuffle_epi8(iv, bswap_mask);+		DO_ENC_BLOCK(tmp);++		__m128i m = _mm_loadu_si128((__m128i *) &block);+		m = _mm_xor_si128(m, tmp);+		m = _mm_shuffle_epi8(m, mask);++		tag = ghash_add(tag, h, m);++		/* make output */+		_mm_storeu_si128((__m128i *) &block.b, m);+		memcpy(output, &block.b, part_block_len);+	}+	/* store back IV & tag */+	__m128i tmp = _mm_shuffle_epi8(iv, bswap_mask);+	_mm_storeu_si128((__m128i *) &gcm->civ, tmp);+	_mm_storeu_si128((__m128i *) &gcm->tag, tag);+}

@@ -84,4 +84,16 @@ 		b->q[1] = cpu_to_be64(v); } +#ifdef IMPL_DEBUG+#include <stdio.h>+static inline void block128_print(block128 *b)+{+	int i;+	for (i = 0; i < 16; i++) {+		printf("%02x ", b->b[i]);+	}+	printf("\n");+}+#endif+ #endif

@@ -54,18 +54,22 @@ }  #ifdef USE_AESNI-int have_aesni(void)+void initialize_hw(void (*init_table)(int, int)) {-	static int v = -1;-	if (v == -1) {+	static int inited = 0;+	if (inited == 0) { 		uint32_t eax, ebx, ecx, edx;+		int aesni, pclmul;++		inited = 1; 		cpuid(1, &eax, &ebx, &ecx, &edx);-		v = (ecx & 0x02000000);+		aesni = (ecx & 0x02000000);+		pclmul = (ecx & 0x00000001);+		init_table(aesni, pclmul); 	}-	return v; } #else-#define have_aesni() 	(0)+#define initialize_hw(init_table) 	(0) #endif  #endif

@@ -37,9 +37,9 @@ #endif  #ifdef USE_AESNI-int have_aesni(void);+void initialize_hw(void (*init_table)(int, int)); #else-#define have_aesni() 	(0)+#define initialize_hw(init_table) 	(0) #endif  #endif

@@ -1,5 +1,5 @@ Name:                cipher-aes-Version:             0.1.8+Version:             0.2.0 Description:     Fast AES cipher implementation with advanced mode of operations.     .@@ -12,7 +12,7 @@     .     The software implementation uses S-Boxes, which might suffer for cache timing issues.     However do notes that most other known software implementations, including very popular-    one (openssl, gnutls) also uses same implementation. If it matters for your+    one (openssl, gnutls) also uses similar implementation. If it matters for your     case, you should make sure you have AES-NI available, or you'll need to use a different     implementation.     .@@ -28,12 +28,16 @@ Cabal-Version:       >=1.8 Extra-Source-Files:  Tests/*.hs                      cbits/*.h+                     cbits/aes_x86ni_impl.c  Library   Build-Depends:     base >= 4 && < 5                    , bytestring+                   , byteable+                   , crypto-cipher-types+                   , securemem >= 0.1.2   Exposed-modules:   Crypto.Cipher.AES-  ghc-options:       -Wall+  ghc-options:       -Wall -optc-O3 -fno-cse -fwarn-tabs   C-sources:         cbits/aes_generic.c                      cbits/aes.c                      cbits/gf.c@@ -48,7 +52,10 @@   Main-Is:           Tests.hs   Build-depends:     base >= 4 && < 5                    , cipher-aes+                   , crypto-cipher-types+                   , crypto-cipher-tests                    , bytestring+                   , byteable                    , QuickCheck >= 2                    , test-framework >= 0.3.3                    , test-framework-quickcheck2 >= 0.2.9@@ -60,6 +67,8 @@   Build-depends:     base >= 4 && < 5                    , bytestring                    , cipher-aes+                   , crypto-cipher-types+                   , crypto-cipher-benchmarks                    , criterion                    , mtl