crypton 2.1.10 → 2.2.0

raw patch · 65 files changed

+14452/−894 lines, 65 filesdep +unixdep ~bytestringPVP ok

version bump matches the API change (PVP)

Dependencies added: unix

Dependency ranges changed: bytestring

API changes (from Hackage documentation)

Files

• CHANGELOG.md +42/−0
• Crypto/PubKey/RSA/PKCS15.hs +23/−0
• Crypto/Random.hs +139/−12
• Crypto/Random/Entropy.hs +25/−4
• Crypto/Random/Entropy/Backend.hs +18/−6
• Crypto/Random/Entropy/RDRand.hs +0/−39
• Crypto/Random/Entropy/Source.hs +24/−0
• Crypto/Random/Entropy/SysRandom.hs +50/−0
• Crypto/Random/Entropy/Unix.hs +1/−1
• Crypto/Random/Entropy/Unsafe.hs +8/−6
• Crypto/Random/Entropy/Windows.hs +5/−2
• Crypto/Random/SysDRG.hs +82/−0
• Crypto/Random/Types.hs +7/−1
• Crypto/System/CPU.hs +185/−41
• Crypto/Tutorial.hs +262/−47
• README.md +40/−19
• cbits/aes/generic.c +141/−370
• cbits/aes/generic.h +32/−0
• cbits/aes/gf.c +24/−98
• cbits/aes/ppc8.c +287/−0
• cbits/aes/ppc8.h +29/−0
• cbits/asm/README.md +3/−1
• cbits/asm/aesp8-ppc-linux64le.S +3658/−0
• cbits/asm/aesp8-ppc.pl +3801/−0
• cbits/asm/generate.sh +23/−0
• cbits/asm/ghashp8-ppc-linux64le.S +575/−0
• cbits/asm/ghashp8-ppc.pl +664/−0
• cbits/asm/ppc-xlate.pl +352/−0
• cbits/bearssl/LICENSE +21/−0
• cbits/bearssl/README.md +57/−0
• cbits/bearssl/VERSION +1/−0
• cbits/bearssl/aes_ct64.c +398/−0
• cbits/bearssl/aes_ct64_dec.c +159/−0
• cbits/bearssl/aes_ct64_enc.c +115/−0
• cbits/bearssl/dec32le.c +38/−0
• cbits/bearssl/ghash_ctmul64.c +154/−0
• cbits/bearssl/import.sh +34/−0
• cbits/bearssl/inner.h +92/−0
• cbits/crypton_aes.c +283/−32
• cbits/crypton_armv8_target.h +13/−1
• cbits/crypton_cpu.c +187/−10
• cbits/crypton_cpu.h +40/−1
• cbits/crypton_rdrand.c +0/−126
• cbits/crypton_sysdrg.c +388/−0
• cbits/crypton_sysrandom.c +119/−0
• cbits/tests/bearssl_diff.c +432/−0
• cbits/tests/ct/README +24/−20
• cbits/tests/ct/ct_aes.c +11/−0
• cbits/tests/ct/known.txt +0/−9
• cbits/tests/ct/run.sh +84/−27
• cbits/tests/endian/endian.c +130/−0
• cbits/tests/endian/run.sh +14/−2
• cbits/tests/endian/vectors.txt +129/−0
• cbits/tests/fuzz/run.sh +4/−1
• cbits/tests/ppc8_diff.c +254/−0
• cbits/tests/sysdrg/run.sh +22/−0
• cbits/tests/sysdrg/sysdrg.c +140/−0
• crypton.cabal +120/−16
• tests/EntropySpec.hs +58/−0
• tests/PubKey/RSASpec.hs +4/−0
• tests/RuntimeSpec.hs +67/−2
• tests/SysDRGSpec.hs +100/−0
• tests/forkprocess/ForkProcess.hs +81/−0
• tests/tutorial/extract.awk +53/−0
• tests/tutorial/run.sh +126/−0

@@ -1,5 +1,47 @@ # CHANGELOG for crypton +## 2.2.0++`ProcessorOption` is no longer a sum of constructors.  It is a number with+pattern synonyms for the names it knows, so it has `Ord` now and has lost+`Enum` and `Data`, and a `case` over the names needs a catch-all.  `RDRAND`+is gone from it, and with it the `support_rdrand` flag.  An AArch64 machine+reports `ARMAES` and `ARMPMULL` where it used to report `AESNI` and `PCLMUL`.++Two more things change under a caller without the type checker saying so:+`MonadRandom IO` draws from a per-thread generator rather than reading the+system on every call, and a missing or failing entropy source raises+`EntropyError` rather than an `ErrorCall` or an `IOException`.++* feat: take entropy from the kernel without a file descriptor+  [#300](https://github.com/kazu-yamamoto/crypton/pull/300)+* feat: a ChaCha20 generator per operating system thread+  [#300](https://github.com/kazu-yamamoto/crypton/pull/300)+* feat: MonadRandom IO goes through the per-thread generator+  [#300](https://github.com/kazu-yamamoto/crypton/pull/300)+* fix: say that the system gave no entropy, rather than error+  [#300](https://github.com/kazu-yamamoto/crypton/pull/300)+* fix: stop using RDRAND at all, since it cannot add anything+  [#300](https://github.com/kazu-yamamoto/crypton/pull/300)+* feat: name every processor feature crypton dispatches on, not three of them+  [#302](https://github.com/kazu-yamamoto/crypton/pull/302)+* fix: an AArch64 machine no longer reports AESNI and PCLMUL+  [#302](https://github.com/kazu-yamamoto/crypton/pull/302)+* chore(rsa): deprecate `sign`, `signSafer` and `verify` in favour of them+  [#305](https://github.com/kazu-yamamoto/crypton/pull/305)+* fix: compute the portable AES and GHASH without a table+  [#308](https://github.com/kazu-yamamoto/crypton/pull/308)+* feat: use the AES instructions on 32-bit ARM+  [#309](https://github.com/kazu-yamamoto/crypton/pull/309)+* test: skip the accelerated constant-time driver where the processor has no instructions+  [#310](https://github.com/kazu-yamamoto/crypton/pull/310)+* feat: use the POWER8 AES and GHASH instructions on ppc64le+  [#311](https://github.com/kazu-yamamoto/crypton/pull/311)+* test: the big-endian harness now asks about AES+  [#312](https://github.com/kazu-yamamoto/crypton/pull/312)+* doc(tutorial): compile the examples, and teach the things people come for+  [#313](https://github.com/kazu-yamamoto/crypton/pull/313)+ ## 2.1.10  * build: ask the processor for AES-NI on every x86 system, not four of them

@@ -512,6 +512,12 @@  -- | sign message using private key, a hash and its ASN1 description --+-- __Deprecated.__  Use 'signDigest', or 'signDigestInfo' where this would+-- have been given 'Nothing'.  The @Maybe hashAlg@ argument says both \"hash+-- it with this\" and \"do not hash it\", and down the first path the value+-- inside the 'Just' is never read: it is there to fix the type, which+-- leaves a caller that has only a type with nothing to pass.+-- -- The blinder is optional and 'Nothing' is accepted, but see t'Blinder' for -- what it covers and when leaving it out is a decision rather than a default. -- 'signSafer' generates one for you.@@ -527,8 +533,14 @@     -- ^ message to sign     -> Either Error ByteString sign blinder hashDescr pk m = dp blinder pk `fmap` makeSignature hashDescr (private_size pk) m+{-# DEPRECATED+    sign+    "Use signDigest, or signDigestInfo when the message is already a DigestInfo"+    #-}  -- | sign message using the private key and by automatically generating a blinder.+--+-- __Deprecated.__  Use 'signSaferDigest', or 'signSaferDigestInfo'. signSafer     :: (HashAlgorithmASN1 hashAlg, MonadRandom m)     => Maybe hashAlg@@ -541,9 +553,16 @@ signSafer hashAlg pk m = do     blinder <- generateBlinder (private_n pk)     return (sign (Just blinder) hashAlg pk m)+{-# DEPRECATED+    signSafer+    "Use signSaferDigest, or signSaferDigestInfo when the message is already a DigestInfo"+    #-}  -- | verify message with the signed message --+-- __Deprecated.__  Use 'verifyDigest', or 'verifyDigestInfo' where this+-- would have been given 'Nothing'.+-- -- Following RFC 8017, the signature is rejected unless it is exactly as long -- as the modulus (section 8.2.2, step 1) and its integer representative is -- below the modulus (section 5.2.2, step 1).  Verification works by@@ -562,6 +581,10 @@     -> Bool verify hashAlg pk m sm =     verifyEncoded pk (makeSignature hashAlg (public_size pk) m) sm+{-# DEPRECATED+    verify+    "Use verifyDigest, or verifyDigestInfo when the message is already a DigestInfo"+    #-}  -- | The two checks of RFC 8017 and the comparison, shared by the three -- verification entry points.  The expected encoding is a thunk and is

@@ -7,35 +7,159 @@ -- Maintainer  : Vincent Hanquez <vincent@snarc.org> -- Stability   : stable -- Portability : good+--+-- Random bytes, drawn either from the system or from a generator whose+-- seed you hold.+--+-- == Drawing from the system+--+-- 'getRandomBytes' in 'IO' is the ordinary way to get bytes nobody can+-- predict.  The length is in bytes and the type is any 'ByteArray', so the+-- result is usually pinned down by where it goes:+--+-- > import Crypto.Random (getRandomBytes)+-- > import Data.ByteString (ByteString)+-- >+-- > nonce <- getRandomBytes 12 :: IO ByteString+--+-- Everything in this library that needs randomness takes it the same way,+-- through a @MonadRandom m =>@ constraint, so running it in 'IO' is the+-- whole of choosing this source:+--+-- > import qualified Crypto.PubKey.RSA as RSA+-- >+-- > (pub, priv) <- RSA.generate 256 0x10001+--+-- Where the operating system offers @getrandom(2)@ or @getentropy(3)@ the+-- bytes come from a ChaCha20 generator belonging to the operating system+-- thread the call runs on, seeded from the system and reseeded as it goes;+-- see "Crypto.Random.SysDRG" for what that is and what it is not.  Where+-- it does not -- Windows, for now -- they come from the system on every+-- call, as they always did.  Either way the caller writes the same line.+--+-- Any Haskell thread may draw, and none has to say which generator it+-- wants:+--+-- > import Control.Concurrent (forkIO)+-- >+-- > mapM_ (\_ -> forkIO (getRandomBytes 32 >>= use)) [1 .. 64 :: Int]+--+-- The generator belongs to the operating system thread that happens to be+-- carrying the Haskell thread when the call is made, and is found through+-- that thread's own storage rather than by anything the caller passes.  A+-- @forkIO@ thread moves between capabilities, so two draws from one Haskell+-- thread may be answered by two generators; they are seeded independently+-- of each other, so it makes no difference which one answers.+--+-- == Drawing from a generator you hold+--+-- A generator of your own is the way to draw many times from one seed.+-- The system is asked once, when the generator is made, and not again.+-- That is how @tls@ does a connection: 'seedNew' when it opens, and+-- everything the handshake needs afterwards from the generator, whose+-- state the connection carries from one draw to the next.+--+-- > import Crypto.Random (ChaChaDRG, drgNewSeed, seedNew, withDRG)+-- >+-- > data Connection = Connection { connRNG :: ChaChaDRG }+-- >+-- > newConnection :: IO Connection+-- > newConnection = do+-- >     seed <- seedNew                   -- the only draw from the system+-- >     return (Connection (drgNewSeed seed))+-- >+-- > -- every later draw advances the connection's own generator+-- > connRandom :: Int -> Connection -> (ByteString, Connection)+-- > connRandom n conn =+-- >     let (bytes, rng') = withDRG (connRNG conn) (getRandomBytes n)+-- >      in (bytes, conn{connRNG = rng'})+--+-- Inside 'withDRG' the same 'getRandomBytes' resolves to the instance for+-- 'MonadPseudoRandom' rather than the one for 'IO', so it touches neither+-- the system nor the per-thread generator: it advances the generator it was+-- given and hands it back.  Keep that generator and the bytes are+-- reproducible from the seed, which is what makes a test repeatable -- and+-- what makes a generator unfit for keys unless its seed came from the+-- system.+--+-- 'drgNew' is 'seedNew' and 'drgNewSeed' in one step; 'drgNewTest' takes+-- four numbers instead of a seed, for a test that must give the same answer+-- twice.+--+-- == Drawing from a monad stacked on IO+--+-- There are two instances, one for 'IO' and one for 'MonadPseudoRandom',+-- and none for the transformers, so a @ReaderT env IO@ or a @StateT s IO@+-- reaches the system through 'liftIO':+--+-- > import Control.Monad.IO.Class (liftIO)+-- > import Control.Monad.Trans.Reader (ReaderT)+-- >+-- > newKey :: ReaderT env IO ByteString+-- > newKey = liftIO (getRandomBytes 32)+--+-- What that draws is what the first section describes, no more and no less:+-- the 'IO' instance, the generator of the operating system thread the call+-- lands on, seeded from the system and reseeded as it goes.  'liftIO' says+-- which monad the draw happens in and nothing about where the bytes come+-- from.+--+-- Writing an instance for the stack instead would make the choice invisible+-- at the call site, and the class cannot tell a strong source from a weak+-- one -- see 'MonadRandom'.+--+-- == When the system will not give any+--+-- Drawing randomness is the one thing here with nothing to fall back on,+-- so the failure is an exception rather than a value: there is no sensible+-- 'Maybe' to return and no partial answer worth having.  Since 2.2 it is an+-- 'EntropyError' and not a @Control.Exception.ErrorCall@ or an+-- @Control.Exception.IOException@, which is worth knowing if you were+-- catching one of those:+--+-- > import Control.Exception (catch)+-- > import Crypto.Random (EntropyError (..), getRandomBytes)+-- >+-- > key <- getRandomBytes 32 `catch` \e -> case e of+-- >     NoEntropySource   -> fail "this system offers no randomness at all"+-- >     EntropyShort w g  -> fail (show w ++ " bytes wanted, " ++ show g ++ " arrived")+-- >     EntropySourceLost s -> fail ("the source " ++ s ++ " went away")+--+-- Catching it at all is a decision rather than a default.  A program that+-- cannot get randomness cannot make a key, and stopping is usually the+-- honest thing; the reason to catch is to say so in the program's own+-- terms rather than in crypton's. module Crypto.Random (-    -- * Deterministic instances-    ChaChaDRG,-    SystemDRG,-    Seed,+    -- * Drawing from the system+    MonadRandom (..), -    -- * Seed+    -- * Drawing from a generator you hold+    Seed,     seedNew,     seedFromInteger,     seedToInteger,     seedFromBinary,--    -- * Deterministic Random class-    getSystemDRG,-    drgNew,     drgNewSeed,+    drgNew,     drgNewTest,     withDRG,     withRandomBytes,+    MonadPseudoRandom,     DRG (..), -    -- * Random abstraction-    MonadRandom (..),-    MonadPseudoRandom,+    -- * The generators+    ChaChaDRG,+    SystemDRG,+    getSystemDRG,++    -- * When the system will not give any+    EntropyError (..), ) where  import Crypto.Error import Crypto.Internal.Imports import Crypto.Random.ChaChaDRG+import Crypto.Random.Entropy (EntropyError (..)) import Crypto.Random.SystemDRG import Crypto.Random.Types import Data.ByteArray (ByteArray, ByteArrayAccess, ScrubbedBytes)@@ -50,6 +174,9 @@ import Foreign.Ptr (Ptr, castPtr) #endif +-- | The material a deterministic generator is built from.  Two generators+-- made from one seed produce the same bytes, which is what 'drgNewSeed' is+-- for and why a seed kept anywhere is as good as the keys drawn from it. newtype Seed = Seed ScrubbedBytes     deriving (ByteArrayAccess) 

@@ -6,16 +6,37 @@ -- Portability : Good module Crypto.Random.Entropy (     getEntropy,+    EntropyError (..), ) where  import Crypto.Internal.ByteArray (ByteArray) import qualified Crypto.Internal.ByteArray as B-import Data.Maybe (catMaybes)+import System.IO.Unsafe (unsafeInterleaveIO, unsafePerformIO)  import Crypto.Random.Entropy.Unsafe +-- | The backends this system has, worked out once and no further than+-- needed.+--+-- Opening one is not free: for a device file it means opening and closing+-- @\/dev\/random@ or @\/dev\/urandom@ just to learn that it is there.  This+-- used to be done on every call, and for the whole list before any backend+-- was asked for a byte, so a program paid for both devices even when the+-- first backend answered everything.+--+-- Once, now, and lazily: 'replenish' stops as soon as the buffer is full,+-- which leaves the tail of this list unforced, so a system where the first+-- backend answers never opens a device at all.+{-# NOINLINE openedBackends #-}+openedBackends :: [EntropyBackend]+openedBackends = unsafePerformIO (openAsNeeded supportedBackends)+  where+    openAsNeeded [] = return []+    openAsNeeded (o : os) = do+        m <- o+        rest <- unsafeInterleaveIO (openAsNeeded os)+        return $ maybe rest (: rest) m+ -- | Get some entropy from the system source of entropy getEntropy :: ByteArray byteArray => Int -> IO byteArray-getEntropy n = do-    backends <- catMaybes `fmap` sequence supportedBackends-    B.alloc n (replenish n backends)+getEntropy n = B.alloc n (replenish n openedBackends)

@@ -11,27 +11,39 @@     ( EntropyBackend     , supportedBackends     , gatherBackend+    , EntropyError(..)     ) where  import Foreign.Ptr import Data.Proxy import Data.Word (Word8) import Crypto.Random.Entropy.Source-#ifdef SUPPORT_RDRAND-import Crypto.Random.Entropy.RDRand-#endif #ifdef WINDOWS import Crypto.Random.Entropy.Windows #else+import Crypto.Random.Entropy.SysRandom import Crypto.Random.Entropy.Unix #endif --- | All supported backends +-- | All supported backends, best first.+--+-- The system call comes before everything else: it is the kernel's own+-- generator, it needs no descriptor, and it is what the rest of the world+-- reaches for now.  The device files are what is left when the system has+-- no such call.+--+-- RDRAND is deliberately not here, though it used to be first on x86.  A+-- list like this one is a list of alternatives, and whichever answers+-- first decides the bytes on its own -- which is the one thing #298 says+-- RDRAND should not do.  Nor does it contribute anywhere else: every+-- system this builds for already feeds the instruction into the pool the+-- call above draws from, so asking it again adds nothing.  See the note in+-- @cbits\/crypton_sysdrg.c@. supportedBackends :: [IO (Maybe EntropyBackend)] supportedBackends =     [-#ifdef SUPPORT_RDRAND-    openBackend (Proxy :: Proxy RDRand),+#ifndef WINDOWS+    openBackend (Proxy :: Proxy SysRandom), #endif #ifdef WINDOWS     openBackend (Proxy :: Proxy WinCryptoAPI)

@@ -1,39 +0,0 @@-{-# LANGUAGE ForeignFunctionInterface #-}---- |--- Module      : Crypto.Random.Entropy.RDRand--- License     : BSD-style--- Maintainer  : Vincent Hanquez <vincent@snarc.org>--- Stability   : experimental--- Portability : Good-module Crypto.Random.Entropy.RDRand (-    RDRand,-) where--import Crypto.Random.Entropy.Source-import Data.Word (Word8)-import Foreign.C.Types-import Foreign.Ptr--foreign import ccall unsafe "crypton_cpu_has_rdrand"-    c_cpu_has_rdrand :: IO CInt--foreign import ccall unsafe "crypton_get_rand_bytes"-    c_get_rand_bytes :: Ptr Word8 -> CInt -> IO CInt---- | Fake handle to Intel RDRand entropy CPU instruction-data RDRand = RDRand--instance EntropySource RDRand where-    entropyOpen = rdrandGrab-    entropyGather _ = rdrandGetBytes-    entropyClose _ = return ()--rdrandGrab :: IO (Maybe RDRand)-rdrandGrab = supported `fmap` c_cpu_has_rdrand-  where-    supported 0 = Nothing-    supported _ = Just RDRand--rdrandGetBytes :: Ptr Word8 -> Int -> IO Int-rdrandGetBytes ptr sz = fromIntegral `fmap` c_get_rand_bytes ptr (fromIntegral sz)

@@ -6,8 +6,32 @@ -- Portability : Good module Crypto.Random.Entropy.Source where +import Control.Exception (Exception) import Data.Word (Word8) import Foreign.Ptr++-- | The system would not give the entropy it was asked for.+--+-- This is the one failure in the library with nothing to fall back on and+-- nothing sensible to return: a key drawn from bytes that are not random+-- is worse than no key.  It used to be reported with 'error' and 'fail',+-- which left a caller no way to tell it from a bug in the library, and no+-- way to say anything useful about it.+data EntropyError+    = -- | The system offers no source of entropy at all.  On Unix that+      -- means no @getrandom(2)@, no @getentropy(3)@ and no @\/dev@ to read+      -- from; a container built from nothing would look like this.+      NoEntropySource+    | -- | The sources between them gave fewer bytes than were asked for,+      -- three times over.  The two numbers are how many were wanted and+      -- how many arrived.+      EntropyShort Int Int+    | -- | A source that could be opened once could not be opened again.+      -- The string names it.+      EntropySourceLost String+    deriving (Show, Eq)++instance Exception EntropyError  -- | A handle to an entropy maker, either a system capability -- or a hardware generator.

@@ -0,0 +1,50 @@+{-# LANGUAGE ForeignFunctionInterface #-}++-- |+-- Module      : Crypto.Random.Entropy.SysRandom+-- License     : BSD-style+-- Maintainer  : Kazu Yamamoto <kazu@iij.ad.jp>+-- Stability   : experimental+-- Portability : Unix+--+-- The kernel's own generator, reached without a file descriptor:+-- @getrandom(2)@ on Linux and FreeBSD, @getentropy(3)@ where that is what+-- the system has.+--+-- This is the source to prefer over reading @\/dev\/urandom@.  It needs no+-- path and no descriptor, so it still answers where @\/dev@ is not mounted+-- or not populated, and it cannot be defeated by a full descriptor table.+module Crypto.Random.Entropy.SysRandom (+    SysRandom,+) where++import Crypto.Random.Entropy.Source+import Data.Word (Word8)+import Foreign.C.Types+import Foreign.Ptr++-- Both are @safe@ rather than @unsafe@: at early boot, before the kernel+-- pool is initialised, these calls block, and an unsafe call that blocks+-- holds the capability it runs on.+foreign import ccall safe "crypton_sysrandom_available"+    c_sysrandom_available :: IO CInt++foreign import ccall safe "crypton_sysrandom_bytes"+    c_sysrandom_bytes :: Ptr Word8 -> CInt -> IO CInt++-- | The system call, where there is one.+data SysRandom = SysRandom++instance EntropySource SysRandom where+    -- Asked at run time and not only at compile time: a binary built where+    -- the header declares the call can still run on a kernel that answers+    -- ENOSYS.+    entropyOpen = available `fmap` c_sysrandom_available+      where+        available 0 = Nothing+        available _ = Just SysRandom++    entropyGather _ ptr n =+        fromIntegral `fmap` c_sysrandom_bytes ptr (fromIntegral n)++    entropyClose _ = return ()

@@ -61,7 +61,7 @@ withDev filepath f =     openDev filepath >>= \h ->         case h of-            Nothing -> error ("device " ++ filepath ++ " cannot be grabbed")+            Nothing -> E.throwIO (EntropySourceLost filepath)             Just fd -> f fd `E.finally` closeDev fd  closeDev :: H -> IO ()

@@ -9,25 +9,27 @@     module Crypto.Random.Entropy.Backend, ) where +import Control.Exception (throwIO)+ import Crypto.Random.Entropy.Backend import Data.Word (Word8) import Foreign.Ptr (Ptr, plusPtr)  -- | Refill the entropy in a buffer ----- Call each entropy backend in turn until the buffer has--- been replenished.+-- Call each entropy backend in turn until the buffer has been replenished. ----- If the buffer cannot be refill after 3 loopings, this will raise--- an User Error exception+-- Throws 'EntropyError': 'NoEntropySource' when there is no backend at all,+-- and 'EntropyShort' when three passes over the backends still leave the+-- buffer unfilled. replenish :: Int -> [EntropyBackend] -> Ptr Word8 -> IO ()-replenish _ [] _ = fail "crypton: random: cannot get any source of entropy on this system"+replenish _ [] _ = throwIO NoEntropySource replenish poolSize backends ptr = loop 0 backends ptr poolSize   where     loop :: Int -> [EntropyBackend] -> Ptr Word8 -> Int -> IO ()     loop _ _ _ 0 = return ()     loop retry [] p n-        | retry == 3 = error "crypton: random: cannot fully replenish"+        | retry == 3 = throwIO $ EntropyShort poolSize (poolSize - n)         | otherwise = loop (retry + 1) backends p n     loop retry (b : bs) p n = do         r <- gatherBackend b p n

@@ -23,6 +23,7 @@ import Foreign.Storable (peek) import System.Win32.Types (getLastError) +import Control.Exception (throwIO) import Crypto.Random.Entropy.Source  @@ -38,7 +39,8 @@         case mctx of             Nothing  -> do                 lastError <- getLastError-                fail $ "cannot re-grab win crypto api: error " ++ show lastError+                throwIO $ EntropySourceLost+                    ("the Windows crypto API: error " ++ show lastError)             Just ctx -> do                 r <- cryptGenRandom ctx ptr n                 cryptReleaseCtx ctx@@ -100,4 +102,5 @@         then return ()         else do             lastError <- getLastError-            fail $ "cryptReleaseCtx: error " ++ show lastError+            throwIO $ EntropySourceLost+                ("cryptReleaseCtx: error " ++ show lastError)

@@ -0,0 +1,82 @@+{-# LANGUAGE ForeignFunctionInterface #-}++-- |+-- Module      : Crypto.Random.SysDRG+-- License     : BSD-style+-- Maintainer  : Kazu Yamamoto <kazu@iij.ad.jp>+-- Stability   : experimental+-- Portability : Unix, Windows+--+-- The generator behind the 'Crypto.Random.MonadRandom' instance for 'IO'.+--+-- A ChaCha20 generator per operating system thread, seeded from a+-- process-wide generator, seeded in turn from the system entropy pool.+-- The state and the reseeding live in+-- @cbits\/crypton_sysdrg.c@, because a @forkIO@ thread is not an operating+-- system thread -- it moves between capabilities -- so state held against+-- one would be shared by threads running at the same time.+--+-- == What it is, and what it is not+--+-- It is not a DRBG of NIST SP 800-90A.  That standard names three --+-- @Hash_DRBG@, @HMAC_DRBG@ and @CTR_DRBG@ -- and none of them is built on a+-- stream cipher.  The name here is the older and looser sense of the word.+--+-- What it is is the shape @arc4random@ on OpenBSD and @get_random_bytes@ in+-- the Linux kernel both have, and the one asked for in+-- <https://github.com/kazu-yamamoto/crypton/issues/298>.  That shape is+-- common; it is not specified anywhere, so the parts a standard would have+-- fixed were chosen here instead:+--+-- * ChaCha20, rather than AES in counter mode.+-- * SHA-512 over the system's bytes, rather than a derivation function a+--   standard would have named.+-- * A mebibyte per thread, and a mebibyte of issued seed for the process+--   generator, as the points at which to reseed.  Those numbers are a+--   choice, not a result.+--+-- The pieces underneath are specified: ChaCha20 is RFC 8439, SHA-512 is+-- FIPS 180-4.  The way they are put together is not.+--+-- == What a compromised state gives away+--+-- Each draw ends by taking the next forty bytes of keystream as the key and+-- nonce and dropping the ones that made the output, so a state read after a+-- draw is not the state that produced it.  ChaCha20 does not run backwards+-- and the key that would be needed is gone, which is backtracking+-- resistance in the terms of SP 800-90A.  Without it, a key stands until+-- the next reseed and anyone holding the state can wind the counter back+-- over everything issued since -- a mebibyte of output that was meant to be+-- secret.  It is what @arc4random@ does, and for the same reason.+--+-- Prediction resistance is what the reseeding gives.  A reseed draws from+-- the system again, so a state that has been read does not determine what+-- comes after one.+module Crypto.Random.SysDRG (+    sysDRGBytes,+) where++import Data.Word (Word8)+import Foreign.C.Types (CInt (..))+import Foreign.Ptr (Ptr)++import Crypto.Internal.ByteArray (ByteArray)+import qualified Crypto.Internal.ByteArray as B++-- Safe, not unsafe: seeding can reach a getrandom(2) that blocks until the+-- kernel pool is ready, and an unsafe call that blocks holds the capability+-- it runs on.+foreign import ccall safe "crypton_sysdrg_bytes"+    c_sysdrg_bytes :: Ptr Word8 -> CInt -> IO CInt++-- | Draw bytes, or 'Nothing' if the generator cannot be seeded.+--+-- It cannot be seeded where the system has no @getrandom(2)@ or+-- @getentropy(3)@; the caller falls back to+-- 'Crypto.Random.Entropy.getEntropy' there, which is the path this system+-- had before the generator existed.+sysDRGBytes :: ByteArray byteArray => Int -> IO (Maybe byteArray)+sysDRGBytes n = do+    (got, out) <- B.allocRet n $ \ptr ->+        fromIntegral `fmap` c_sysdrg_bytes ptr (fromIntegral n)+    return $ if got == n then Just out else Nothing

@@ -13,6 +13,7 @@  import Crypto.Internal.ByteArray import Crypto.Random.Entropy+import Crypto.Random.SysDRG (sysDRGBytes)  -- | A monad constraint that allows to generate random bytes --@@ -35,8 +36,13 @@     -- | Generate N bytes of randomness from a DRG     randomBytesGenerate :: ByteArray byteArray => Int -> gen -> (byteArray, gen) +-- | Through the generator of 'Crypto.Random.SysDRG': a ChaCha20 generator+-- per operating system thread, reseeded from the system.  Where that+-- cannot be seeded -- a system with no @getrandom(2)@ or @getentropy(3)@,+-- which includes Windows for now -- this is the system entropy source+-- directly, as it was before the generator existed. instance MonadRandom IO where-    getRandomBytes = getEntropy+    getRandomBytes n = sysDRGBytes n >>= maybe (getEntropy n) return  -- | A simple Monad class very similar to a State Monad -- with the state being a DRG.

@@ -1,6 +1,6 @@ {-# LANGUAGE CPP #-}-{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE ForeignFunctionInterface #-}+{-# LANGUAGE PatternSynonyms #-}  -- | -- Module      : Crypto.System.CPU@@ -11,57 +11,201 @@ -- -- Gives information about crypton runtime environment. module Crypto.System.CPU (-    ProcessorOption (..),+    -- The names are bundled with the type rather than listed as+    -- `pattern' exports, so an importer writes ProcessorOption (..), or+    -- names the ones it wants, as it would for a type with constructors.+    ProcessorOption (+        -- x86+        AESNI,+        PCLMUL,+        SSSE3,+        AVX,+        AVX2,+        SHANI,+        MOVBE,+        ADX,+        VAES,+        VAES512,+        -- AArch64+        NEON,+        ARMAES,+        ARMPMULL,+        ARMSHA1,+        ARMSHA2,+        ARMSHA512,+        -- PowerISA+        PPCAES,+        PPCVPMSUM+    ),     processorOptions,-) where -import Data.Data-import Data.List (findIndices)-#ifdef SUPPORT_RDRAND-import Data.Maybe (isJust)-#endif-import Data.Word (Word8)-import Foreign.Ptr-import Foreign.Storable+    -- * Questions that do not name an architecture+    hasAESAcceleration,+    hasGHASHAcceleration,+) where +import Control.Monad (filterM)+import Data.List (sort)+import Data.Word (Word16)+import Foreign.C.Types (CInt (..), CUInt (..)) import Crypto.Internal.Compat -#ifdef SUPPORT_RDRAND-import Crypto.Random.Entropy.RDRand-import Crypto.Random.Entropy.Source-#endif+-- | A processor feature crypton looked for, and dispatches on where it+-- finds it.+--+-- This is a number with names rather than a sum of constructors, and the+-- names are pattern synonyms with no @COMPLETE@ pragma, so a @case@ over+-- them needs a catch-all and a feature named in a later release breaks+-- nothing that compiled against this one.  The same reason 'Show' is+-- written out below: a program built against an older crypton still says+-- something useful about a value from a newer one.+--+-- The names are the processor's, not the operation's.  'AESNI' is x86's+-- and 'ARMAES' is AArch64's, and a machine reports only the ones it has;+-- ask 'hasAESAcceleration' if the question is whether AES is fast here.+--+-- They are bundled with the type in the export list, so @ProcessorOption+-- (..)@ brings in all of them and naming one brings in that one, as for a+-- type with constructors.  The constructor underneath is not exported:+-- these values say what the processor was found to have, and a caller has+-- nothing to build.+newtype ProcessorOption = ProcessorOption Word16+    deriving (Eq, Ord) --- | CPU options impacting cryptography implementation and library performance.-data ProcessorOption-    = -- | Support for AES instructions, with flag @support_aesni@-      AESNI-    | -- | Support for CLMUL instructions, with flag @support_pclmuldq@-      PCLMUL-    | -- | Support for RDRAND instruction, with flag @support_rdrand@-      RDRAND-    deriving (Show, Eq, Enum, Data)+-- | Support for AES instructions, with flag @support_aesni@.+pattern AESNI :: ProcessorOption+pattern AESNI = ProcessorOption 0 +-- | Support for CLMUL instructions, with flag @support_pclmuldq@.+pattern PCLMUL :: ProcessorOption+pattern PCLMUL = ProcessorOption 1++-- | Supplemental SSE3.+pattern SSSE3 :: ProcessorOption+pattern SSSE3 = ProcessorOption 3++-- | AVX, and an operating system that saves its registers.+pattern AVX :: ProcessorOption+pattern AVX = ProcessorOption 4++-- | AVX2, and an operating system that saves its registers.+pattern AVX2 :: ProcessorOption+pattern AVX2 = ProcessorOption 5++-- | The SHA extensions, @sha1rnds4@ and @sha256rnds2@ and their neighbours.+pattern SHANI :: ProcessorOption+pattern SHANI = ProcessorOption 6++-- | The byte-swapping load.+pattern MOVBE :: ProcessorOption+pattern MOVBE = ProcessorOption 7++-- | @MULX@, @ADCX@ and @ADOX@: the two independent carry chains.+pattern ADX :: ProcessorOption+pattern ADX = ProcessorOption 8++-- | The AES and carry-less multiply instructions in their 256-bit form.+pattern VAES :: ProcessorOption+pattern VAES = ProcessorOption 9++-- | The same pair in their 512-bit form.+pattern VAES512 :: ProcessorOption+pattern VAES512 = ProcessorOption 10++-- | Advanced SIMD, which is not optional on AArch64.+pattern NEON :: ProcessorOption+pattern NEON = ProcessorOption 11++-- | The ARMv8 AES instructions.+pattern ARMAES :: ProcessorOption+pattern ARMAES = ProcessorOption 12++-- | @PMULL@, the ARMv8 carry-less multiply.+pattern ARMPMULL :: ProcessorOption+pattern ARMPMULL = ProcessorOption 13++-- | The ARMv8 SHA-1 instructions.+pattern ARMSHA1 :: ProcessorOption+pattern ARMSHA1 = ProcessorOption 14++-- | The ARMv8 SHA-256 instructions.+pattern ARMSHA2 :: ProcessorOption+pattern ARMSHA2 = ProcessorOption 15++-- | The ARMv8.2 SHA-512 instructions, which are optional where SHA-256's+-- are not.+pattern ARMSHA512 :: ProcessorOption+pattern ARMSHA512 = ProcessorOption 16++-- | Support for the PowerISA 2.07 vector AES instructions, which POWER8 was+-- the first to implement.+pattern PPCAES :: ProcessorOption+pattern PPCAES = ProcessorOption 17++-- | Support for @vpmsumd@, the vector carry-less multiply that came with+-- them, which is what makes GHASH fast.+pattern PPCVPMSUM :: ProcessorOption+pattern PPCVPMSUM = ProcessorOption 18++-- | Named where the name is known, numbered where it is not, so that a+-- binary built against an older crypton can still print a value a newer one+-- produced.+instance Show ProcessorOption where+    show AESNI = "AESNI"+    show PCLMUL = "PCLMUL"+    show SSSE3 = "SSSE3"+    show AVX = "AVX"+    show AVX2 = "AVX2"+    show SHANI = "SHANI"+    show MOVBE = "MOVBE"+    show ADX = "ADX"+    show VAES = "VAES"+    show VAES512 = "VAES512"+    show NEON = "NEON"+    show ARMAES = "ARMAES"+    show ARMPMULL = "ARMPMULL"+    show ARMSHA1 = "ARMSHA1"+    show ARMSHA2 = "ARMSHA2"+    show ARMSHA512 = "ARMSHA512"+    show PPCAES = "PPCAES"+    show PPCVPMSUM = "PPCVPMSUM"+    show (ProcessorOption n) = "ProcessorOption " ++ show n+ -- | Options which have been enabled at compile time and are supported by the -- current CPU.+--+-- Sorted, and without repeats.  A machine reports the names of its own+-- architecture only: an AArch64 processor with AES says 'ARMAES', not+-- 'AESNI', which it does not have. processorOptions :: [ProcessorOption]-processorOptions = unsafeDoIO $ do-    p <- crypton_aes_cpu_init-    options <- traverse (getOption p) aesOptions-    rdrand <- hasRDRand-    return (decodeOptions options ++ [RDRAND | rdrand])+processorOptions = unsafeDoIO (sort <$> filterM askC allOptions)   where-    aesOptions = [AESNI .. PCLMUL]-    getOption p = peekElemOff p . fromEnum-    decodeOptions = map toEnum . findIndices (> 0)+    -- 2 is not asked for and has no name: it was RDRAND, which crypton no+    -- longer dispatches on.  The number is left out rather than reused, so+    -- that the others keep the values they had.+    allOptions = [ProcessorOption n | n <- [0 .. 18], n /= 2]+    askC (ProcessorOption n) =+        (/= 0) <$> crypton_cpu_option (fromIntegral n) {-# NOINLINE processorOptions #-} -hasRDRand :: IO Bool-#ifdef SUPPORT_RDRAND-hasRDRand = fmap isJust getRDRand-  where getRDRand = entropyOpen :: IO (Maybe RDRand)-#else-hasRDRand = return False-#endif+-- | Is there hardware AES on this machine?+--+-- The instructions have different names on different architectures, and a+-- caller that wants to know whether AES-GCM will be fast wants this rather+-- than either name.+hasAESAcceleration :: Bool+hasAESAcceleration =+    any+        (`elem` processorOptions)+        [AESNI, ARMAES, PPCAES] -foreign import ccall unsafe "crypton_aes_cpu_init"-    crypton_aes_cpu_init :: IO (Ptr Word8)+-- | Is there a hardware carry-less multiply, which is what GHASH, and so+-- AES-GCM, spends its time in once AES itself is fast?+hasGHASHAcceleration :: Bool+hasGHASHAcceleration =+    any+        (`elem` processorOptions)+        [PCLMUL, ARMPMULL, PPCVPMSUM]++foreign import ccall unsafe "crypton_cpu_option"+    crypton_cpu_option :: CUInt -> IO CInt

@@ -1,4 +1,7 @@ -- | Examples of how to use @crypton@.+--+-- Every code block here is extracted and compiled against this version of+-- the library by @tests\/tutorial\/run.sh@, so what is written below builds. module Crypto.Tutorial (     -- * API design     -- $api_design@@ -6,9 +9,21 @@     -- * Hash algorithms     -- $hash_algorithms -    -- * Symmetric block ciphers-    -- $symmetric_block_ciphers+    -- * Authenticated encryption+    -- $authenticated_encryption +    -- * Comparing secrets+    -- $comparing_secrets++    -- * Password storage+    -- $password_storage++    -- * Key derivation+    -- $key_derivation++    -- * Digital signatures+    -- $digital_signatures+     -- * Combining primitives     -- $combining_primitives ) where@@ -31,6 +46,14 @@ -- Error conditions are returned with data type 'Crypto.Error.CryptoFailable'. -- Functions in module "Crypto.Error" can convert those values to runtime -- exceptions, 'Maybe' or 'Either' values.+--+-- Types that hold a secret do not print it.  A private key's 'Show' renders+-- whatever is public and @\<secret\>@ or @\<scrubbed-bytes\>@ for the rest,+-- because 'Show' is what @print@, @error@, an exception and a failing test+-- all reach for, and a key arriving in a log that way is an accident nobody+-- asked for.  "Crypto.Debug" is how one is printed when printing it is what+-- was meant.  Several of those types keep their bytes in+-- 'Data.ByteArray.ScrubbedBytes', which is wiped when it is collected.  -- $hash_algorithms --@@ -90,72 +113,258 @@ -- >     hashMutableUpdate ctx ("dog"    :: ByteString) -- >     hashMutableFinalize ctx >>= print --- $symmetric_block_ciphers+-- $authenticated_encryption --+-- Encrypting hides a message; it does not stop anyone changing it.  Under a+-- counter or stream mode, flipping a bit of the ciphertext flips the same+-- bit of the plaintext, and the receiver has no way to tell.  An AEAD mode+-- binds the message, and anything else named as associated data, to a short+-- authentication tag, and decrypting something that does not match that tag+-- returns nothing at all.+--+-- That is the mode to use.  The unauthenticated ones are in this library+-- for protocols that authenticate separately, not as a starting point.+-- -- > {-# LANGUAGE OverloadedStrings #-}--- > {-# LANGUAGE ScopedTypeVariables #-}--- > {-# LANGUAGE GADTs #-} -- > -- > import           Crypto.Cipher.AES (AES256)--- > import           Crypto.Cipher.Types (BlockCipher(..), Cipher(..), nullIV, KeySizeSpecifier(..), IV, makeIV)--- > import           Crypto.Error (CryptoFailable(..), CryptoError(..))+-- > import           Crypto.Cipher.Types+-- >                      ( AEADMode (AEAD_GCM)+-- >                      , AuthTag+-- >                      , BlockCipher (aeadInit)+-- >                      , Cipher (cipherInit, cipherKeySize)+-- >                      , KeySizeSpecifier (..)+-- >                      , aeadSimpleDecrypt+-- >                      , aeadSimpleEncrypt+-- >                      )+-- > import           Crypto.Error (CryptoError, eitherCryptoError)+-- > import qualified Crypto.Random.Types as CRT -- >+-- > import           Data.ByteArray (ByteArray, ByteArrayAccess, ScrubbedBytes)+-- > import           Data.ByteString (ByteString)+-- >+-- > -- | A key of the length the cipher asks for, rather than a length+-- > -- written out here.  ScrubbedBytes rather than ByteString, so that it+-- > -- is wiped when it is collected and does not print.+-- > genSecretKey :: (Cipher c, CRT.MonadRandom m) => c -> m ScrubbedBytes+-- > genSecretKey c = CRT.getRandomBytes (longest (cipherKeySize c))+-- >   where+-- >     longest (KeySizeFixed n)   = n+-- >     longest (KeySizeRange _ n) = n+-- >     longest (KeySizeEnum ns)   = maximum ns+-- >+-- > -- | A fresh nonce for every message.  GCM must never see one twice+-- > -- under the same key: a repeat does not just expose those two messages,+-- > -- it hands over the key that authenticates all of them.  Twelve random+-- > -- bytes, sent along with the ciphertext.+-- > genNonce :: CRT.MonadRandom m => m ByteString+-- > genNonce = CRT.getRandomBytes 12+-- >+-- > -- | Encrypt and authenticate.  The associated data is authenticated but+-- > -- not encrypted: it is for what the receiver can already see and must+-- > -- not have had altered, such as a header or an address.+-- > encrypt+-- >     :: (ByteArray key, ByteArrayAccess nonce, ByteArrayAccess aad, ByteArray ba)+-- >     => key -> nonce -> aad -> ba -> Either CryptoError (AuthTag, ba)+-- > encrypt key nonce aad plaintext = do+-- >     cipher <- eitherCryptoError (cipherInit key) :: Either CryptoError AES256+-- >     aead <- eitherCryptoError (aeadInit AEAD_GCM cipher nonce)+-- >     return (aeadSimpleEncrypt aead aad plaintext 16)+-- >+-- > -- | And back, with two different failures.  Left is this code used+-- > -- wrongly -- a key of the wrong length, a mode the cipher has not got.+-- > -- Nothing is a message that is not the one that was sent; it carries no+-- > -- plaintext and says nothing about which byte was wrong, both of which+-- > -- are the point.+-- > decrypt+-- >     :: (ByteArray key, ByteArrayAccess nonce, ByteArrayAccess aad, ByteArray ba)+-- >     => key -> nonce -> aad -> AuthTag -> ba -> Either CryptoError (Maybe ba)+-- > decrypt key nonce aad tag ciphertext = do+-- >     cipher <- eitherCryptoError (cipherInit key) :: Either CryptoError AES256+-- >     aead <- eitherCryptoError (aeadInit AEAD_GCM cipher nonce)+-- >     return (aeadSimpleDecrypt aead aad ciphertext tag)+-- >+-- > exampleAES256GCM :: ByteString -> IO ()+-- > exampleAES256GCM msg = do+-- >     key <- genSecretKey (undefined :: AES256)+-- >     nonce <- genNonce+-- >     let aad = "to: alice" :: ByteString+-- >     case encrypt key nonce aad msg of+-- >         Left err -> error (show err)+-- >         Right (tag, ciphertext) -> do+-- >             putStrLn $ "ciphertext: " ++ show ciphertext+-- >             putStrLn $ "       tag: " ++ show tag+-- >             putStrLn $ " recovered: "+-- >                 ++ show (decrypt key nonce aad tag ciphertext)+-- >             -- The same bytes and the same tag, with one thing changed+-- >             -- that was never encrypted: Right Nothing.+-- >             putStrLn $ "redirected: "+-- >                 ++ show (decrypt key nonce ("to: eve" :: ByteString) tag ciphertext)+--+-- The two functions above work for any cipher that has an AEAD mode; what+-- changes is the mode given to 'Crypto.Cipher.Types.aeadInit'.+-- "Crypto.Cipher.ChaChaPoly1305" is the one to prefer on a machine with no+-- AES instructions, and "Crypto.Cipher.AESGCMSIV" is the one that survives+-- a repeated nonce, at the price of needing the whole message before it can+-- begin.++-- $comparing_secrets+--+-- Comparing two byte strings with '==' stops at the first byte that+-- differs, so how long it takes says where that byte was.  Against an+-- authentication tag that is the whole secret: someone who can send a guess+-- and time the answer finds the first byte in a few hundred tries, then the+-- second, and has a tag that was supposed to cost 2^128 in a few thousand.+--+-- crypton's own authentication types already compare in constant time, so+-- for those there is nothing to do: 'Crypto.MAC.HMAC.HMAC',+-- 'Crypto.MAC.CMAC.CMAC', 'Crypto.MAC.Poly1305.Auth' and+-- 'Crypto.Cipher.Types.AuthTag' have an 'Eq' that looks at every byte+-- whatever it finds.  What needs care is a tag that arrives as bytes, and+-- 'Data.ByteArray.constEq' is the comparison for it.+--+-- > {-# LANGUAGE OverloadedStrings #-}+-- >+-- > import           Crypto.Hash.Algorithms (SHA256)+-- > import           Crypto.MAC.HMAC (HMAC, hmac)+-- >+-- > import qualified Data.ByteArray as BA+-- > import           Data.ByteString (ByteString)+-- >+-- > -- | The tag came off the wire as bytes, so it is compared as bytes.+-- > authentic :: ByteString -> ByteString -> ByteString -> Bool+-- > authentic key message tag =+-- >     BA.constEq tag (hmac key message :: HMAC SHA256)+-- >+-- > -- | Once it has been parsed into the library's own type, (==) is+-- > -- already the constant-time comparison.+-- > authentic' :: ByteString -> ByteString -> HMAC SHA256 -> Bool+-- > authentic' key message tag = tag == hmac key message++-- $password_storage+--+-- A password is not a key.  It is short and it is guessable, and whoever+-- takes the database can try every likely one without being watched.  What+-- answers that is a function that is deliberately expensive to compute, and+-- crypton has four: "Crypto.KDF.BCrypt", "Crypto.KDF.Scrypt",+-- "Crypto.KDF.Argon2" and "Crypto.KDF.PBKDF2".  A plain hash is not one of+-- them, however many times it is applied by hand.+--+-- bcrypt leaves the least to get wrong, because the record it returns+-- carries the salt and the cost inside it:+--+-- > import Crypto.KDF.BCrypt (hashPassword, validatePassword)+-- >+-- > import Data.ByteString (ByteString)+-- >+-- > -- | What goes in the database.  The salt is drawn inside and ends up in+-- > -- the result, so two accounts with the same password do not look alike.+-- > register :: ByteString -> IO ByteString+-- > register password = hashPassword 12 password+-- >+-- > -- | And what is checked against it.  The cost comes out of the stored+-- > -- record, so raising it for new accounts leaves the old ones working.+-- > login :: ByteString -> ByteString -> Bool+-- > login password stored = validatePassword password stored+--+-- Argon2 is the stronger choice and the one to pick for something new: it+-- asks for memory as well as time, which is what takes the advantage away+-- from the hardware that bcrypt's small working set leaves room for.+-- Nothing is encoded for the caller, though -- the salt and the options are+-- theirs to store, and without all three the hash cannot be recomputed when+-- the user comes back.+--+-- > import           Crypto.Error (CryptoFailable)+-- > import qualified Crypto.KDF.Argon2 as Argon2 -- > import qualified Crypto.Random.Types as CRT -- >--- > import           Data.ByteArray (ByteArray) -- > import           Data.ByteString (ByteString) -- >--- > -- | Not required, but most general implementation--- > data Key c a where--- >   Key :: (BlockCipher c, ByteArray a) => a -> Key c a+-- > -- | Argon2id, which is the variant to prefer: it resists both a machine+-- > -- built to guess and a process watching the cache.+-- > options :: Argon2.Options+-- > options = Argon2.defaultOptions{Argon2.variant = Argon2.Argon2id} -- >--- > -- | Generates a string of bytes (key) of a specific length for a given block cipher--- > genSecretKey :: forall m c a. (CRT.MonadRandom m, BlockCipher c, ByteArray a) => c -> Int -> m (Key c a)--- > genSecretKey _ = fmap Key . CRT.getRandomBytes+-- > newSalt :: CRT.MonadRandom m => m ByteString+-- > newSalt = CRT.getRandomBytes 16 -- >--- > -- | Generate a random initialization vector for a given block cipher--- > genRandomIV :: forall m c. (CRT.MonadRandom m, BlockCipher c) => c -> m (Maybe (IV c))--- > genRandomIV _ = do--- >   bytes :: ByteString <- CRT.getRandomBytes $ blockSize (undefined :: c)--- >   return $ makeIV bytes+-- > derive :: ByteString -> ByteString -> CryptoFailable ByteString+-- > derive salt password = Argon2.hash options password salt 32++-- $key_derivation+--+-- HKDF turns one secret into as many keys as a protocol needs.  It is for+-- material that is already unguessable -- what comes out of a+-- Diffie-Hellman, or a key already agreed -- and it is deliberately cheap,+-- which is exactly what makes it the wrong thing for a password.  Those go+-- to the section above.+--+-- The info string is what keeps the outputs independent: the same secret+-- with a different info gives an unrelated key, so each use of a secret+-- names itself there.+--+-- > {-# LANGUAGE OverloadedStrings #-} -- >--- > -- | Initialize a block cipher--- > initCipher :: (BlockCipher c, ByteArray a) => Key c a -> Either CryptoError c--- > initCipher (Key k) = case cipherInit k of--- >   CryptoFailed e -> Left e--- >   CryptoPassed a -> Right a+-- > import           Crypto.Hash.Algorithms (SHA256)+-- > import qualified Crypto.KDF.HKDF as HKDF -- >--- > encrypt :: (BlockCipher c, ByteArray a) => Key c a -> IV c -> a -> Either CryptoError a--- > encrypt secretKey initIV msg =--- >   case initCipher secretKey of--- >     Left e -> Left e--- >     Right c -> Right $ ctrCombine c initIV msg+-- > import           Data.ByteArray (ScrubbedBytes)+-- > import           Data.ByteString (ByteString) -- >--- > decrypt :: (BlockCipher c, ByteArray a) => Key c a -> IV c -> a -> Either CryptoError a--- > decrypt = encrypt+-- > -- | One shared secret in, two unrelated keys out.+-- > directionKeys :: ByteString -> ByteString -> (ScrubbedBytes, ScrubbedBytes)+-- > directionKeys salt shared = (keyFor "client write", keyFor "server write")+-- >   where+-- >     prk = HKDF.extract salt shared :: HKDF.PRK SHA256+-- >     keyFor info = HKDF.expand prk (info :: ByteString) 32++-- $digital_signatures+--+-- Ed25519 is the one to reach for.  The keys are thirty-two bytes, there is+-- nothing to choose and nothing to encode, and signing needs no randomness,+-- so it cannot be ruined by a bad source of it.  "Crypto.PubKey.Ed448" is+-- the same shape at a larger size, "Crypto.PubKey.ECDSA" and+-- "Crypto.PubKey.RSA.PSS" are there for protocols that ask for them, and+-- "Crypto.PubKey.MLDSA" is the post-quantum one.+--+-- > {-# LANGUAGE OverloadedStrings #-} -- >--- > exampleAES256 :: ByteString -> IO ()--- > exampleAES256 msg = do--- >   -- secret key needs 256 bits (32 * 8)--- >   secretKey <- genSecretKey (undefined :: AES256) 32--- >   mInitIV <- genRandomIV (undefined :: AES256)--- >   case mInitIV of--- >     Nothing -> error "Failed to generate and initialization vector."--- >     Just initIV -> do--- >       let encryptedMsg = encrypt secretKey initIV msg--- >           decryptedMsg = decrypt secretKey initIV =<< encryptedMsg--- >       case (,) <$> encryptedMsg <*> decryptedMsg of--- >         Left err -> error $ show err--- >         Right (eMsg, dMsg) -> do--- >           putStrLn $ "Original Message: " ++ show msg--- >           putStrLn $ "Message after encryption: " ++ show eMsg--- >           putStrLn $ "Message after decryption: " ++ show dMsg+-- > import           Crypto.Error (throwCryptoError)+-- > import qualified Crypto.PubKey.Ed25519 as Ed25519+-- >+-- > import qualified Data.ByteArray as BA+-- > import           Data.ByteString (ByteString)+-- >+-- > exampleEd25519 :: ByteString -> IO ()+-- > exampleEd25519 msg = do+-- >     sk <- Ed25519.generateSecretKey+-- >     let pk = Ed25519.toPublic sk+-- >         sig = Ed25519.sign sk pk msg+-- >     print (Ed25519.verify pk msg sig)+-- >     -- The same signature against a message one byte longer: False.+-- >     print (Ed25519.verify pk (msg <> "!") sig)+-- >+-- > -- | A secret key on its way to storage.  Printing one does not reveal+-- > -- it -- see the first section -- so this is the way out.+-- > store :: Ed25519.SecretKey -> ByteString+-- > store = BA.convert+-- >+-- > -- | And the way back in, which is checked, because the bytes read from+-- > -- a file may be anything at all.+-- > load :: ByteString -> Ed25519.SecretKey+-- > load = throwCryptoError . Ed25519.secretKey  -- $combining_primitives -- -- This example shows how to use Curve25519, XSalsa and Poly1305 primitives to -- emulate NaCl's @crypto_box@ construct. --+-- It is here to show how the pieces fit together, not as something to+-- deploy.  An authenticated encryption scheme assembled by hand is the kind+-- of code that is wrong in ways no test notices; for actual use,+-- "Crypto.Cipher.ChaChaPoly1305" does this job with the mistakes already+-- made.+-- -- > import qualified Data.ByteArray as BA -- > import           Data.ByteString (ByteString) -- > import qualified Data.ByteString as B@@ -167,6 +376,9 @@ -- > -- > -- | Build a @crypto_box@ packet encrypting the specified content with a -- > -- 192-bit nonce, receiver public key and sender private key.+-- > crypto_box+-- >     :: ByteString -> ByteString -> X25519.PublicKey -> X25519.SecretKey+-- >     -> ByteString -- > crypto_box content nonce pk sk = BA.convert tag `B.append` c -- >   where -- >     zero         = B.replicate 16 0@@ -181,6 +393,9 @@ -- > -- > -- | Try to open a @crypto_box@ packet and recover the content using the -- > -- 192-bit nonce, sender public key and receiver private key.+-- > crypto_box_open+-- >     :: ByteString -> ByteString -> X25519.PublicKey -> X25519.SecretKey+-- >     -> Maybe ByteString -- > crypto_box_open packet nonce pk sk -- >     | B.length packet < 16 = Nothing -- >     | BA.constEq tag' tag  = Just content

@@ -14,33 +14,49 @@ Side channels ------------- +### AES+ AES is where this matters most, and which implementation runs is decided at runtime from what the processor has. -On x86-64 with AES-NI and carry-less multiply, and on AArch64 with the ARMv8-cryptographic extension, AES and GHASH are instructions rather than tables.-crypton's AES and AES-GCM then make no branch and no memory access that-depends on the key or on the data: the secrets stay in vector registers and-never reach one a branch can test, which the generated code is checked-against.  Every x86-64 part since about 2010 and every AArch64 part in-ordinary use has these.+On x86-64 with AES-NI and carry-less multiply, on AArch64 and on 32-bit ARM+with the ARMv8 cryptographic extension, and on ppc64le with the vector+instructions PowerISA 2.07 brought to POWER8, AES and GHASH are instructions+rather than software.  crypton's AES and AES-GCM then make no branch and no+memory access that depends on the key or on the data: the secrets stay in+vector registers and never reach one a branch can test.  Every x86-64 part+since about 2010 and every AArch64 part in ordinary use has these, and the+ppc64le path is little-endian only. -Where neither is present crypton falls back to a table-driven AES, which-indexes a 256-byte substitution table with data derived from the key and the-input.  **That is not constant time**, and on a machine where an attacker can-observe the cache it is open to a timing attack.  The fallback exists so that-the library builds and runs everywhere; it is not meant for a setting where-that matters.+Where none of them is there, crypton computes AES and GHASH without a table.+The portable AES is bitsliced -- the S-box is boolean algebra over four+blocks held across eight 64-bit words -- and the GF(2^128) multiply is built+from shifts, masks and integer multiplies.  Neither looks anything up, so+neither derives an address from a secret, and **the portable path is constant+time as well**.  The code is BearSSL's, under MIT, in `cbits/bearssl`. -`Crypto.System.CPU.processorOptions` says which is in use.  `AESNI` in that-list means the instruction path, and `PCLMUL` that GHASH has its instruction-too; without `AESNI` it is the tables.  The list also reports `RDRAND`, which-is unrelated to this.+The constant-time harness in `cbits/tests/ct` runs the same driver against+the portable implementation and against each architecture's instructions, and+every one of them is required to report nothing at all. +So the answer does not depend on the machine any more, and what is left to+ask is whether AES is fast on it.  `hasAESAcceleration` and+`hasGHASHAcceleration` answer that without naming an architecture.+`processorOptions` says what was found, in the processor's own names --+`AESNI` and `PCLMUL` on x86, `ARMAES` and `ARMPMULL` on AArch64 and 32-bit+ARM, `PPCAES` and `PPCVPMSUM` on ppc64le -- along with everything else it was+asked about, which is unrelated to this.+     ghci> import Crypto.System.CPU-    ghci> processorOptions-    [AESNI,PCLMUL]+    ghci> hasAESAcceleration+    True+    ghci> processorOptions           -- on an x86-64 machine+    [AESNI,PCLMUL,SSSE3,AVX,AVX2,SHANI,MOVBE,ADX,VAES]+    ghci> processorOptions           -- and on an AArch64 one+    [NEON,ARMAES,ARMPMULL,ARMSHA1,ARMSHA2,ARMSHA512] +### RSA+ RSA is the other place to know about, and there the choice is the caller's. The private key operations in `Crypto.PubKey.RSA.PKCS15`, `.OAEP` and `.PSS` take a `Maybe Blinder`, and `Nothing` is no harder to write than the safe@@ -252,3 +268,8 @@ ask for it, not because it is a good choice for anything new.  The algorithms that nothing should ask for any more -- MD5, 3DES, RC4, CBC mode -- are left out.++ML-KEM and ML-DSA are left out too, though 2.1.8 brought both and they are+fast -- `mlkem-native` and `mldsa-native`, from the PQ Code Package.  No+comparison is reported here: the implementations to measure against are+still developing.

@@ -27,420 +27,191 @@  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF  * SUCH DAMAGE.  *- * AES implementation+ * AES, portable, for machines whose processor has no AES instructions or+ * whose build was not given them.+ *+ * This was a table-driven implementation: an S-box indexed by a byte of the+ * state, in every round and in the key expansion, which is what made it fast+ * and what made it variable-time.  It is now BearSSL's bitsliced aes_ct64,+ * which holds four blocks interleaved across eight 64-bit words and computes+ * the S-box as boolean algebra -- no table, and so no address derived from a+ * secret.  See cbits/bearssl/README.md.+ *+ * What is here is only the glue.  Nothing in this file branches or indexes on+ * the key or the data.  */  #include <stdint.h>-#include <stdlib.h>+#include <string.h> #include <crypton_aes.h>-#include <crypton_bitfn.h>--static uint8_t sbox[256] = {-	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe,-	0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4,-	0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7,-	0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3,-	0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09,-	0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3,-	0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe,-	0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,-	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92,-	0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c,-	0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19,-	0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14,-	0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2,-	0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5,-	0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25,-	0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,-	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86,-	0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e,-	0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42,-	0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16-};--static uint8_t rsbox[256] = {-	0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81,-	0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e,-	0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23,-	0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 0x08, 0x2e, 0xa1, 0x66,-	0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 0x72,-	0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65,-	0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46,-	0x57, 0xa7, 0x8d, 0x9d, 0x84, 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a,-	0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca,-	0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91,-	0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6,-	0x73, 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8,-	0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f,-	0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2,-	0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 0x1f, 0xdd, 0xa8,-	0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,-	0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93,-	0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb,-	0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6,-	0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d-};--static uint8_t Rcon[] = {-	0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8,-	0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3,-	0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f,-	0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb,-};+#include "bearssl/inner.h"+#include "aes/block128.h"+#include "aes/generic.h" -#define G(a,b,c,d,e,f) { a,b,c,d,e,f }-static uint8_t gmtab[256][6] =-{-	G(0x00, 0x00, 0x00, 0x00, 0x00, 0x00), G(0x02, 0x03, 0x09, 0x0b, 0x0d, 0x0e),-	G(0x04, 0x06, 0x12, 0x16, 0x1a, 0x1c), G(0x06, 0x05, 0x1b, 0x1d, 0x17, 0x12),-	G(0x08, 0x0c, 0x24, 0x2c, 0x34, 0x38), G(0x0a, 0x0f, 0x2d, 0x27, 0x39, 0x36),-	G(0x0c, 0x0a, 0x36, 0x3a, 0x2e, 0x24), G(0x0e, 0x09, 0x3f, 0x31, 0x23, 0x2a),-	G(0x10, 0x18, 0x48, 0x58, 0x68, 0x70), G(0x12, 0x1b, 0x41, 0x53, 0x65, 0x7e),-	G(0x14, 0x1e, 0x5a, 0x4e, 0x72, 0x6c), G(0x16, 0x1d, 0x53, 0x45, 0x7f, 0x62),-	G(0x18, 0x14, 0x6c, 0x74, 0x5c, 0x48), G(0x1a, 0x17, 0x65, 0x7f, 0x51, 0x46),-	G(0x1c, 0x12, 0x7e, 0x62, 0x46, 0x54), G(0x1e, 0x11, 0x77, 0x69, 0x4b, 0x5a),-	G(0x20, 0x30, 0x90, 0xb0, 0xd0, 0xe0), G(0x22, 0x33, 0x99, 0xbb, 0xdd, 0xee),-	G(0x24, 0x36, 0x82, 0xa6, 0xca, 0xfc), G(0x26, 0x35, 0x8b, 0xad, 0xc7, 0xf2),-	G(0x28, 0x3c, 0xb4, 0x9c, 0xe4, 0xd8), G(0x2a, 0x3f, 0xbd, 0x97, 0xe9, 0xd6),-	G(0x2c, 0x3a, 0xa6, 0x8a, 0xfe, 0xc4), G(0x2e, 0x39, 0xaf, 0x81, 0xf3, 0xca),-	G(0x30, 0x28, 0xd8, 0xe8, 0xb8, 0x90), G(0x32, 0x2b, 0xd1, 0xe3, 0xb5, 0x9e),-	G(0x34, 0x2e, 0xca, 0xfe, 0xa2, 0x8c), G(0x36, 0x2d, 0xc3, 0xf5, 0xaf, 0x82),-	G(0x38, 0x24, 0xfc, 0xc4, 0x8c, 0xa8), G(0x3a, 0x27, 0xf5, 0xcf, 0x81, 0xa6),-	G(0x3c, 0x22, 0xee, 0xd2, 0x96, 0xb4), G(0x3e, 0x21, 0xe7, 0xd9, 0x9b, 0xba),-	G(0x40, 0x60, 0x3b, 0x7b, 0xbb, 0xdb), G(0x42, 0x63, 0x32, 0x70, 0xb6, 0xd5),-	G(0x44, 0x66, 0x29, 0x6d, 0xa1, 0xc7), G(0x46, 0x65, 0x20, 0x66, 0xac, 0xc9),-	G(0x48, 0x6c, 0x1f, 0x57, 0x8f, 0xe3), G(0x4a, 0x6f, 0x16, 0x5c, 0x82, 0xed),-	G(0x4c, 0x6a, 0x0d, 0x41, 0x95, 0xff), G(0x4e, 0x69, 0x04, 0x4a, 0x98, 0xf1),-	G(0x50, 0x78, 0x73, 0x23, 0xd3, 0xab), G(0x52, 0x7b, 0x7a, 0x28, 0xde, 0xa5),-	G(0x54, 0x7e, 0x61, 0x35, 0xc9, 0xb7), G(0x56, 0x7d, 0x68, 0x3e, 0xc4, 0xb9),-	G(0x58, 0x74, 0x57, 0x0f, 0xe7, 0x93), G(0x5a, 0x77, 0x5e, 0x04, 0xea, 0x9d),-	G(0x5c, 0x72, 0x45, 0x19, 0xfd, 0x8f), G(0x5e, 0x71, 0x4c, 0x12, 0xf0, 0x81),-	G(0x60, 0x50, 0xab, 0xcb, 0x6b, 0x3b), G(0x62, 0x53, 0xa2, 0xc0, 0x66, 0x35),-	G(0x64, 0x56, 0xb9, 0xdd, 0x71, 0x27), G(0x66, 0x55, 0xb0, 0xd6, 0x7c, 0x29),-	G(0x68, 0x5c, 0x8f, 0xe7, 0x5f, 0x03), G(0x6a, 0x5f, 0x86, 0xec, 0x52, 0x0d),-	G(0x6c, 0x5a, 0x9d, 0xf1, 0x45, 0x1f), G(0x6e, 0x59, 0x94, 0xfa, 0x48, 0x11),-	G(0x70, 0x48, 0xe3, 0x93, 0x03, 0x4b), G(0x72, 0x4b, 0xea, 0x98, 0x0e, 0x45),-	G(0x74, 0x4e, 0xf1, 0x85, 0x19, 0x57), G(0x76, 0x4d, 0xf8, 0x8e, 0x14, 0x59),-	G(0x78, 0x44, 0xc7, 0xbf, 0x37, 0x73), G(0x7a, 0x47, 0xce, 0xb4, 0x3a, 0x7d),-	G(0x7c, 0x42, 0xd5, 0xa9, 0x2d, 0x6f), G(0x7e, 0x41, 0xdc, 0xa2, 0x20, 0x61),-	G(0x80, 0xc0, 0x76, 0xf6, 0x6d, 0xad), G(0x82, 0xc3, 0x7f, 0xfd, 0x60, 0xa3),-	G(0x84, 0xc6, 0x64, 0xe0, 0x77, 0xb1), G(0x86, 0xc5, 0x6d, 0xeb, 0x7a, 0xbf),-	G(0x88, 0xcc, 0x52, 0xda, 0x59, 0x95), G(0x8a, 0xcf, 0x5b, 0xd1, 0x54, 0x9b),-	G(0x8c, 0xca, 0x40, 0xcc, 0x43, 0x89), G(0x8e, 0xc9, 0x49, 0xc7, 0x4e, 0x87),-	G(0x90, 0xd8, 0x3e, 0xae, 0x05, 0xdd), G(0x92, 0xdb, 0x37, 0xa5, 0x08, 0xd3),-	G(0x94, 0xde, 0x2c, 0xb8, 0x1f, 0xc1), G(0x96, 0xdd, 0x25, 0xb3, 0x12, 0xcf),-	G(0x98, 0xd4, 0x1a, 0x82, 0x31, 0xe5), G(0x9a, 0xd7, 0x13, 0x89, 0x3c, 0xeb),-	G(0x9c, 0xd2, 0x08, 0x94, 0x2b, 0xf9), G(0x9e, 0xd1, 0x01, 0x9f, 0x26, 0xf7),-	G(0xa0, 0xf0, 0xe6, 0x46, 0xbd, 0x4d), G(0xa2, 0xf3, 0xef, 0x4d, 0xb0, 0x43),-	G(0xa4, 0xf6, 0xf4, 0x50, 0xa7, 0x51), G(0xa6, 0xf5, 0xfd, 0x5b, 0xaa, 0x5f),-	G(0xa8, 0xfc, 0xc2, 0x6a, 0x89, 0x75), G(0xaa, 0xff, 0xcb, 0x61, 0x84, 0x7b),-	G(0xac, 0xfa, 0xd0, 0x7c, 0x93, 0x69), G(0xae, 0xf9, 0xd9, 0x77, 0x9e, 0x67),-	G(0xb0, 0xe8, 0xae, 0x1e, 0xd5, 0x3d), G(0xb2, 0xeb, 0xa7, 0x15, 0xd8, 0x33),-	G(0xb4, 0xee, 0xbc, 0x08, 0xcf, 0x21), G(0xb6, 0xed, 0xb5, 0x03, 0xc2, 0x2f),-	G(0xb8, 0xe4, 0x8a, 0x32, 0xe1, 0x05), G(0xba, 0xe7, 0x83, 0x39, 0xec, 0x0b),-	G(0xbc, 0xe2, 0x98, 0x24, 0xfb, 0x19), G(0xbe, 0xe1, 0x91, 0x2f, 0xf6, 0x17),-	G(0xc0, 0xa0, 0x4d, 0x8d, 0xd6, 0x76), G(0xc2, 0xa3, 0x44, 0x86, 0xdb, 0x78),-	G(0xc4, 0xa6, 0x5f, 0x9b, 0xcc, 0x6a), G(0xc6, 0xa5, 0x56, 0x90, 0xc1, 0x64),-	G(0xc8, 0xac, 0x69, 0xa1, 0xe2, 0x4e), G(0xca, 0xaf, 0x60, 0xaa, 0xef, 0x40),-	G(0xcc, 0xaa, 0x7b, 0xb7, 0xf8, 0x52), G(0xce, 0xa9, 0x72, 0xbc, 0xf5, 0x5c),-	G(0xd0, 0xb8, 0x05, 0xd5, 0xbe, 0x06), G(0xd2, 0xbb, 0x0c, 0xde, 0xb3, 0x08),-	G(0xd4, 0xbe, 0x17, 0xc3, 0xa4, 0x1a), G(0xd6, 0xbd, 0x1e, 0xc8, 0xa9, 0x14),-	G(0xd8, 0xb4, 0x21, 0xf9, 0x8a, 0x3e), G(0xda, 0xb7, 0x28, 0xf2, 0x87, 0x30),-	G(0xdc, 0xb2, 0x33, 0xef, 0x90, 0x22), G(0xde, 0xb1, 0x3a, 0xe4, 0x9d, 0x2c),-	G(0xe0, 0x90, 0xdd, 0x3d, 0x06, 0x96), G(0xe2, 0x93, 0xd4, 0x36, 0x0b, 0x98),-	G(0xe4, 0x96, 0xcf, 0x2b, 0x1c, 0x8a), G(0xe6, 0x95, 0xc6, 0x20, 0x11, 0x84),-	G(0xe8, 0x9c, 0xf9, 0x11, 0x32, 0xae), G(0xea, 0x9f, 0xf0, 0x1a, 0x3f, 0xa0),-	G(0xec, 0x9a, 0xeb, 0x07, 0x28, 0xb2), G(0xee, 0x99, 0xe2, 0x0c, 0x25, 0xbc),-	G(0xf0, 0x88, 0x95, 0x65, 0x6e, 0xe6), G(0xf2, 0x8b, 0x9c, 0x6e, 0x63, 0xe8),-	G(0xf4, 0x8e, 0x87, 0x73, 0x74, 0xfa), G(0xf6, 0x8d, 0x8e, 0x78, 0x79, 0xf4),-	G(0xf8, 0x84, 0xb1, 0x49, 0x5a, 0xde), G(0xfa, 0x87, 0xb8, 0x42, 0x57, 0xd0),-	G(0xfc, 0x82, 0xa3, 0x5f, 0x40, 0xc2), G(0xfe, 0x81, 0xaa, 0x54, 0x4d, 0xcc),-	G(0x1b, 0x9b, 0xec, 0xf7, 0xda, 0x41), G(0x19, 0x98, 0xe5, 0xfc, 0xd7, 0x4f),-	G(0x1f, 0x9d, 0xfe, 0xe1, 0xc0, 0x5d), G(0x1d, 0x9e, 0xf7, 0xea, 0xcd, 0x53),-	G(0x13, 0x97, 0xc8, 0xdb, 0xee, 0x79), G(0x11, 0x94, 0xc1, 0xd0, 0xe3, 0x77),-	G(0x17, 0x91, 0xda, 0xcd, 0xf4, 0x65), G(0x15, 0x92, 0xd3, 0xc6, 0xf9, 0x6b),-	G(0x0b, 0x83, 0xa4, 0xaf, 0xb2, 0x31), G(0x09, 0x80, 0xad, 0xa4, 0xbf, 0x3f),-	G(0x0f, 0x85, 0xb6, 0xb9, 0xa8, 0x2d), G(0x0d, 0x86, 0xbf, 0xb2, 0xa5, 0x23),-	G(0x03, 0x8f, 0x80, 0x83, 0x86, 0x09), G(0x01, 0x8c, 0x89, 0x88, 0x8b, 0x07),-	G(0x07, 0x89, 0x92, 0x95, 0x9c, 0x15), G(0x05, 0x8a, 0x9b, 0x9e, 0x91, 0x1b),-	G(0x3b, 0xab, 0x7c, 0x47, 0x0a, 0xa1), G(0x39, 0xa8, 0x75, 0x4c, 0x07, 0xaf),-	G(0x3f, 0xad, 0x6e, 0x51, 0x10, 0xbd), G(0x3d, 0xae, 0x67, 0x5a, 0x1d, 0xb3),-	G(0x33, 0xa7, 0x58, 0x6b, 0x3e, 0x99), G(0x31, 0xa4, 0x51, 0x60, 0x33, 0x97),-	G(0x37, 0xa1, 0x4a, 0x7d, 0x24, 0x85), G(0x35, 0xa2, 0x43, 0x76, 0x29, 0x8b),-	G(0x2b, 0xb3, 0x34, 0x1f, 0x62, 0xd1), G(0x29, 0xb0, 0x3d, 0x14, 0x6f, 0xdf),-	G(0x2f, 0xb5, 0x26, 0x09, 0x78, 0xcd), G(0x2d, 0xb6, 0x2f, 0x02, 0x75, 0xc3),-	G(0x23, 0xbf, 0x10, 0x33, 0x56, 0xe9), G(0x21, 0xbc, 0x19, 0x38, 0x5b, 0xe7),-	G(0x27, 0xb9, 0x02, 0x25, 0x4c, 0xf5), G(0x25, 0xba, 0x0b, 0x2e, 0x41, 0xfb),-	G(0x5b, 0xfb, 0xd7, 0x8c, 0x61, 0x9a), G(0x59, 0xf8, 0xde, 0x87, 0x6c, 0x94),-	G(0x5f, 0xfd, 0xc5, 0x9a, 0x7b, 0x86), G(0x5d, 0xfe, 0xcc, 0x91, 0x76, 0x88),-	G(0x53, 0xf7, 0xf3, 0xa0, 0x55, 0xa2), G(0x51, 0xf4, 0xfa, 0xab, 0x58, 0xac),-	G(0x57, 0xf1, 0xe1, 0xb6, 0x4f, 0xbe), G(0x55, 0xf2, 0xe8, 0xbd, 0x42, 0xb0),-	G(0x4b, 0xe3, 0x9f, 0xd4, 0x09, 0xea), G(0x49, 0xe0, 0x96, 0xdf, 0x04, 0xe4),-	G(0x4f, 0xe5, 0x8d, 0xc2, 0x13, 0xf6), G(0x4d, 0xe6, 0x84, 0xc9, 0x1e, 0xf8),-	G(0x43, 0xef, 0xbb, 0xf8, 0x3d, 0xd2), G(0x41, 0xec, 0xb2, 0xf3, 0x30, 0xdc),-	G(0x47, 0xe9, 0xa9, 0xee, 0x27, 0xce), G(0x45, 0xea, 0xa0, 0xe5, 0x2a, 0xc0),-	G(0x7b, 0xcb, 0x47, 0x3c, 0xb1, 0x7a), G(0x79, 0xc8, 0x4e, 0x37, 0xbc, 0x74),-	G(0x7f, 0xcd, 0x55, 0x2a, 0xab, 0x66), G(0x7d, 0xce, 0x5c, 0x21, 0xa6, 0x68),-	G(0x73, 0xc7, 0x63, 0x10, 0x85, 0x42), G(0x71, 0xc4, 0x6a, 0x1b, 0x88, 0x4c),-	G(0x77, 0xc1, 0x71, 0x06, 0x9f, 0x5e), G(0x75, 0xc2, 0x78, 0x0d, 0x92, 0x50),-	G(0x6b, 0xd3, 0x0f, 0x64, 0xd9, 0x0a), G(0x69, 0xd0, 0x06, 0x6f, 0xd4, 0x04),-	G(0x6f, 0xd5, 0x1d, 0x72, 0xc3, 0x16), G(0x6d, 0xd6, 0x14, 0x79, 0xce, 0x18),-	G(0x63, 0xdf, 0x2b, 0x48, 0xed, 0x32), G(0x61, 0xdc, 0x22, 0x43, 0xe0, 0x3c),-	G(0x67, 0xd9, 0x39, 0x5e, 0xf7, 0x2e), G(0x65, 0xda, 0x30, 0x55, 0xfa, 0x20),-	G(0x9b, 0x5b, 0x9a, 0x01, 0xb7, 0xec), G(0x99, 0x58, 0x93, 0x0a, 0xba, 0xe2),-	G(0x9f, 0x5d, 0x88, 0x17, 0xad, 0xf0), G(0x9d, 0x5e, 0x81, 0x1c, 0xa0, 0xfe),-	G(0x93, 0x57, 0xbe, 0x2d, 0x83, 0xd4), G(0x91, 0x54, 0xb7, 0x26, 0x8e, 0xda),-	G(0x97, 0x51, 0xac, 0x3b, 0x99, 0xc8), G(0x95, 0x52, 0xa5, 0x30, 0x94, 0xc6),-	G(0x8b, 0x43, 0xd2, 0x59, 0xdf, 0x9c), G(0x89, 0x40, 0xdb, 0x52, 0xd2, 0x92),-	G(0x8f, 0x45, 0xc0, 0x4f, 0xc5, 0x80), G(0x8d, 0x46, 0xc9, 0x44, 0xc8, 0x8e),-	G(0x83, 0x4f, 0xf6, 0x75, 0xeb, 0xa4), G(0x81, 0x4c, 0xff, 0x7e, 0xe6, 0xaa),-	G(0x87, 0x49, 0xe4, 0x63, 0xf1, 0xb8), G(0x85, 0x4a, 0xed, 0x68, 0xfc, 0xb6),-	G(0xbb, 0x6b, 0x0a, 0xb1, 0x67, 0x0c), G(0xb9, 0x68, 0x03, 0xba, 0x6a, 0x02),-	G(0xbf, 0x6d, 0x18, 0xa7, 0x7d, 0x10), G(0xbd, 0x6e, 0x11, 0xac, 0x70, 0x1e),-	G(0xb3, 0x67, 0x2e, 0x9d, 0x53, 0x34), G(0xb1, 0x64, 0x27, 0x96, 0x5e, 0x3a),-	G(0xb7, 0x61, 0x3c, 0x8b, 0x49, 0x28), G(0xb5, 0x62, 0x35, 0x80, 0x44, 0x26),-	G(0xab, 0x73, 0x42, 0xe9, 0x0f, 0x7c), G(0xa9, 0x70, 0x4b, 0xe2, 0x02, 0x72),-	G(0xaf, 0x75, 0x50, 0xff, 0x15, 0x60), G(0xad, 0x76, 0x59, 0xf4, 0x18, 0x6e),-	G(0xa3, 0x7f, 0x66, 0xc5, 0x3b, 0x44), G(0xa1, 0x7c, 0x6f, 0xce, 0x36, 0x4a),-	G(0xa7, 0x79, 0x74, 0xd3, 0x21, 0x58), G(0xa5, 0x7a, 0x7d, 0xd8, 0x2c, 0x56),-	G(0xdb, 0x3b, 0xa1, 0x7a, 0x0c, 0x37), G(0xd9, 0x38, 0xa8, 0x71, 0x01, 0x39),-	G(0xdf, 0x3d, 0xb3, 0x6c, 0x16, 0x2b), G(0xdd, 0x3e, 0xba, 0x67, 0x1b, 0x25),-	G(0xd3, 0x37, 0x85, 0x56, 0x38, 0x0f), G(0xd1, 0x34, 0x8c, 0x5d, 0x35, 0x01),-	G(0xd7, 0x31, 0x97, 0x40, 0x22, 0x13), G(0xd5, 0x32, 0x9e, 0x4b, 0x2f, 0x1d),-	G(0xcb, 0x23, 0xe9, 0x22, 0x64, 0x47), G(0xc9, 0x20, 0xe0, 0x29, 0x69, 0x49),-	G(0xcf, 0x25, 0xfb, 0x34, 0x7e, 0x5b), G(0xcd, 0x26, 0xf2, 0x3f, 0x73, 0x55),-	G(0xc3, 0x2f, 0xcd, 0x0e, 0x50, 0x7f), G(0xc1, 0x2c, 0xc4, 0x05, 0x5d, 0x71),-	G(0xc7, 0x29, 0xdf, 0x18, 0x4a, 0x63), G(0xc5, 0x2a, 0xd6, 0x13, 0x47, 0x6d),-	G(0xfb, 0x0b, 0x31, 0xca, 0xdc, 0xd7), G(0xf9, 0x08, 0x38, 0xc1, 0xd1, 0xd9),-	G(0xff, 0x0d, 0x23, 0xdc, 0xc6, 0xcb), G(0xfd, 0x0e, 0x2a, 0xd7, 0xcb, 0xc5),-	G(0xf3, 0x07, 0x15, 0xe6, 0xe8, 0xef), G(0xf1, 0x04, 0x1c, 0xed, 0xe5, 0xe1),-	G(0xf7, 0x01, 0x07, 0xf0, 0xf2, 0xf3), G(0xf5, 0x02, 0x0e, 0xfb, 0xff, 0xfd),-	G(0xeb, 0x13, 0x79, 0x92, 0xb4, 0xa7), G(0xe9, 0x10, 0x70, 0x99, 0xb9, 0xa9),-	G(0xef, 0x15, 0x6b, 0x84, 0xae, 0xbb), G(0xed, 0x16, 0x62, 0x8f, 0xa3, 0xb5),-	G(0xe3, 0x1f, 0x5d, 0xbe, 0x80, 0x9f), G(0xe1, 0x1c, 0x54, 0xb5, 0x8d, 0x91),-	G(0xe7, 0x19, 0x4f, 0xa8, 0x9a, 0x83), G(0xe5, 0x1a, 0x46, 0xa3, 0x97, 0x8d),-};-#undef G+/*+ * The schedule is kept in its compressed form, 30 words of it, inside the+ * aes_key the caller already has -- comfortably inside the 448 bytes that+ * held the round keys before.  br_aes_ct64_skey_expand blows it up to 120+ * words on the stack once per call, which is how BearSSL's own CTR and CBC+ * use it.+ *+ * It travels through memcpy rather than a cast because aes_key is all+ * uint8_t and so carries no alignment of its own, whatever the allocator+ * happens to give it.+ */+#define COMP_SKEY_WORDS 30 -static void expand_key(uint8_t *expandedKey, uint8_t *key, int size, size_t expandedKeySize)+void crypton_aes_generic_schedule(aes_sched *sched, const aes_key *key) {-	int csz;-	int i;-	uint8_t t[4] = { 0 };--	for (i = 0; i < size; i++)-		expandedKey[i] = key[i];-	csz = size;+	uint64_t comp_skey[COMP_SKEY_WORDS]; -	i = 1;-	while (csz < expandedKeySize) {-		t[0] = expandedKey[(csz - 4) + 0];-		t[1] = expandedKey[(csz - 4) + 1];-		t[2] = expandedKey[(csz - 4) + 2];-		t[3] = expandedKey[(csz - 4) + 3];+	memcpy(comp_skey, key->data, sizeof comp_skey);+	sched->nbr = key->nbr;+	br_aes_ct64_skey_expand(sched->sk_exp, sched->nbr, comp_skey);+} -		if (csz % size == 0) {-			uint8_t tmp;+/*+ * Up to four blocks through the bitsliced core at once.  Fewer than four is+ * the same work as four -- the lanes are there whether anything is in them+ * -- so a caller with four to offer gets them for what one used to cost.+ */+static void pass(uint8_t *output, const uint8_t *input, unsigned n,+                 const aes_sched *sched, int decrypt)+{+	uint32_t w[16];+	uint64_t q[8];+	unsigned i; -			tmp = t[0];-			t[0] = sbox[t[1]] ^ Rcon[i++ % sizeof(Rcon)];-			t[1] = sbox[t[2]];-			t[2] = sbox[t[3]];-			t[3] = sbox[tmp];-		}+	memset(w, 0, sizeof w);+	for (i = 0; i < n; i++) {+		w[4 * i]     = br_dec32le(input + 16 * i);+		w[4 * i + 1] = br_dec32le(input + 16 * i + 4);+		w[4 * i + 2] = br_dec32le(input + 16 * i + 8);+		w[4 * i + 3] = br_dec32le(input + 16 * i + 12);+	} -		if (size == 32 && ((csz % size) == 16)) {-			t[0] = sbox[t[0]];-			t[1] = sbox[t[1]];-			t[2] = sbox[t[2]];-			t[3] = sbox[t[3]];-		}+	for (i = 0; i < 4; i++)+		br_aes_ct64_interleave_in(&q[i], &q[i + 4], w + 4 * i);+	br_aes_ct64_ortho(q);+	if (decrypt)+		br_aes_ct64_bitslice_decrypt(sched->nbr, sched->sk_exp, q);+	else+		br_aes_ct64_bitslice_encrypt(sched->nbr, sched->sk_exp, q);+	br_aes_ct64_ortho(q);+	for (i = 0; i < 4; i++)+		br_aes_ct64_interleave_out(w + 4 * i, q[i], q[i + 4]); -		expandedKey[csz] = expandedKey[csz - size] ^ t[0]; csz++;-		expandedKey[csz] = expandedKey[csz - size] ^ t[1]; csz++;-		expandedKey[csz] = expandedKey[csz - size] ^ t[2]; csz++;-		expandedKey[csz] = expandedKey[csz - size] ^ t[3]; csz++;+	for (i = 0; i < n; i++) {+		br_enc32le(output + 16 * i,      w[4 * i]);+		br_enc32le(output + 16 * i + 4,  w[4 * i + 1]);+		br_enc32le(output + 16 * i + 8,  w[4 * i + 2]);+		br_enc32le(output + 16 * i + 12, w[4 * i + 3]); 	} } -static void shift_rows(uint8_t *state)+void crypton_aes_generic_blocks(uint8_t *output, const uint8_t *input,+                                uint32_t nb_blocks, const aes_sched *sched,+                                int decrypt) {-	uint32_t *s32;-	int i;--	for (i = 0; i < 16; i++)-		state[i] = sbox[state[i]];-	s32 = (uint32_t *) state;-	s32[1] = rol32_be(s32[1], 8);-	s32[2] = rol32_be(s32[2], 16);-	s32[3] = rol32_be(s32[3], 24);+	while (nb_blocks >= 4) {+		pass(output, input, 4, sched, decrypt);+		output += 64;+		input += 64;+		nb_blocks -= 4;+	}+	if (nb_blocks > 0)+		pass(output, input, (unsigned) nb_blocks, sched, decrypt); } -static void add_round_key(uint8_t *state, uint8_t *rk)+static void one(aes_block *output, aes_key *key, aes_block *input, int decrypt) {-	uint32_t *s32, *r32;+	aes_sched sched; -	s32 = (uint32_t *) state;-	r32 = (uint32_t *) rk;-	s32[0] ^= r32[0];-	s32[1] ^= r32[1];-	s32[2] ^= r32[2];-	s32[3] ^= r32[3];+	crypton_aes_generic_schedule(&sched, key);+	crypton_aes_generic_blocks((uint8_t *) output, (const uint8_t *) input,+	                           1, &sched, decrypt); } -#define gm1(a) (a)-#define gm2(a) gmtab[a][0]-#define gm3(a) gmtab[a][1]-#define gm9(a) gmtab[a][2]-#define gm11(a) gmtab[a][3]-#define gm13(a) gmtab[a][4]-#define gm14(a) gmtab[a][5]--static void mix_columns(uint8_t *state)+void crypton_aes_generic_encrypt_block(aes_block *output, aes_key *key, aes_block *input) {-	int i;-	uint8_t cpy[4];--	for (i = 0; i < 4; i++) {-		cpy[0] = state[0 * 4 + i];-		cpy[1] = state[1 * 4 + i];-		cpy[2] = state[2 * 4 + i];-		cpy[3] = state[3 * 4 + i];-		state[i] = gm2(cpy[0]) ^ gm1(cpy[3]) ^ gm1(cpy[2]) ^ gm3(cpy[1]);-		state[4+i] = gm2(cpy[1]) ^ gm1(cpy[0]) ^ gm1(cpy[3]) ^ gm3(cpy[2]);-		state[8+i] = gm2(cpy[2]) ^ gm1(cpy[1]) ^ gm1(cpy[0]) ^ gm3(cpy[3]);-		state[12+i] = gm2(cpy[3]) ^ gm1(cpy[2]) ^ gm1(cpy[1]) ^ gm3(cpy[0]);-	}+	one(output, key, input, 0); } -static void create_round_key(uint8_t *expandedKey, uint8_t *rk)+void crypton_aes_generic_decrypt_block(aes_block *output, aes_key *key, aes_block *input) {-	int i,j;-	for (i = 0; i < 4; i++)-		for (j = 0; j < 4; j++)-			rk[i + j * 4] = expandedKey[i * 4 + j];+	one(output, key, input, 1); } -static void aes_main(aes_key *key, uint8_t *state)+void crypton_aes_generic_init(aes_key *key, uint8_t *origkey, uint8_t size) {-	int i = 0;-	uint32_t rk[4];-	uint8_t *rkptr = (uint8_t *) rk;--	create_round_key(key->data, rkptr);-	add_round_key(state, rkptr);--	for (i = 1; i < key->nbr; i++) {-		create_round_key(key->data + 16 * i, rkptr);-		shift_rows(state);-		mix_columns(state);-		add_round_key(state, rkptr);-	}--	create_round_key(key->data + 16 * key->nbr, rkptr);-	shift_rows(state);-	add_round_key(state, rkptr);-}+	uint64_t comp_skey[COMP_SKEY_WORDS];+	unsigned nbr; -static void shift_rows_inv(uint8_t *state)-{-	uint32_t *s32;-	int i;+	/* 0 for a key length that is not 16, 24 or 32; the old code returned+	 * without touching the key in that case and so does this */+	nbr = br_aes_ct64_keysched(comp_skey, origkey, size);+	if (nbr == 0)+		return; -	s32 = (uint32_t *) state;-	s32[1] = ror32_be(s32[1], 8);-	s32[2] = ror32_be(s32[2], 16);-	s32[3] = ror32_be(s32[3], 24);-	for (i = 0; i < 16; i++)-		state[i] = rsbox[state[i]];+	key->nbr = (uint8_t) nbr;+	memcpy(key->data, comp_skey, sizeof comp_skey); } -static void mix_columns_inv(uint8_t *state)+/*+ * CTR, four counter blocks at a time.  The counter itself is serial, but+ * nothing about it depends on the keystream, so the four blocks it will+ * reach next can be written down before any of them is encrypted.+ */+static void ctr(uint8_t *output, aes_key *key, aes_block *iv,+                uint8_t *input, uint32_t len, int c32) {-	int i;-	uint8_t cpy[4];+	aes_sched sched;+	aes_block counter;+	uint8_t ks[64];+	uint32_t nb_blocks = len / 16;+	uint32_t tail = len % 16;+	uint32_t i; -	for (i = 0; i < 4; i++) {-		cpy[0] = state[0 * 4 + i];-		cpy[1] = state[1 * 4 + i];-		cpy[2] = state[2 * 4 + i];-		cpy[3] = state[3 * 4 + i];-		state[i] = gm14(cpy[0]) ^ gm9(cpy[3]) ^ gm13(cpy[2]) ^ gm11(cpy[1]);-		state[4+i] = gm14(cpy[1]) ^ gm9(cpy[0]) ^ gm13(cpy[3]) ^ gm11(cpy[2]);-		state[8+i] = gm14(cpy[2]) ^ gm9(cpy[1]) ^ gm13(cpy[0]) ^ gm11(cpy[3]);-		state[12+i] = gm14(cpy[3]) ^ gm9(cpy[2]) ^ gm13(cpy[1]) ^ gm11(cpy[0]);-	}-}+	crypton_aes_generic_schedule(&sched, key);+	block128_copy(&counter, iv); -static void aes_main_inv(aes_key *key, uint8_t *state)-{-	int i = 0;-	uint32_t rk[4];-	uint8_t *rkptr = (uint8_t *) rk;+	while (nb_blocks > 0) {+		uint32_t n = nb_blocks < 4 ? nb_blocks : 4; -	create_round_key(key->data + 16 * key->nbr, rkptr);-	add_round_key(state, rkptr);+		for (i = 0; i < n; i++) {+			block128_copy((block128 *) (ks + 16 * i), &counter);+			if (c32)+				block128_inc32_le(&counter);+			else+				block128_inc_be(&counter);+		}+		crypton_aes_generic_blocks(ks, ks, n, &sched, 0);+		for (i = 0; i < n * 16; i++)+			output[i] = ks[i] ^ input[i]; -	for (i = key->nbr - 1; i > 0; i--) {-		create_round_key(key->data + 16 * i, rkptr);-		shift_rows_inv(state);-		add_round_key(state, rkptr);-		mix_columns_inv(state);+		output += n * 16;+		input += n * 16;+		nb_blocks -= n; 	} -	create_round_key(key->data, rkptr);-	shift_rows_inv(state);-	add_round_key(state, rkptr);-}--/* Set the block values, for the block:- * a0,0 a0,1 a0,2 a0,3- * a1,0 a1,1 a1,2 a1,3 -> a0,0 a1,0 a2,0 a3,0 a0,1 a1,1 ... a2,3 a3,3- * a2,0 a2,1 a2,2 a2,3- * a3,0 a3,1 a3,2 a3,3- */-#define swap_block(t, f) \-	t[0] = f[0]; t[4] = f[1]; t[8] = f[2]; t[12] = f[3]; \-	t[1] = f[4]; t[5] = f[5]; t[9] = f[6]; t[13] = f[7]; \-	t[2] = f[8]; t[6] = f[9]; t[10] = f[10]; t[14] = f[11]; \-	t[3] = f[12]; t[7] = f[13]; t[11] = f[14]; t[15] = f[15]--void crypton_aes_generic_encrypt_block(aes_block *output, aes_key *key, aes_block *input)-{-	uint32_t block[4];-	uint8_t *iptr, *optr, *bptr;--	iptr = (uint8_t *) input;-	optr = (uint8_t *) output;-	bptr = (uint8_t *) block;-	swap_block(bptr, iptr);-	aes_main(key, bptr);-	swap_block(optr, bptr);+	if (tail != 0) {+		block128_copy((block128 *) ks, &counter);+		crypton_aes_generic_blocks(ks, ks, 1, &sched, 0);+		for (i = 0; i < tail; i++)+			output[i] = ks[i] ^ input[i];+	} } -void crypton_aes_generic_decrypt_block(aes_block *output, aes_key *key, aes_block *input)+void crypton_aes_bitsliced_encrypt_ctr(uint8_t *output, aes_key *key,+                                       aes_block *iv, uint8_t *input,+                                       uint32_t len) {-	uint32_t block[4];-	uint8_t *iptr, *optr, *bptr;--	iptr = (uint8_t *) input;-	optr = (uint8_t *) output;-	bptr = (uint8_t *) block;-	swap_block(bptr, iptr);-	aes_main_inv(key, bptr);-	swap_block(optr, bptr);+	ctr(output, key, iv, input, len, 0); } -void crypton_aes_generic_init(aes_key *key, uint8_t *origkey, uint8_t size)+void crypton_aes_bitsliced_encrypt_c32(uint8_t *output, aes_key *key,+                                       aes_block *iv, uint8_t *input,+                                       uint32_t len) {-	int esz;--	switch (size) {-	case 16: key->nbr = 10; esz = 176; break;-	case 24: key->nbr = 12; esz = 208; break;-	case 32: key->nbr = 14; esz = 240; break;-	default: return;-	}-	expand_key(key->data, origkey, size, esz);-	return;+	ctr(output, key, iv, input, len, 1); }

@@ -32,3 +32,35 @@ void crypton_aes_generic_encrypt_block(aes_block *output, aes_key *key, aes_block *input); void crypton_aes_generic_decrypt_block(aes_block *output, aes_key *key, aes_block *input); void crypton_aes_generic_init(aes_key *key, uint8_t *origkey, uint8_t size);++/*+ * The bitsliced core takes four blocks at a time, and the schedule it reads+ * is the expanded one rather than the compressed form the aes_key holds.+ * Expanding costs about what a block costs, so a mode with more than one+ * block to do expands once, here, and hands the result to every group.+ */+typedef struct {+	uint64_t sk_exp[120];+	unsigned nbr;+} aes_sched;++void crypton_aes_generic_schedule(aes_sched *sched, const aes_key *key);++/* nb_blocks of them, four at a pass; decrypt selects the direction */+void crypton_aes_generic_blocks(uint8_t *output, const uint8_t *input,+                                uint32_t nb_blocks, const aes_sched *sched,+                                int decrypt);++/*+ * CTR with the two counters crypton uses.  These are not the generic+ * entries: those go through the branch table for the block itself and so+ * run on accelerated machines too, where the key holds a different+ * schedule.  crypton_aes.c installs these only when nothing was+ * accelerated.+ */+void crypton_aes_bitsliced_encrypt_ctr(uint8_t *output, aes_key *key,+                                       aes_block *iv, uint8_t *input,+                                       uint32_t len);+void crypton_aes_bitsliced_encrypt_c32(uint8_t *output, aes_key *key,+                                       aes_block *iv, uint8_t *input,+                                       uint32_t len);

@@ -33,6 +33,7 @@ #include <crypton_cpu.h> #include <aes/gf.h> #include <aes/x86ni.h>+#include "bearssl/inner.h"  /* inplace GFMUL for xts mode */ void crypton_aes_generic_gf_mulx(block128 *a)@@ -45,118 +46,43 @@   /*- * GF multiplication with Shoup's method and 4-bit table.+ * GHASH, without a table.  *- * We precompute the products of H with all 4-bit polynomials and store them in- * a 'table_4bit' array.  To avoid unnecessary byte swapping, the 16 blocks are- * written to the table with qwords already converted to CPU order.  Table- * indices use the reflected bit ordering, i.e. polynomials X^0, X^1, X^2, X^3- * map to bit positions 3, 2, 1, 0 respectively.+ * This was Shoup's method: the products of H with all sixteen 4-bit+ * polynomials, precomputed, and thirty-two lookups a block at indices taken+ * from the accumulator -- which is to say, thirty-two addresses derived from+ * a secret.  It is now BearSSL's ghash_ctmul64, which builds the GF(2^128)+ * multiply out of shifts, masks and integer multiplies and looks nothing up.+ * See cbits/bearssl/README.md.  *- * To multiply an arbitrary block with H, the input block is decomposed in 4-bit- * segments.  We get the final result after 32 table lookups and additions, one- * for each segment, interleaving multiplication by P(X)=X^4.+ * The table_4bit the interface names is sixteen blocks wide because the+ * table needed it.  This keeps H in the first and leaves the rest alone; the+ * PMULL and PCLMUL implementations keep their own powers of H in that same+ * space, so the width stays as it is.  */ -/* convert block128 qwords between BE and CPU order */-static inline void block128_cpu_swap_be(block128 *a, const block128 *b)-{-	a->q[1] = cpu_to_be64(b->q[1]);-	a->q[0] = cpu_to_be64(b->q[0]);-}--/* multiplication by P(X)=X, assuming qwords already in CPU order */-static inline void cpu_gf_mulx(block128 *a, const block128 *b)-{-	uint64_t v0 = b->q[0];-	uint64_t v1 = b->q[1];-	a->q[1] = v1 >> 1 | v0 << 63;-	a->q[0] = v0 >> 1 ^ ((0-(v1 & 1)) & 0xe100000000000000ULL);-}--static const uint64_t r4_0[] =-	{ 0x0000000000000000ULL, 0x1c20000000000000ULL-	, 0x3840000000000000ULL, 0x2460000000000000ULL-	, 0x7080000000000000ULL, 0x6ca0000000000000ULL-	, 0x48c0000000000000ULL, 0x54e0000000000000ULL-	, 0xe100000000000000ULL, 0xfd20000000000000ULL-	, 0xd940000000000000ULL, 0xc560000000000000ULL-	, 0x9180000000000000ULL, 0x8da0000000000000ULL-	, 0xa9c0000000000000ULL, 0xb5e0000000000000ULL-	};--/* multiplication by P(X)=X^4, assuming qwords already in CPU order */-static inline void cpu_gf_mulx4(block128 *a, const block128 *b)-{-	uint64_t v0 = b->q[0];-	uint64_t v1 = b->q[1];-	a->q[1] = v1 >> 4 | v0 << 60;-	a->q[0] = v0 >> 4 ^ r4_0[v1 & 0xf];-}--/* initialize the 4-bit table given H */+/* remember H */ void crypton_aes_generic_hinit(table_4bit htable, const block128 *h) {-	block128 v, *p;-	int i, j;--	/* multiplication by 0 is 0 */-	block128_zero(&htable[0]);--	/* at index 8=2^3 we have H.X^0 = H */-	i = 8;-	block128_cpu_swap_be(&htable[i], h); /* in CPU order */-	p = &htable[i];--	/* for other powers of 2, repeat multiplication by P(X)=X */-	for (i = 4; i > 0; i >>= 1)-	{-		cpu_gf_mulx(&htable[i], p);-		p = &htable[i];-	}--	/* remaining elements are linear combinations */-	for (i = 2; i < 16; i <<= 1) {-		p = &htable[i];-		v = *p;-		for (j = 1; j < i; j++) {-			p[j] = v;-			block128_xor_aligned(&p[j], &htable[j]);-		}-	}+	block128_copy(&htable[0], h); } -/* multiply a block with H */+/*+ * br_ghash_ctmul64 computes y = (y ^ x) * H for each block x it is given, so+ * a block of zeros is the bare multiply this entry is asked for.+ */ void crypton_aes_generic_gf_mul(block128 *a, const table_4bit htable) {-	block128 b;-	int i;-	block128_zero(&b);-	for (i = 15; i >= 0; i--)-	{-		uint8_t v = a->b[i];-		block128_xor_aligned(&b, &htable[v & 0xf]); /* high bits (reflected) */-		cpu_gf_mulx4(&b, &b);-		block128_xor_aligned(&b, &htable[v >> 4]);  /* low bits (reflected) */-		if (i > 0)-			cpu_gf_mulx4(&b, &b);-		else-			block128_cpu_swap_be(a, &b); /* restore BE order when done */-	}+	static const uint8_t zero[16] = { 0 };++	br_ghash_ctmul64(a, &htable[0], zero, sizeof zero); }  /*- * Four GHASH steps at once.  The generic table-driven multiply has no cheaper- * way to do this than one block at a time; the point of the entry is that the- * PMULL and PCLMUL versions can fold the four products into one reduction, so- * the GCM loops hand over four blocks whenever they have them.+ * Four GHASH steps at once, which here is one call rather than four: the+ * loop inside br_ghash_ctmul64 takes the blocks as they come.  */ void crypton_aes_generic_gf_mul4(block128 *a, const block128 *blocks, const table_4bit htable) {-	int i;--	for (i = 0; i < 4; i++) {-		block128_xor(a, &blocks[i]);-		crypton_aes_generic_gf_mul(a, htable);-	}+	br_ghash_ctmul64(a, &htable[0], blocks, 4 * sizeof(block128)); }

@@ -0,0 +1,287 @@+/*+ * AES and GHASH using the PowerISA 2.07 vector instructions, first+ * implemented by POWER8.+ *+ * The instructions themselves come from CRYPTOGAMS, assembled from+ * cbits/asm/aesp8-ppc-*.S and cbits/asm/ghashp8-ppc-*.S; what is here is the+ * glue that puts them behind crypton's branch table.  Nothing in this file+ * branches or indexes on a key or on data.+ *+ * == Where the key schedule lives+ *+ * The assembly takes OpenSSL's AES_KEY -- sixty round-key words and a round+ * count, 244 bytes -- and its set_decrypt_key writes a second, complete one+ * rather than sharing ends with the first the way cbits/aes/armv8.c does.+ * Two of those are 488 bytes and aes_key.data is 448, so both do not fit.+ *+ * So the forward schedule is kept there, with the key the caller gave after+ * it, and the inverse is built on the stack by the operations that need it.+ * Those are all bulk -- ECB, CBC and XTS decryption -- so it is one key+ * schedule per call rather than per block.  Single-block decryption pays for+ * one too, and is reached by nothing that runs in a loop: OCB and CCM drive+ * the block function in the encrypting direction.+ */++#include <stdint.h>+#include <string.h>+#include <crypton_aes.h>+#include <crypton_bitfn.h>+#include "aes/block128.h"+#include "aes/gf.h"+#include "aes/ppc8.h"++/* OpenSSL's AES_KEY, which is what the assembly was written against */+typedef struct {+	unsigned int rd_key[60];+	int rounds;+} p8_key;++int  crypton_aes_p8_set_encrypt_key(const unsigned char *, int, p8_key *);+int  crypton_aes_p8_set_decrypt_key(const unsigned char *, int, p8_key *);+void crypton_aes_p8_encrypt(const unsigned char *, unsigned char *, const p8_key *);+void crypton_aes_p8_decrypt(const unsigned char *, unsigned char *, const p8_key *);+void crypton_aes_p8_cbc_encrypt(const unsigned char *, unsigned char *, size_t,+                                const p8_key *, unsigned char *, int);+void crypton_aes_p8_ctr32_encrypt_blocks(const unsigned char *, unsigned char *,+                                         size_t, const p8_key *,+                                         const unsigned char *);+void crypton_aes_p8_xts_encrypt(const unsigned char *, unsigned char *, size_t,+                                const p8_key *, const p8_key *,+                                const unsigned char *);+void crypton_aes_p8_xts_decrypt(const unsigned char *, unsigned char *, size_t,+                                const p8_key *, const p8_key *,+                                const unsigned char *);+/* void * rather than uint64_t *: the assembly loads these with lvx_u and so+ * does not want the alignment a uint64_t pointer would promise, and+ * block128 is packed. */+void crypton_gcm_init_p8(void *Htable, const void *H);+void crypton_gcm_gmult_p8(void *Xi, const void *Htable);+void crypton_gcm_ghash_p8(void *Xi, const void *Htable,+                          const unsigned char *inp, size_t len);++#define FORWARD(k)  ((p8_key *) (void *) (k)->data)+#define USERKEY(k)  ((uint8_t *) (k)->data + sizeof(p8_key))+#define USERLEN(k)  (*((uint8_t *) (k)->data + sizeof(p8_key) + 32))++static void inverse(p8_key *dk, aes_key *key)+{+	crypton_aes_p8_set_decrypt_key(USERKEY(key), USERLEN(key) * 8, dk);+}++void crypton_aes_ppc8_init(aes_key *key, uint8_t *origkey, uint8_t size)+{+	if (size != 16 && size != 24 && size != 32)+		return;+	crypton_aes_p8_set_encrypt_key(origkey, size * 8, FORWARD(key));+	memcpy(USERKEY(key), origkey, size);+	USERLEN(key) = size;+}++void crypton_aes_ppc8_encrypt_block(aes_block *output, aes_key *key, aes_block *input)+{+	crypton_aes_p8_encrypt((const unsigned char *) input,+	                       (unsigned char *) output, FORWARD(key));+}++void crypton_aes_ppc8_decrypt_block(aes_block *output, aes_key *key, aes_block *input)+{+	p8_key dk;++	inverse(&dk, key);+	crypton_aes_p8_decrypt((const unsigned char *) input,+	                       (unsigned char *) output, &dk);+	memset(&dk, 0, sizeof dk);+}++/* The assembly has no ECB entry: it is the block function in a loop, which+ * is what the generic implementation does too. */+void crypton_aes_ppc8_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input,+                                  uint32_t nb_blocks)+{+	for (; nb_blocks-- > 0; input++, output++)+		crypton_aes_p8_encrypt((const unsigned char *) input,+		                       (unsigned char *) output, FORWARD(key));+}++void crypton_aes_ppc8_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input,+                                  uint32_t nb_blocks)+{+	p8_key dk;++	inverse(&dk, key);+	for (; nb_blocks-- > 0; input++, output++)+		crypton_aes_p8_decrypt((const unsigned char *) input,+		                       (unsigned char *) output, &dk);+	memset(&dk, 0, sizeof dk);+}++void crypton_aes_ppc8_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv,+                                  aes_block *input, uint32_t nb_blocks)+{+	uint8_t ivbuf[16];++	/* a copy, because the assembly writes the last block back through this+	 * pointer and the callers of this entry do not expect their IV touched */+	memcpy(ivbuf, iv, 16);+	crypton_aes_p8_cbc_encrypt((const unsigned char *) input,+	                           (unsigned char *) output,+	                           (size_t) nb_blocks * 16, FORWARD(key), ivbuf, 1);+}++void crypton_aes_ppc8_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv,+                                  aes_block *input, uint32_t nb_blocks)+{+	p8_key dk;+	uint8_t ivbuf[16];++	inverse(&dk, key);+	memcpy(ivbuf, iv, 16);+	crypton_aes_p8_cbc_encrypt((const unsigned char *) input,+	                           (unsigned char *) output,+	                           (size_t) nb_blocks * 16, &dk, ivbuf, 0);+	memset(&dk, 0, sizeof dk);+}++/*+ * CTR, which is where the two counters have to be reconciled.+ *+ * crypton counts with block128_inc_be, over the whole 128 bits; the assembly+ * counts over the low 32 only.  They agree until that word wraps, so the+ * work is handed over in runs that stop there, and the carry into the upper+ * bits is done here.  This is the same arrangement OpenSSL makes for the+ * same reason.+ */+void crypton_aes_ppc8_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv,+                                  uint8_t *input, uint32_t len)+{+	block128 ctr;+	uint32_t nb_blocks = len / 16;+	uint32_t tail = len % 16;+	uint32_t i;++	block128_copy(&ctr, iv);++	while (nb_blocks > 0) {+		uint64_t low = (uint64_t) be32_to_cpu(ctr.d[3]);+		uint64_t room = 0x100000000ULL - low;   /* blocks before it wraps */+		uint32_t n;++		/* room is 2^32 when the word is at zero, which does not fit the+		 * type the comparison would narrow it to -- and a zero n here+		 * would not advance */+		if (room > (uint64_t) nb_blocks)+			room = (uint64_t) nb_blocks;+		n = (uint32_t) room;++		crypton_aes_p8_ctr32_encrypt_blocks((const unsigned char *) input,+		                                    (unsigned char *) output,+		                                    n, FORWARD(key),+		                                    (const unsigned char *) &ctr);+		/* the assembly does not write the counter back */+		for (i = 0; i < n; i++)+			block128_inc_be(&ctr);++		output += (size_t) n * 16;+		input += (size_t) n * 16;+		nb_blocks -= n;+	}++	if (tail != 0) {+		block128 ks;++		crypton_aes_p8_encrypt((const unsigned char *) &ctr,+		                       (unsigned char *) &ks, FORWARD(key));+		for (i = 0; i < tail; i++)+			output[i] = ks.b[i] ^ input[i];+	}+}++/*+ * XTS.  The assembly encrypts the tweak with its second key, unless that key+ * is NULL, in which case it takes the tweak already encrypted -- which is+ * what this needs, because crypton's entry also carries a starting point and+ * the tweak has to be advanced that many doublings before any block is+ * enciphered.+ */+static void xts_tweak(block128 *tweak, aes_key *k2, aes_block *dataunit,+                      uint32_t spoint)+{+	block128_copy(tweak, dataunit);+	crypton_aes_p8_encrypt((const unsigned char *) tweak,+	                       (unsigned char *) tweak, FORWARD(k2));+	while (spoint-- > 0)+		crypton_aes_generic_gf_mulx(tweak);+}++void crypton_aes_ppc8_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2,+                                  aes_block *dataunit, uint32_t spoint,+                                  aes_block *input, uint32_t nb_blocks)+{+	block128 tweak;++	xts_tweak(&tweak, k2, dataunit, spoint);+	crypton_aes_p8_xts_encrypt((const unsigned char *) input,+	                           (unsigned char *) output,+	                           (size_t) nb_blocks * 16, FORWARD(k1), NULL,+	                           (const unsigned char *) &tweak);+}++void crypton_aes_ppc8_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2,+                                  aes_block *dataunit, uint32_t spoint,+                                  aes_block *input, uint32_t nb_blocks)+{+	block128 tweak;+	p8_key dk;++	/* the tweak is enciphered with k2 forwards either way */+	xts_tweak(&tweak, k2, dataunit, spoint);+	inverse(&dk, k1);+	crypton_aes_p8_xts_decrypt((const unsigned char *) input,+	                           (unsigned char *) output,+	                           (size_t) nb_blocks * 16, &dk, NULL,+	                           (const unsigned char *) &tweak);+	memset(&dk, 0, sizeof dk);+}++/*+ * GHASH.  The table the assembly builds is 192 bytes, which fits the sixteen+ * blocks the interface names.+ *+ * The two arguments do not take the same convention, which is worth saying+ * because it does not look like an accident until it is checked.  H arrives+ * here as the sixteen bytes GCM defines, and gcm_init_p8 wants those as a+ * pair of host-order words -- so they are swapped on a little-endian machine+ * and left alone on a big-endian one, which be64_to_cpu does.  The+ * accumulator is the byte string throughout, in and out, and is passed as it+ * stands.  OpenSSL makes the same pair of choices.+ *+ * Checked rather than assumed: against crypton's own GHASH, over two+ * different H and with a non-zero accumulator, so that neither a symmetric+ * value nor a zero start could hide a wrong one.+ */+void crypton_aes_ppc8_hinit(table_4bit htable, const block128 *h)+{+	uint64_t H[2];++	memcpy(H, h, sizeof H);+	H[0] = be64_to_cpu(H[0]);+	H[1] = be64_to_cpu(H[1]);+	crypton_gcm_init_p8(htable, H);+}++void crypton_aes_ppc8_gf_mul(block128 *a, const table_4bit htable)+{+	crypton_gcm_gmult_p8(a, htable);+}++void crypton_aes_ppc8_gf_mul4(block128 *a, const block128 *blocks,+                              const table_4bit htable)+{+	crypton_gcm_ghash_p8(a, htable, (const unsigned char *) blocks,+	                     4 * sizeof(block128));+}++int crypton_aes_ppc8_available(void)+{+	return (crypton_ppc_features() & CRYPTON_PPC_VCRYPTO) != 0;+}

@@ -0,0 +1,29 @@+/*+ * AES and GHASH on PowerISA 2.07, as implemented by cbits/aes/ppc8.c over+ * the CRYPTOGAMS assembly.  crypton_aes.c installs these when+ * crypton_aes_ppc8_available says the processor has the instructions.+ */+#ifndef CRYPTON_AES_PPC8_H+#define CRYPTON_AES_PPC8_H++#include "crypton_aes.h"+#include "aes/gf.h"+#include "crypton_cpu.h"++int  crypton_aes_ppc8_available(void);++void crypton_aes_ppc8_init(aes_key *key, uint8_t *origkey, uint8_t size);+void crypton_aes_ppc8_encrypt_block(aes_block *output, aes_key *key, aes_block *input);+void crypton_aes_ppc8_decrypt_block(aes_block *output, aes_key *key, aes_block *input);+void crypton_aes_ppc8_encrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_decrypt_ecb(aes_block *output, aes_key *key, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_encrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_decrypt_cbc(aes_block *output, aes_key *key, aes_block *iv, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_encrypt_ctr(uint8_t *output, aes_key *key, aes_block *iv, uint8_t *input, uint32_t len);+void crypton_aes_ppc8_encrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_decrypt_xts(aes_block *output, aes_key *k1, aes_key *k2, aes_block *dataunit, uint32_t spoint, aes_block *input, uint32_t nb_blocks);+void crypton_aes_ppc8_hinit(table_4bit htable, const block128 *h);+void crypton_aes_ppc8_gf_mul(block128 *a, const table_4bit htable);+void crypton_aes_ppc8_gf_mul4(block128 *a, const block128 *blocks, const table_4bit htable);++#endif

@@ -2,7 +2,7 @@  ## What is here -Two modules from [CRYPTOGAMS](https://github.com/dot-asm/cryptogams), by Andy+Modules from [CRYPTOGAMS](https://github.com/dot-asm/cryptogams), by Andy Polyakov, checked in unmodified together with the translators they need:  | generator | what it is |@@ -17,6 +17,8 @@ | `sha1-armv8.pl` | SHA-1 for AArch64 | | `sha512-armv8.pl` | SHA-256 for AArch64 (the generator emits SHA-512 or SHA-256 according to the name it is given, and only the latter is wanted) | | `keccak1600-armv8.pl` | Keccak for AArch64 |+| `aesp8-ppc.pl` | AES for PowerISA 2.07, which POWER8 was the first to implement |+| `ghashp8-ppc.pl` | GHASH for the same, over `vpmsumd` |  `x86_64-xlate.pl`, `arm-xlate.pl` and `arm_arch.h` are the machinery those modules use.  `generate.sh` runs the generators to produce the `.S` files, which are

@@ -0,0 +1,3658 @@+.machine	"any"++.abiversion	2+.text++.align	7+rcon:+.byte	0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01+.byte	0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b+.byte	0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d+.byte	0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00+.Lconsts:+	mflr	0+	bcl	20,31,$+4+	mflr	6+	addi	6,6,-0x48+	mtlr	0+	blr	+.long	0+.byte	0,12,0x14,0,0,0,0,0+.byte	65,69,83,32,102,111,114,32,80,111,119,101,114,73,83,65,32,50,46,48,55,44,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0+.align	2++.globl	crypton_aes_p8_set_encrypt_key+.type	crypton_aes_p8_set_encrypt_key,@function+.align	5+crypton_aes_p8_set_encrypt_key:+.localentry	crypton_aes_p8_set_encrypt_key,0++.Lset_encrypt_key:+	mflr	11+	std	11,16(1)++	li	6,-1+	cmpldi	3,0+	beq-	.Lenc_key_abort+	cmpldi	5,0+	beq-	.Lenc_key_abort+	li	6,-2+	cmpwi	4,128+	blt-	.Lenc_key_abort+	cmpwi	4,256+	bgt-	.Lenc_key_abort+	andi.	0,4,0x3f+	bne-	.Lenc_key_abort++	lis	0,0xfff0+	li	12,-1+	or	0,0,0++	bl	.Lconsts+	mtlr	11++	neg	9,3+	lvx	1,0,3+	addi	3,3,15+	lvsr	3,0,9+	li	8,0x20+	cmpwi	4,192+	lvx	2,0,3+	vspltisb	5,0x0f+	lvx	4,0,6+	vxor	3,3,5+	lvx	5,8,6+	addi	6,6,0x10+	vperm	1,1,2,3+	li	7,8+	vxor	0,0,0+	mtctr	7++	lvsl	8,0,5+	vspltisb	9,-1+	lvx	10,0,5+	vperm	9,9,0,8++	blt	.Loop128+	addi	3,3,8+	beq	.L192+	addi	3,3,8+	b	.L256++.align	4+.Loop128:+	vperm	3,1,1,5+	vsldoi	6,0,1,12+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	.long	0x10632509+	stvx	7,0,5+	addi	5,5,16++	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vadduwm	4,4,4+	vxor	1,1,3+	bdnz	.Loop128++	lvx	4,0,6++	vperm	3,1,1,5+	vsldoi	6,0,1,12+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	.long	0x10632509+	stvx	7,0,5+	addi	5,5,16++	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vadduwm	4,4,4+	vxor	1,1,3++	vperm	3,1,1,5+	vsldoi	6,0,1,12+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	.long	0x10632509+	stvx	7,0,5+	addi	5,5,16++	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vxor	1,1,3+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	stvx	7,0,5++	addi	3,5,15+	addi	5,5,0x50++	li	8,10+	b	.Ldone++.align	4+.L192:+	lvx	6,0,3+	li	7,4+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	stvx	7,0,5+	addi	5,5,16+	vperm	2,2,6,3+	vspltisb	3,8+	mtctr	7+	vsububm	5,5,3++.Loop192:+	vperm	3,2,2,5+	vsldoi	6,0,1,12+	.long	0x10632509++	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6++	vsldoi	7,0,2,8+	vspltw	6,1,3+	vxor	6,6,2+	vsldoi	2,0,2,12+	vadduwm	4,4,4+	vxor	2,2,6+	vxor	1,1,3+	vxor	2,2,3+	vsldoi	7,7,1,8++	vperm	3,2,2,5+	vsldoi	6,0,1,12+	vperm	11,7,7,8+	vsel	7,10,11,9+	vor	10,11,11+	.long	0x10632509+	stvx	7,0,5+	addi	5,5,16++	vsldoi	7,1,2,8+	vxor	1,1,6+	vsldoi	6,0,6,12+	vperm	11,7,7,8+	vsel	7,10,11,9+	vor	10,11,11+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	stvx	7,0,5+	addi	5,5,16++	vspltw	6,1,3+	vxor	6,6,2+	vsldoi	2,0,2,12+	vadduwm	4,4,4+	vxor	2,2,6+	vxor	1,1,3+	vxor	2,2,3+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	stvx	7,0,5+	addi	3,5,15+	addi	5,5,16+	bdnz	.Loop192++	li	8,12+	addi	5,5,0x20+	b	.Ldone++.align	4+.L256:+	lvx	6,0,3+	li	7,7+	li	8,14+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	stvx	7,0,5+	addi	5,5,16+	vperm	2,2,6,3+	mtctr	7++.Loop256:+	vperm	3,2,2,5+	vsldoi	6,0,1,12+	vperm	11,2,2,8+	vsel	7,10,11,9+	vor	10,11,11+	.long	0x10632509+	stvx	7,0,5+	addi	5,5,16++	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vsldoi	6,0,6,12+	vxor	1,1,6+	vadduwm	4,4,4+	vxor	1,1,3+	vperm	11,1,1,8+	vsel	7,10,11,9+	vor	10,11,11+	stvx	7,0,5+	addi	3,5,15+	addi	5,5,16+	bdz	.Ldone++	vspltw	3,1,3+	vsldoi	6,0,2,12+	.long	0x106305C8++	vxor	2,2,6+	vsldoi	6,0,6,12+	vxor	2,2,6+	vsldoi	6,0,6,12+	vxor	2,2,6++	vxor	2,2,3+	b	.Loop256++.align	4+.Ldone:+	lvx	2,0,3+	vsel	2,10,2,9+	stvx	2,0,3+	li	6,0+	or	12,12,12+	stw	8,0(5)++.Lenc_key_abort:+	mr	3,6+	blr	+.long	0+.byte	0,12,0x14,1,0,0,3,0+.long	0+.size	crypton_aes_p8_set_encrypt_key,.-crypton_aes_p8_set_encrypt_key++.globl	crypton_aes_p8_set_decrypt_key+.type	crypton_aes_p8_set_decrypt_key,@function+.align	5+crypton_aes_p8_set_decrypt_key:+.localentry	crypton_aes_p8_set_decrypt_key,0++	stdu	1,-64(1)+	mflr	10+	std	10,64+16(1)+	bl	.Lset_encrypt_key+	mtlr	10++	cmpwi	3,0+	bne-	.Ldec_key_abort++	slwi	7,8,4+	subi	3,5,240+	srwi	8,8,1+	add	5,3,7+	mtctr	8++.Ldeckey:+	lwz	0, 0(3)+	lwz	6, 4(3)+	lwz	7, 8(3)+	lwz	8, 12(3)+	addi	3,3,16+	lwz	9, 0(5)+	lwz	10,4(5)+	lwz	11,8(5)+	lwz	12,12(5)+	stw	0, 0(5)+	stw	6, 4(5)+	stw	7, 8(5)+	stw	8, 12(5)+	subi	5,5,16+	stw	9, -16(3)+	stw	10,-12(3)+	stw	11,-8(3)+	stw	12,-4(3)+	bdnz	.Ldeckey++	xor	3,3,3+.Ldec_key_abort:+	addi	1,1,64+	blr	+.long	0+.byte	0,12,4,1,0x80,0,3,0+.long	0+.size	crypton_aes_p8_set_decrypt_key,.-crypton_aes_p8_set_decrypt_key+.globl	crypton_aes_p8_encrypt+.type	crypton_aes_p8_encrypt,@function+.align	5+crypton_aes_p8_encrypt:+.localentry	crypton_aes_p8_encrypt,0++	lwz	6,240(5)+	lis	0,0xfc00+	li	12,-1+	li	7,15+	or	0,0,0++	lvx	0,0,3+	neg	11,4+	lvx	1,7,3+	lvsl	2,0,3+	vspltisb	4,0x0f+	lvsr	3,0,11+	vxor	2,2,4+	li	7,16+	vperm	0,0,1,2+	lvx	1,0,5+	lvsr	5,0,5+	srwi	6,6,1+	lvx	2,7,5+	addi	7,7,16+	subi	6,6,1+	vperm	1,2,1,5++	vxor	0,0,1+	lvx	1,7,5+	addi	7,7,16+	mtctr	6++.Loop_enc:+	vperm	2,1,2,5+	.long	0x10001508+	lvx	2,7,5+	addi	7,7,16+	vperm	1,2,1,5+	.long	0x10000D08+	lvx	1,7,5+	addi	7,7,16+	bdnz	.Loop_enc++	vperm	2,1,2,5+	.long	0x10001508+	lvx	2,7,5+	vperm	1,2,1,5+	.long	0x10000D09++	vspltisb	2,-1+	vxor	1,1,1+	li	7,15+	vperm	2,2,1,3+	vxor	3,3,4+	lvx	1,0,4+	vperm	0,0,0,3+	vsel	1,1,0,2+	lvx	4,7,4+	stvx	1,0,4+	vsel	0,0,4,2+	stvx	0,7,4++	or	12,12,12+	blr	+.long	0+.byte	0,12,0x14,0,0,0,3,0+.long	0+.size	crypton_aes_p8_encrypt,.-crypton_aes_p8_encrypt+.globl	crypton_aes_p8_decrypt+.type	crypton_aes_p8_decrypt,@function+.align	5+crypton_aes_p8_decrypt:+.localentry	crypton_aes_p8_decrypt,0++	lwz	6,240(5)+	lis	0,0xfc00+	li	12,-1+	li	7,15+	or	0,0,0++	lvx	0,0,3+	neg	11,4+	lvx	1,7,3+	lvsl	2,0,3+	vspltisb	4,0x0f+	lvsr	3,0,11+	vxor	2,2,4+	li	7,16+	vperm	0,0,1,2+	lvx	1,0,5+	lvsr	5,0,5+	srwi	6,6,1+	lvx	2,7,5+	addi	7,7,16+	subi	6,6,1+	vperm	1,2,1,5++	vxor	0,0,1+	lvx	1,7,5+	addi	7,7,16+	mtctr	6++.Loop_dec:+	vperm	2,1,2,5+	.long	0x10001548+	lvx	2,7,5+	addi	7,7,16+	vperm	1,2,1,5+	.long	0x10000D48+	lvx	1,7,5+	addi	7,7,16+	bdnz	.Loop_dec++	vperm	2,1,2,5+	.long	0x10001548+	lvx	2,7,5+	vperm	1,2,1,5+	.long	0x10000D49++	vspltisb	2,-1+	vxor	1,1,1+	li	7,15+	vperm	2,2,1,3+	vxor	3,3,4+	lvx	1,0,4+	vperm	0,0,0,3+	vsel	1,1,0,2+	lvx	4,7,4+	stvx	1,0,4+	vsel	0,0,4,2+	stvx	0,7,4++	or	12,12,12+	blr	+.long	0+.byte	0,12,0x14,0,0,0,3,0+.long	0+.size	crypton_aes_p8_decrypt,.-crypton_aes_p8_decrypt+.globl	crypton_aes_p8_cbc_encrypt+.type	crypton_aes_p8_cbc_encrypt,@function+.align	5+crypton_aes_p8_cbc_encrypt:+.localentry	crypton_aes_p8_cbc_encrypt,0++	cmpldi	5,16+	.long	0x4dc00020++	cmpwi	8,0+	lis	0,0xffe0+	li	12,-1+	or	0,0,0++	li	10,15+	vxor	0,0,0+	vspltisb	3,0x0f++	lvx	4,0,7+	lvsl	6,0,7+	lvx	5,10,7+	vxor	6,6,3+	vperm	4,4,5,6++	neg	11,3+	lvsr	10,0,6+	lwz	9,240(6)++	lvsr	6,0,11+	lvx	5,0,3+	addi	3,3,15+	vxor	6,6,3++	lvsl	8,0,4+	vspltisb	9,-1+	lvx	7,0,4+	vperm	9,9,0,8+	vxor	8,8,3++	srwi	9,9,1+	li	10,16+	subi	9,9,1+	beq	.Lcbc_dec++.Lcbc_enc:+	vor	2,5,5+	lvx	5,0,3+	addi	3,3,16+	mtctr	9+	subi	5,5,16++	lvx	0,0,6+	vperm	2,2,5,6+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	vxor	2,2,0+	lvx	0,10,6+	addi	10,10,16+	vxor	2,2,4++.Loop_cbc_enc:+	vperm	1,0,1,10+	.long	0x10420D08+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	.long	0x10420508+	lvx	0,10,6+	addi	10,10,16+	bdnz	.Loop_cbc_enc++	vperm	1,0,1,10+	.long	0x10420D08+	lvx	1,10,6+	li	10,16+	vperm	0,1,0,10+	.long	0x10820509+	cmpldi	5,16++	vperm	3,4,4,8+	vsel	2,7,3,9+	vor	7,3,3+	stvx	2,0,4+	addi	4,4,16+	bge	.Lcbc_enc++	b	.Lcbc_done++.align	4+.Lcbc_dec:+	cmpldi	5,128+	bge	_aesp8_cbc_decrypt8x+	vor	3,5,5+	lvx	5,0,3+	addi	3,3,16+	mtctr	9+	subi	5,5,16++	lvx	0,0,6+	vperm	3,3,5,6+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	vxor	2,3,0+	lvx	0,10,6+	addi	10,10,16++.Loop_cbc_dec:+	vperm	1,0,1,10+	.long	0x10420D48+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	.long	0x10420548+	lvx	0,10,6+	addi	10,10,16+	bdnz	.Loop_cbc_dec++	vperm	1,0,1,10+	.long	0x10420D48+	lvx	1,10,6+	li	10,16+	vperm	0,1,0,10+	.long	0x10420549+	cmpldi	5,16++	vxor	2,2,4+	vor	4,3,3+	vperm	3,2,2,8+	vsel	2,7,3,9+	vor	7,3,3+	stvx	2,0,4+	addi	4,4,16+	bge	.Lcbc_dec++.Lcbc_done:+	addi	4,4,-1+	lvx	2,0,4+	vsel	2,7,2,9+	stvx	2,0,4++	neg	8,7+	li	10,15+	vxor	0,0,0+	vspltisb	9,-1+	vspltisb	3,0x0f+	lvsr	8,0,8+	vperm	9,9,0,8+	vxor	8,8,3+	lvx	7,0,7+	vperm	4,4,4,8+	vsel	2,7,4,9+	lvx	5,10,7+	stvx	2,0,7+	vsel	2,4,5,9+	stvx	2,10,7++	or	12,12,12+	blr	+.long	0+.byte	0,12,0x14,0,0,0,6,0+.long	0+.align	5+_aesp8_cbc_decrypt8x:+	stdu	1,-448(1)+	li	10,207+	li	11,223+	stvx	20,10,1+	addi	10,10,32+	stvx	21,11,1+	addi	11,11,32+	stvx	22,10,1+	addi	10,10,32+	stvx	23,11,1+	addi	11,11,32+	stvx	24,10,1+	addi	10,10,32+	stvx	25,11,1+	addi	11,11,32+	stvx	26,10,1+	addi	10,10,32+	stvx	27,11,1+	addi	11,11,32+	stvx	28,10,1+	addi	10,10,32+	stvx	29,11,1+	addi	11,11,32+	stvx	30,10,1+	stvx	31,11,1+	li	0,-1+	stw	12,396(1)+	li	8,0x10+	std	26,400(1)+	li	26,0x20+	std	27,408(1)+	li	27,0x30+	std	28,416(1)+	li	28,0x40+	std	29,424(1)+	li	29,0x50+	std	30,432(1)+	li	30,0x60+	std	31,440(1)+	li	31,0x70+	or	0,0,0++	subi	9,9,3+	subi	5,5,128++	lvx	23,0,6+	lvx	30,8,6+	addi	6,6,0x20+	lvx	31,0,6+	vperm	23,30,23,10+	addi	11,1,64+15+	mtctr	9++.Load_cbc_dec_key:+	vperm	24,31,30,10+	lvx	30,8,6+	addi	6,6,0x20+	stvx	24,0,11+	vperm	25,30,31,10+	lvx	31,0,6+	stvx	25,8,11+	addi	11,11,0x20+	bdnz	.Load_cbc_dec_key++	lvx	26,8,6+	vperm	24,31,30,10+	lvx	27,26,6+	stvx	24,0,11+	vperm	25,26,31,10+	lvx	28,27,6+	stvx	25,8,11+	addi	11,1,64+15+	vperm	26,27,26,10+	lvx	29,28,6+	vperm	27,28,27,10+	lvx	30,29,6+	vperm	28,29,28,10+	lvx	31,30,6+	vperm	29,30,29,10+	lvx	14,31,6+	vperm	30,31,30,10+	lvx	24,0,11+	vperm	31,14,31,10+	lvx	25,8,11++++	subi	3,3,15++	li	10,8+	.long	0x7C001E99+	lvsl	6,0,10+	vspltisb	3,0x0f+	.long	0x7C281E99+	vxor	6,6,3+	.long	0x7C5A1E99+	vperm	0,0,0,6+	.long	0x7C7B1E99+	vperm	1,1,1,6+	.long	0x7D5C1E99+	vperm	2,2,2,6+	vxor	14,0,23+	.long	0x7D7D1E99+	vperm	3,3,3,6+	vxor	15,1,23+	.long	0x7D9E1E99+	vperm	10,10,10,6+	vxor	16,2,23+	.long	0x7DBF1E99+	addi	3,3,0x80+	vperm	11,11,11,6+	vxor	17,3,23+	vperm	12,12,12,6+	vxor	18,10,23+	vperm	13,13,13,6+	vxor	19,11,23+	vxor	20,12,23+	vxor	21,13,23++	mtctr	9+	b	.Loop_cbc_dec8x+.align	5+.Loop_cbc_dec8x:+	.long	0x11CEC548+	.long	0x11EFC548+	.long	0x1210C548+	.long	0x1231C548+	.long	0x1252C548+	.long	0x1273C548+	.long	0x1294C548+	.long	0x12B5C548+	lvx	24,26,11+	addi	11,11,0x20++	.long	0x11CECD48+	.long	0x11EFCD48+	.long	0x1210CD48+	.long	0x1231CD48+	.long	0x1252CD48+	.long	0x1273CD48+	.long	0x1294CD48+	.long	0x12B5CD48+	lvx	25,8,11+	bdnz	.Loop_cbc_dec8x++	subic	5,5,128+	.long	0x11CEC548+	.long	0x11EFC548+	.long	0x1210C548+	.long	0x1231C548+	.long	0x1252C548+	.long	0x1273C548+	.long	0x1294C548+	.long	0x12B5C548++	subfe.	0,0,0+	.long	0x11CECD48+	.long	0x11EFCD48+	.long	0x1210CD48+	.long	0x1231CD48+	.long	0x1252CD48+	.long	0x1273CD48+	.long	0x1294CD48+	.long	0x12B5CD48++	and	0,0,5+	.long	0x11CED548+	.long	0x11EFD548+	.long	0x1210D548+	.long	0x1231D548+	.long	0x1252D548+	.long	0x1273D548+	.long	0x1294D548+	.long	0x12B5D548++	add	3,3,0++++	.long	0x11CEDD48+	.long	0x11EFDD48+	.long	0x1210DD48+	.long	0x1231DD48+	.long	0x1252DD48+	.long	0x1273DD48+	.long	0x1294DD48+	.long	0x12B5DD48++	addi	11,1,64+15+	.long	0x11CEE548+	.long	0x11EFE548+	.long	0x1210E548+	.long	0x1231E548+	.long	0x1252E548+	.long	0x1273E548+	.long	0x1294E548+	.long	0x12B5E548+	lvx	24,0,11++	.long	0x11CEED48+	.long	0x11EFED48+	.long	0x1210ED48+	.long	0x1231ED48+	.long	0x1252ED48+	.long	0x1273ED48+	.long	0x1294ED48+	.long	0x12B5ED48+	lvx	25,8,11++	.long	0x11CEF548+	vxor	4,4,31+	.long	0x11EFF548+	vxor	0,0,31+	.long	0x1210F548+	vxor	1,1,31+	.long	0x1231F548+	vxor	2,2,31+	.long	0x1252F548+	vxor	3,3,31+	.long	0x1273F548+	vxor	10,10,31+	.long	0x1294F548+	vxor	11,11,31+	.long	0x12B5F548+	vxor	12,12,31++	.long	0x11CE2549+	.long	0x11EF0549+	.long	0x7C001E99+	.long	0x12100D49+	.long	0x7C281E99+	.long	0x12311549+	vperm	0,0,0,6+	.long	0x7C5A1E99+	.long	0x12521D49+	vperm	1,1,1,6+	.long	0x7C7B1E99+	.long	0x12735549+	vperm	2,2,2,6+	.long	0x7D5C1E99+	.long	0x12945D49+	vperm	3,3,3,6+	.long	0x7D7D1E99+	.long	0x12B56549+	vperm	10,10,10,6+	.long	0x7D9E1E99+	vor	4,13,13+	vperm	11,11,11,6+	.long	0x7DBF1E99+	addi	3,3,0x80++	vperm	14,14,14,6+	vperm	15,15,15,6+	.long	0x7DC02799+	vperm	12,12,12,6+	vxor	14,0,23+	vperm	16,16,16,6+	.long	0x7DE82799+	vperm	13,13,13,6+	vxor	15,1,23+	vperm	17,17,17,6+	.long	0x7E1A2799+	vxor	16,2,23+	vperm	18,18,18,6+	.long	0x7E3B2799+	vxor	17,3,23+	vperm	19,19,19,6+	.long	0x7E5C2799+	vxor	18,10,23+	vperm	20,20,20,6+	.long	0x7E7D2799+	vxor	19,11,23+	vperm	21,21,21,6+	.long	0x7E9E2799+	vxor	20,12,23+	.long	0x7EBF2799+	addi	4,4,0x80+	vxor	21,13,23++	mtctr	9+	beq	.Loop_cbc_dec8x++	addic.	5,5,128+	beq	.Lcbc_dec8x_done+	nop	+	nop	++.Loop_cbc_dec8x_tail:+	.long	0x11EFC548+	.long	0x1210C548+	.long	0x1231C548+	.long	0x1252C548+	.long	0x1273C548+	.long	0x1294C548+	.long	0x12B5C548+	lvx	24,26,11+	addi	11,11,0x20++	.long	0x11EFCD48+	.long	0x1210CD48+	.long	0x1231CD48+	.long	0x1252CD48+	.long	0x1273CD48+	.long	0x1294CD48+	.long	0x12B5CD48+	lvx	25,8,11+	bdnz	.Loop_cbc_dec8x_tail++	.long	0x11EFC548+	.long	0x1210C548+	.long	0x1231C548+	.long	0x1252C548+	.long	0x1273C548+	.long	0x1294C548+	.long	0x12B5C548++	.long	0x11EFCD48+	.long	0x1210CD48+	.long	0x1231CD48+	.long	0x1252CD48+	.long	0x1273CD48+	.long	0x1294CD48+	.long	0x12B5CD48++	.long	0x11EFD548+	.long	0x1210D548+	.long	0x1231D548+	.long	0x1252D548+	.long	0x1273D548+	.long	0x1294D548+	.long	0x12B5D548++	.long	0x11EFDD48+	.long	0x1210DD48+	.long	0x1231DD48+	.long	0x1252DD48+	.long	0x1273DD48+	.long	0x1294DD48+	.long	0x12B5DD48++	.long	0x11EFE548+	.long	0x1210E548+	.long	0x1231E548+	.long	0x1252E548+	.long	0x1273E548+	.long	0x1294E548+	.long	0x12B5E548++	.long	0x11EFED48+	.long	0x1210ED48+	.long	0x1231ED48+	.long	0x1252ED48+	.long	0x1273ED48+	.long	0x1294ED48+	.long	0x12B5ED48++	.long	0x11EFF548+	vxor	4,4,31+	.long	0x1210F548+	vxor	1,1,31+	.long	0x1231F548+	vxor	2,2,31+	.long	0x1252F548+	vxor	3,3,31+	.long	0x1273F548+	vxor	10,10,31+	.long	0x1294F548+	vxor	11,11,31+	.long	0x12B5F548+	vxor	12,12,31++	cmplwi	5,32+	blt	.Lcbc_dec8x_one+	nop	+	beq	.Lcbc_dec8x_two+	cmplwi	5,64+	blt	.Lcbc_dec8x_three+	nop	+	beq	.Lcbc_dec8x_four+	cmplwi	5,96+	blt	.Lcbc_dec8x_five+	nop	+	beq	.Lcbc_dec8x_six++.Lcbc_dec8x_seven:+	.long	0x11EF2549+	.long	0x12100D49+	.long	0x12311549+	.long	0x12521D49+	.long	0x12735549+	.long	0x12945D49+	.long	0x12B56549+	vor	4,13,13++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	vperm	19,19,19,6+	.long	0x7E5B2799+	vperm	20,20,20,6+	.long	0x7E7C2799+	vperm	21,21,21,6+	.long	0x7E9D2799+	.long	0x7EBE2799+	addi	4,4,0x70+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_six:+	.long	0x12102549+	.long	0x12311549+	.long	0x12521D49+	.long	0x12735549+	.long	0x12945D49+	.long	0x12B56549+	vor	4,13,13++	vperm	16,16,16,6+	vperm	17,17,17,6+	.long	0x7E002799+	vperm	18,18,18,6+	.long	0x7E282799+	vperm	19,19,19,6+	.long	0x7E5A2799+	vperm	20,20,20,6+	.long	0x7E7B2799+	vperm	21,21,21,6+	.long	0x7E9C2799+	.long	0x7EBD2799+	addi	4,4,0x60+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_five:+	.long	0x12312549+	.long	0x12521D49+	.long	0x12735549+	.long	0x12945D49+	.long	0x12B56549+	vor	4,13,13++	vperm	17,17,17,6+	vperm	18,18,18,6+	.long	0x7E202799+	vperm	19,19,19,6+	.long	0x7E482799+	vperm	20,20,20,6+	.long	0x7E7A2799+	vperm	21,21,21,6+	.long	0x7E9B2799+	.long	0x7EBC2799+	addi	4,4,0x50+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_four:+	.long	0x12522549+	.long	0x12735549+	.long	0x12945D49+	.long	0x12B56549+	vor	4,13,13++	vperm	18,18,18,6+	vperm	19,19,19,6+	.long	0x7E402799+	vperm	20,20,20,6+	.long	0x7E682799+	vperm	21,21,21,6+	.long	0x7E9A2799+	.long	0x7EBB2799+	addi	4,4,0x40+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_three:+	.long	0x12732549+	.long	0x12945D49+	.long	0x12B56549+	vor	4,13,13++	vperm	19,19,19,6+	vperm	20,20,20,6+	.long	0x7E602799+	vperm	21,21,21,6+	.long	0x7E882799+	.long	0x7EBA2799+	addi	4,4,0x30+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_two:+	.long	0x12942549+	.long	0x12B56549+	vor	4,13,13++	vperm	20,20,20,6+	vperm	21,21,21,6+	.long	0x7E802799+	.long	0x7EA82799+	addi	4,4,0x20+	b	.Lcbc_dec8x_done++.align	5+.Lcbc_dec8x_one:+	.long	0x12B52549+	vor	4,13,13++	vperm	21,21,21,6+	.long	0x7EA02799+	addi	4,4,0x10++.Lcbc_dec8x_done:+	vperm	4,4,4,6+	.long	0x7C803F99++	li	10,79+	li	11,95+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32++	or	12,12,12+	lvx	20,10,1+	addi	10,10,32+	lvx	21,11,1+	addi	11,11,32+	lvx	22,10,1+	addi	10,10,32+	lvx	23,11,1+	addi	11,11,32+	lvx	24,10,1+	addi	10,10,32+	lvx	25,11,1+	addi	11,11,32+	lvx	26,10,1+	addi	10,10,32+	lvx	27,11,1+	addi	11,11,32+	lvx	28,10,1+	addi	10,10,32+	lvx	29,11,1+	addi	11,11,32+	lvx	30,10,1+	lvx	31,11,1+	ld	26,400(1)+	ld	27,408(1)+	ld	28,416(1)+	ld	29,424(1)+	ld	30,432(1)+	ld	31,440(1)+	addi	1,1,448+	blr	+.long	0+.byte	0,12,0x04,0,0x80,6,6,0+.long	0+.size	crypton_aes_p8_cbc_encrypt,.-crypton_aes_p8_cbc_encrypt+.globl	crypton_aes_p8_ctr32_encrypt_blocks+.type	crypton_aes_p8_ctr32_encrypt_blocks,@function+.align	5+crypton_aes_p8_ctr32_encrypt_blocks:+.localentry	crypton_aes_p8_ctr32_encrypt_blocks,0++	cmpldi	5,1+	.long	0x4dc00020++	lis	0,0xfff0+	li	12,-1+	or	0,0,0++	li	10,15+	vxor	0,0,0+	vspltisb	3,0x0f++	lvx	4,0,7+	lvsl	6,0,7+	lvx	5,10,7+	vspltisb	11,1+	vxor	6,6,3+	vperm	4,4,5,6+	vsldoi	11,0,11,1++	neg	11,3+	lvsr	10,0,6+	lwz	9,240(6)++	lvsr	6,0,11+	lvx	5,0,3+	addi	3,3,15+	vxor	6,6,3++	srwi	9,9,1+	li	10,16+	subi	9,9,1++	cmpldi	5,8+	bge	_aesp8_ctr32_encrypt8x++	lvsl	8,0,4+	vspltisb	9,-1+	lvx	7,0,4+	vperm	9,9,0,8+	vxor	8,8,3++	lvx	0,0,6+	mtctr	9+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	vxor	2,4,0+	lvx	0,10,6+	addi	10,10,16+	b	.Loop_ctr32_enc++.align	5+.Loop_ctr32_enc:+	vperm	1,0,1,10+	.long	0x10420D08+	lvx	1,10,6+	addi	10,10,16+	vperm	0,1,0,10+	.long	0x10420508+	lvx	0,10,6+	addi	10,10,16+	bdnz	.Loop_ctr32_enc++	vadduwm	4,4,11+	vor	3,5,5+	lvx	5,0,3+	addi	3,3,16+	subic.	5,5,1++	vperm	1,0,1,10+	.long	0x10420D08+	lvx	1,10,6+	vperm	3,3,5,6+	li	10,16+	vperm	1,1,0,10+	lvx	0,0,6+	vxor	3,3,1+	.long	0x10421D09++	lvx	1,10,6+	addi	10,10,16+	vperm	2,2,2,8+	vsel	3,7,2,9+	mtctr	9+	vperm	0,1,0,10+	vor	7,2,2+	vxor	2,4,0+	lvx	0,10,6+	addi	10,10,16+	stvx	3,0,4+	addi	4,4,16+	bne	.Loop_ctr32_enc++	addi	4,4,-1+	lvx	2,0,4+	vsel	2,7,2,9+	stvx	2,0,4++	or	12,12,12+	blr	+.long	0+.byte	0,12,0x14,0,0,0,6,0+.long	0+.align	5+_aesp8_ctr32_encrypt8x:+	stdu	1,-448(1)+	li	10,207+	li	11,223+	stvx	20,10,1+	addi	10,10,32+	stvx	21,11,1+	addi	11,11,32+	stvx	22,10,1+	addi	10,10,32+	stvx	23,11,1+	addi	11,11,32+	stvx	24,10,1+	addi	10,10,32+	stvx	25,11,1+	addi	11,11,32+	stvx	26,10,1+	addi	10,10,32+	stvx	27,11,1+	addi	11,11,32+	stvx	28,10,1+	addi	10,10,32+	stvx	29,11,1+	addi	11,11,32+	stvx	30,10,1+	stvx	31,11,1+	li	0,-1+	stw	12,396(1)+	li	8,0x10+	std	26,400(1)+	li	26,0x20+	std	27,408(1)+	li	27,0x30+	std	28,416(1)+	li	28,0x40+	std	29,424(1)+	li	29,0x50+	std	30,432(1)+	li	30,0x60+	std	31,440(1)+	li	31,0x70+	or	0,0,0++	subi	9,9,3++	lvx	23,0,6+	lvx	30,8,6+	addi	6,6,0x20+	lvx	31,0,6+	vperm	23,30,23,10+	addi	11,1,64+15+	mtctr	9++.Load_ctr32_enc_key:+	vperm	24,31,30,10+	lvx	30,8,6+	addi	6,6,0x20+	stvx	24,0,11+	vperm	25,30,31,10+	lvx	31,0,6+	stvx	25,8,11+	addi	11,11,0x20+	bdnz	.Load_ctr32_enc_key++	lvx	26,8,6+	vperm	24,31,30,10+	lvx	27,26,6+	stvx	24,0,11+	vperm	25,26,31,10+	lvx	28,27,6+	stvx	25,8,11+	addi	11,1,64+15+	vperm	26,27,26,10+	lvx	29,28,6+	vperm	27,28,27,10+	lvx	30,29,6+	vperm	28,29,28,10+	lvx	31,30,6+	vperm	29,30,29,10+	lvx	15,31,6+	vperm	30,31,30,10+	lvx	24,0,11+	vperm	31,15,31,10+	lvx	25,8,11++	vadduwm	7,11,11+	subi	3,3,15+	sldi	5,5,4++	vadduwm	16,4,11+	vadduwm	17,4,7+	vxor	15,4,23+	li	10,8+	vadduwm	18,16,7+	vxor	16,16,23+	lvsl	6,0,10+	vadduwm	19,17,7+	vxor	17,17,23+	vspltisb	3,0x0f+	vadduwm	20,18,7+	vxor	18,18,23+	vxor	6,6,3+	vadduwm	21,19,7+	vxor	19,19,23+	vadduwm	22,20,7+	vxor	20,20,23+	vadduwm	4,21,7+	vxor	21,21,23+	vxor	22,22,23++	mtctr	9+	b	.Loop_ctr32_enc8x+.align	5+.Loop_ctr32_enc8x:+	.long	0x11EFC508+	.long	0x1210C508+	.long	0x1231C508+	.long	0x1252C508+	.long	0x1273C508+	.long	0x1294C508+	.long	0x12B5C508+	.long	0x12D6C508+.Loop_ctr32_enc8x_middle:+	lvx	24,26,11+	addi	11,11,0x20++	.long	0x11EFCD08+	.long	0x1210CD08+	.long	0x1231CD08+	.long	0x1252CD08+	.long	0x1273CD08+	.long	0x1294CD08+	.long	0x12B5CD08+	.long	0x12D6CD08+	lvx	25,8,11+	bdnz	.Loop_ctr32_enc8x++	subic	11,5,256+	.long	0x11EFC508+	.long	0x1210C508+	.long	0x1231C508+	.long	0x1252C508+	.long	0x1273C508+	.long	0x1294C508+	.long	0x12B5C508+	.long	0x12D6C508++	subfe	0,0,0+	.long	0x11EFCD08+	.long	0x1210CD08+	.long	0x1231CD08+	.long	0x1252CD08+	.long	0x1273CD08+	.long	0x1294CD08+	.long	0x12B5CD08+	.long	0x12D6CD08++	and	0,0,11+	addi	11,1,64+15+	.long	0x11EFD508+	.long	0x1210D508+	.long	0x1231D508+	.long	0x1252D508+	.long	0x1273D508+	.long	0x1294D508+	.long	0x12B5D508+	.long	0x12D6D508+	lvx	24,0,11++	subic	5,5,129+	.long	0x11EFDD08+	addi	5,5,1+	.long	0x1210DD08+	.long	0x1231DD08+	.long	0x1252DD08+	.long	0x1273DD08+	.long	0x1294DD08+	.long	0x12B5DD08+	.long	0x12D6DD08+	lvx	25,8,11++	.long	0x11EFE508+	.long	0x7C001E99+	.long	0x1210E508+	.long	0x7C281E99+	.long	0x1231E508+	.long	0x7C5A1E99+	.long	0x1252E508+	.long	0x7C7B1E99+	.long	0x1273E508+	.long	0x7D5C1E99+	.long	0x1294E508+	.long	0x7D9D1E99+	.long	0x12B5E508+	.long	0x7DBE1E99+	.long	0x12D6E508+	.long	0x7DDF1E99+	addi	3,3,0x80++	.long	0x11EFED08+	vperm	0,0,0,6+	.long	0x1210ED08+	vperm	1,1,1,6+	.long	0x1231ED08+	vperm	2,2,2,6+	.long	0x1252ED08+	vperm	3,3,3,6+	.long	0x1273ED08+	vperm	10,10,10,6+	.long	0x1294ED08+	vperm	12,12,12,6+	.long	0x12B5ED08+	vperm	13,13,13,6+	.long	0x12D6ED08+	vperm	14,14,14,6++	add	3,3,0++++	subfe.	0,0,0+	.long	0x11EFF508+	vxor	0,0,31+	.long	0x1210F508+	vxor	1,1,31+	.long	0x1231F508+	vxor	2,2,31+	.long	0x1252F508+	vxor	3,3,31+	.long	0x1273F508+	vxor	10,10,31+	.long	0x1294F508+	vxor	12,12,31+	.long	0x12B5F508+	vxor	13,13,31+	.long	0x12D6F508+	vxor	14,14,31++	bne	.Lctr32_enc8x_break++	.long	0x100F0509+	.long	0x10300D09+	vadduwm	16,4,11+	.long	0x10511509+	vadduwm	17,4,7+	vxor	15,4,23+	.long	0x10721D09+	vadduwm	18,16,7+	vxor	16,16,23+	.long	0x11535509+	vadduwm	19,17,7+	vxor	17,17,23+	.long	0x11946509+	vadduwm	20,18,7+	vxor	18,18,23+	.long	0x11B56D09+	vadduwm	21,19,7+	vxor	19,19,23+	.long	0x11D67509+	vadduwm	22,20,7+	vxor	20,20,23+	vperm	0,0,0,6+	vadduwm	4,21,7+	vxor	21,21,23+	vperm	1,1,1,6+	vxor	22,22,23+	mtctr	9++	.long	0x11EFC508+	.long	0x7C002799+	vperm	2,2,2,6+	.long	0x1210C508+	.long	0x7C282799+	vperm	3,3,3,6+	.long	0x1231C508+	.long	0x7C5A2799+	vperm	10,10,10,6+	.long	0x1252C508+	.long	0x7C7B2799+	vperm	12,12,12,6+	.long	0x1273C508+	.long	0x7D5C2799+	vperm	13,13,13,6+	.long	0x1294C508+	.long	0x7D9D2799+	vperm	14,14,14,6+	.long	0x12B5C508+	.long	0x7DBE2799+	.long	0x12D6C508+	.long	0x7DDF2799+	addi	4,4,0x80++	b	.Loop_ctr32_enc8x_middle++.align	5+.Lctr32_enc8x_break:+	cmpwi	5,-0x60+	blt	.Lctr32_enc8x_one+	nop	+	beq	.Lctr32_enc8x_two+	cmpwi	5,-0x40+	blt	.Lctr32_enc8x_three+	nop	+	beq	.Lctr32_enc8x_four+	cmpwi	5,-0x20+	blt	.Lctr32_enc8x_five+	nop	+	beq	.Lctr32_enc8x_six+	cmpwi	5,0x00+	blt	.Lctr32_enc8x_seven++.Lctr32_enc8x_eight:+	.long	0x11EF0509+	.long	0x12100D09+	.long	0x12311509+	.long	0x12521D09+	.long	0x12735509+	.long	0x12946509+	.long	0x12B56D09+	.long	0x12D67509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	vperm	19,19,19,6+	.long	0x7E5B2799+	vperm	20,20,20,6+	.long	0x7E7C2799+	vperm	21,21,21,6+	.long	0x7E9D2799+	vperm	22,22,22,6+	.long	0x7EBE2799+	.long	0x7EDF2799+	addi	4,4,0x80+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_seven:+	.long	0x11EF0D09+	.long	0x12101509+	.long	0x12311D09+	.long	0x12525509+	.long	0x12736509+	.long	0x12946D09+	.long	0x12B57509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	vperm	19,19,19,6+	.long	0x7E5B2799+	vperm	20,20,20,6+	.long	0x7E7C2799+	vperm	21,21,21,6+	.long	0x7E9D2799+	.long	0x7EBE2799+	addi	4,4,0x70+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_six:+	.long	0x11EF1509+	.long	0x12101D09+	.long	0x12315509+	.long	0x12526509+	.long	0x12736D09+	.long	0x12947509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	vperm	19,19,19,6+	.long	0x7E5B2799+	vperm	20,20,20,6+	.long	0x7E7C2799+	.long	0x7E9D2799+	addi	4,4,0x60+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_five:+	.long	0x11EF1D09+	.long	0x12105509+	.long	0x12316509+	.long	0x12526D09+	.long	0x12737509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	vperm	19,19,19,6+	.long	0x7E5B2799+	.long	0x7E7C2799+	addi	4,4,0x50+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_four:+	.long	0x11EF5509+	.long	0x12106509+	.long	0x12316D09+	.long	0x12527509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	vperm	18,18,18,6+	.long	0x7E3A2799+	.long	0x7E5B2799+	addi	4,4,0x40+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_three:+	.long	0x11EF6509+	.long	0x12106D09+	.long	0x12317509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	vperm	17,17,17,6+	.long	0x7E082799+	.long	0x7E3A2799+	addi	4,4,0x30+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_two:+	.long	0x11EF6D09+	.long	0x12107509++	vperm	15,15,15,6+	vperm	16,16,16,6+	.long	0x7DE02799+	.long	0x7E082799+	addi	4,4,0x20+	b	.Lctr32_enc8x_done++.align	5+.Lctr32_enc8x_one:+	.long	0x11EF7509++	vperm	15,15,15,6+	.long	0x7DE02799+	addi	4,4,0x10++.Lctr32_enc8x_done:+	li	10,79+	li	11,95+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32+	stvx	6,10,1+	addi	10,10,32+	stvx	6,11,1+	addi	11,11,32++	or	12,12,12+	lvx	20,10,1+	addi	10,10,32+	lvx	21,11,1+	addi	11,11,32+	lvx	22,10,1+	addi	10,10,32+	lvx	23,11,1+	addi	11,11,32+	lvx	24,10,1+	addi	10,10,32+	lvx	25,11,1+	addi	11,11,32+	lvx	26,10,1+	addi	10,10,32+	lvx	27,11,1+	addi	11,11,32+	lvx	28,10,1+	addi	10,10,32+	lvx	29,11,1+	addi	11,11,32+	lvx	30,10,1+	lvx	31,11,1+	ld	26,400(1)+	ld	27,408(1)+	ld	28,416(1)+	ld	29,424(1)+	ld	30,432(1)+	ld	31,440(1)+	addi	1,1,448+	blr	+.long	0+.byte	0,12,0x04,0,0x80,6,6,0+.long	0+.size	crypton_aes_p8_ctr32_encrypt_blocks,.-crypton_aes_p8_ctr32_encrypt_blocks+.globl	crypton_aes_p8_xts_encrypt+.type	crypton_aes_p8_xts_encrypt,@function+.align	5+crypton_aes_p8_xts_encrypt:+.localentry	crypton_aes_p8_xts_encrypt,0++	mr	10,3+	li	3,-1+	cmpldi	5,16+	.long	0x4dc00020++	lis	0,0xfff0+	li	12,-1+	li	11,0+	or	0,0,0++	vspltisb	9,0x07+	lvsl	6,11,11+	vspltisb	11,0x0f+	vxor	6,6,9++	li	3,15+	lvx	8,0,8+	lvsl	5,0,8+	lvx	4,3,8+	vxor	5,5,11+	vperm	8,8,4,5++	neg	11,10+	lvsr	5,0,11+	lvx	2,0,10+	addi	10,10,15+	vxor	5,5,11++	cmpldi	7,0+	beq	.Lxts_enc_no_key2++	lvsr	7,0,7+	lwz	9,240(7)+	srwi	9,9,1+	subi	9,9,1+	li	3,16++	lvx	0,0,7+	lvx	1,3,7+	addi	3,3,16+	vperm	0,1,0,7+	vxor	8,8,0+	lvx	0,3,7+	addi	3,3,16+	mtctr	9++.Ltweak_xts_enc:+	vperm	1,0,1,7+	.long	0x11080D08+	lvx	1,3,7+	addi	3,3,16+	vperm	0,1,0,7+	.long	0x11080508+	lvx	0,3,7+	addi	3,3,16+	bdnz	.Ltweak_xts_enc++	vperm	1,0,1,7+	.long	0x11080D08+	lvx	1,3,7+	vperm	0,1,0,7+	.long	0x11080509++	li	8,0+	b	.Lxts_enc++.Lxts_enc_no_key2:+	li	3,-16+	and	5,5,3+++.Lxts_enc:+	lvx	4,0,10+	addi	10,10,16++	lvsr	7,0,6+	lwz	9,240(6)+	srwi	9,9,1+	subi	9,9,1+	li	3,16++	vslb	10,9,9+	vor	10,10,9+	vspltisb	11,1+	vsldoi	10,10,11,15++	cmpldi	5,96+	bge	_aesp8_xts_encrypt6x++	andi.	7,5,15+	subic	0,5,32+	subi	7,7,16+	subfe	0,0,0+	and	0,0,7+	add	10,10,0++	lvx	0,0,6+	lvx	1,3,6+	addi	3,3,16+	vperm	2,2,4,5+	vperm	0,1,0,7+	vxor	2,2,8+	vxor	2,2,0+	lvx	0,3,6+	addi	3,3,16+	mtctr	9+	b	.Loop_xts_enc++.align	5+.Loop_xts_enc:+	vperm	1,0,1,7+	.long	0x10420D08+	lvx	1,3,6+	addi	3,3,16+	vperm	0,1,0,7+	.long	0x10420508+	lvx	0,3,6+	addi	3,3,16+	bdnz	.Loop_xts_enc++	vperm	1,0,1,7+	.long	0x10420D08+	lvx	1,3,6+	li	3,16+	vperm	0,1,0,7+	vxor	0,0,8+	.long	0x10620509++	vperm	11,3,3,6++	.long	0x7D602799++	addi	4,4,16++	subic.	5,5,16+	beq	.Lxts_enc_done++	vor	2,4,4+	lvx	4,0,10+	addi	10,10,16+	lvx	0,0,6+	lvx	1,3,6+	addi	3,3,16++	subic	0,5,32+	subfe	0,0,0+	and	0,0,7+	add	10,10,0++	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	8,8,11++	vperm	2,2,4,5+	vperm	0,1,0,7+	vxor	2,2,8+	vxor	3,3,0+	vxor	2,2,0+	lvx	0,3,6+	addi	3,3,16++	mtctr	9+	cmpldi	5,16+	bge	.Loop_xts_enc++	vxor	3,3,8+	lvsr	5,0,5+	vxor	4,4,4+	vspltisb	11,-1+	vperm	4,4,11,5+	vsel	2,2,3,4++	subi	11,4,17+	subi	4,4,16+	mtctr	5+	li	5,16+.Loop_xts_enc_steal:+	lbzu	0,1(11)+	stb	0,16(11)+	bdnz	.Loop_xts_enc_steal++	mtctr	9+	b	.Loop_xts_enc++.Lxts_enc_done:+	cmpldi	8,0+	beq	.Lxts_enc_ret++	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	8,8,11++	vperm	8,8,8,6+	.long	0x7D004799++.Lxts_enc_ret:+	or	12,12,12+	li	3,0+	blr	+.long	0+.byte	0,12,0x04,0,0x80,6,6,0+.long	0+.size	crypton_aes_p8_xts_encrypt,.-crypton_aes_p8_xts_encrypt++.globl	crypton_aes_p8_xts_decrypt+.type	crypton_aes_p8_xts_decrypt,@function+.align	5+crypton_aes_p8_xts_decrypt:+.localentry	crypton_aes_p8_xts_decrypt,0++	mr	10,3+	li	3,-1+	cmpldi	5,16+	.long	0x4dc00020++	lis	0,0xfff8+	li	12,-1+	li	11,0+	or	0,0,0++	andi.	0,5,15+	neg	0,0+	andi.	0,0,16+	sub	5,5,0++	vspltisb	9,0x07+	lvsl	6,11,11+	vspltisb	11,0x0f+	vxor	6,6,9++	li	3,15+	lvx	8,0,8+	lvsl	5,0,8+	lvx	4,3,8+	vxor	5,5,11+	vperm	8,8,4,5++	neg	11,10+	lvsr	5,0,11+	lvx	2,0,10+	addi	10,10,15+	vxor	5,5,11++	cmpldi	7,0+	beq	.Lxts_dec_no_key2++	lvsr	7,0,7+	lwz	9,240(7)+	srwi	9,9,1+	subi	9,9,1+	li	3,16++	lvx	0,0,7+	lvx	1,3,7+	addi	3,3,16+	vperm	0,1,0,7+	vxor	8,8,0+	lvx	0,3,7+	addi	3,3,16+	mtctr	9++.Ltweak_xts_dec:+	vperm	1,0,1,7+	.long	0x11080D08+	lvx	1,3,7+	addi	3,3,16+	vperm	0,1,0,7+	.long	0x11080508+	lvx	0,3,7+	addi	3,3,16+	bdnz	.Ltweak_xts_dec++	vperm	1,0,1,7+	.long	0x11080D08+	lvx	1,3,7+	vperm	0,1,0,7+	.long	0x11080509++	li	8,0+	b	.Lxts_dec++.Lxts_dec_no_key2:+	neg	3,5+	andi.	3,3,15+	add	5,5,3+++.Lxts_dec:+	lvx	4,0,10+	addi	10,10,16++	lvsr	7,0,6+	lwz	9,240(6)+	srwi	9,9,1+	subi	9,9,1+	li	3,16++	vslb	10,9,9+	vor	10,10,9+	vspltisb	11,1+	vsldoi	10,10,11,15++	cmpldi	5,96+	bge	_aesp8_xts_decrypt6x++	lvx	0,0,6+	lvx	1,3,6+	addi	3,3,16+	vperm	2,2,4,5+	vperm	0,1,0,7+	vxor	2,2,8+	vxor	2,2,0+	lvx	0,3,6+	addi	3,3,16+	mtctr	9++	cmpldi	5,16+	blt	.Ltail_xts_dec+++.align	5+.Loop_xts_dec:+	vperm	1,0,1,7+	.long	0x10420D48+	lvx	1,3,6+	addi	3,3,16+	vperm	0,1,0,7+	.long	0x10420548+	lvx	0,3,6+	addi	3,3,16+	bdnz	.Loop_xts_dec++	vperm	1,0,1,7+	.long	0x10420D48+	lvx	1,3,6+	li	3,16+	vperm	0,1,0,7+	vxor	0,0,8+	.long	0x10620549++	vperm	11,3,3,6++	.long	0x7D602799++	addi	4,4,16++	subic.	5,5,16+	beq	.Lxts_dec_done++	vor	2,4,4+	lvx	4,0,10+	addi	10,10,16+	lvx	0,0,6+	lvx	1,3,6+	addi	3,3,16++	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	8,8,11++	vperm	2,2,4,5+	vperm	0,1,0,7+	vxor	2,2,8+	vxor	2,2,0+	lvx	0,3,6+	addi	3,3,16++	mtctr	9+	cmpldi	5,16+	bge	.Loop_xts_dec++.Ltail_xts_dec:+	vsrab	11,8,9+	vaddubm	12,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	12,12,11++	subi	10,10,16+	add	10,10,5++	vxor	2,2,8+	vxor	2,2,12++.Loop_xts_dec_short:+	vperm	1,0,1,7+	.long	0x10420D48+	lvx	1,3,6+	addi	3,3,16+	vperm	0,1,0,7+	.long	0x10420548+	lvx	0,3,6+	addi	3,3,16+	bdnz	.Loop_xts_dec_short++	vperm	1,0,1,7+	.long	0x10420D48+	lvx	1,3,6+	li	3,16+	vperm	0,1,0,7+	vxor	0,0,12+	.long	0x10620549++	vperm	11,3,3,6++	.long	0x7D602799+++	vor	2,4,4+	lvx	4,0,10++	lvx	0,0,6+	lvx	1,3,6+	addi	3,3,16+	vperm	2,2,4,5+	vperm	0,1,0,7++	lvsr	5,0,5+	vxor	4,4,4+	vspltisb	11,-1+	vperm	4,4,11,5+	vsel	2,2,3,4++	vxor	0,0,8+	vxor	2,2,0+	lvx	0,3,6+	addi	3,3,16++	subi	11,4,1+	mtctr	5+	li	5,16+.Loop_xts_dec_steal:+	lbzu	0,1(11)+	stb	0,16(11)+	bdnz	.Loop_xts_dec_steal++	mtctr	9+	b	.Loop_xts_dec++.Lxts_dec_done:+	cmpldi	8,0+	beq	.Lxts_dec_ret++	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	8,8,11++	vperm	8,8,8,6+	.long	0x7D004799++.Lxts_dec_ret:+	or	12,12,12+	li	3,0+	blr	+.long	0+.byte	0,12,0x04,0,0x80,6,6,0+.long	0+.size	crypton_aes_p8_xts_decrypt,.-crypton_aes_p8_xts_decrypt+.align	5+_aesp8_xts_encrypt6x:+	stdu	1,-448(1)+	mflr	11+	li	7,207+	li	3,223+	std	11,464(1)+	stvx	20,7,1+	addi	7,7,32+	stvx	21,3,1+	addi	3,3,32+	stvx	22,7,1+	addi	7,7,32+	stvx	23,3,1+	addi	3,3,32+	stvx	24,7,1+	addi	7,7,32+	stvx	25,3,1+	addi	3,3,32+	stvx	26,7,1+	addi	7,7,32+	stvx	27,3,1+	addi	3,3,32+	stvx	28,7,1+	addi	7,7,32+	stvx	29,3,1+	addi	3,3,32+	stvx	30,7,1+	stvx	31,3,1+	li	0,-1+	stw	12,396(1)+	li	3,0x10+	std	26,400(1)+	li	26,0x20+	std	27,408(1)+	li	27,0x30+	std	28,416(1)+	li	28,0x40+	std	29,424(1)+	li	29,0x50+	std	30,432(1)+	li	30,0x60+	std	31,440(1)+	li	31,0x70+	or	0,0,0++	subi	9,9,3++	lvx	23,0,6+	lvx	30,3,6+	addi	6,6,0x20+	lvx	31,0,6+	vperm	23,30,23,7+	addi	7,1,64+15+	mtctr	9++.Load_xts_enc_key:+	vperm	24,31,30,7+	lvx	30,3,6+	addi	6,6,0x20+	stvx	24,0,7+	vperm	25,30,31,7+	lvx	31,0,6+	stvx	25,3,7+	addi	7,7,0x20+	bdnz	.Load_xts_enc_key++	lvx	26,3,6+	vperm	24,31,30,7+	lvx	27,26,6+	stvx	24,0,7+	vperm	25,26,31,7+	lvx	28,27,6+	stvx	25,3,7+	addi	7,1,64+15+	vperm	26,27,26,7+	lvx	29,28,6+	vperm	27,28,27,7+	lvx	30,29,6+	vperm	28,29,28,7+	lvx	31,30,6+	vperm	29,30,29,7+	lvx	22,31,6+	vperm	30,31,30,7+	lvx	24,0,7+	vperm	31,22,31,7+	lvx	25,3,7++	vperm	0,2,4,5+	subi	10,10,31+	vxor	17,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	7,0,17+	vxor	8,8,11++	.long	0x7C235699+	vxor	18,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	1,1,1,6+	vand	11,11,10+	vxor	12,1,18+	vxor	8,8,11++	.long	0x7C5A5699+	andi.	31,5,15+	vxor	19,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	2,2,2,6+	vand	11,11,10+	vxor	13,2,19+	vxor	8,8,11++	.long	0x7C7B5699+	sub	5,5,31+	vxor	20,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	3,3,3,6+	vand	11,11,10+	vxor	14,3,20+	vxor	8,8,11++	.long	0x7C9C5699+	subi	5,5,0x60+	vxor	21,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	4,4,4,6+	vand	11,11,10+	vxor	15,4,21+	vxor	8,8,11++	.long	0x7CBD5699+	addi	10,10,0x60+	vxor	22,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	5,5,5,6+	vand	11,11,10+	vxor	16,5,22+	vxor	8,8,11++	vxor	31,31,23+	mtctr	9+	b	.Loop_xts_enc6x++.align	5+.Loop_xts_enc6x:+	.long	0x10E7C508+	.long	0x118CC508+	.long	0x11ADC508+	.long	0x11CEC508+	.long	0x11EFC508+	.long	0x1210C508+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD08+	.long	0x118CCD08+	.long	0x11ADCD08+	.long	0x11CECD08+	.long	0x11EFCD08+	.long	0x1210CD08+	lvx	25,3,7+	bdnz	.Loop_xts_enc6x++	subic	5,5,96+	vxor	0,17,31+	.long	0x10E7C508+	.long	0x118CC508+	vsrab	11,8,9+	vxor	17,8,23+	vaddubm	8,8,8+	.long	0x11ADC508+	.long	0x11CEC508+	vsldoi	11,11,11,15+	.long	0x11EFC508+	.long	0x1210C508++	subfe.	0,0,0+	vand	11,11,10+	.long	0x10E7CD08+	.long	0x118CCD08+	vxor	8,8,11+	.long	0x11ADCD08+	.long	0x11CECD08+	vxor	1,18,31+	vsrab	11,8,9+	vxor	18,8,23+	.long	0x11EFCD08+	.long	0x1210CD08++	and	0,0,5+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x10E7D508+	.long	0x118CD508+	vand	11,11,10+	.long	0x11ADD508+	.long	0x11CED508+	vxor	8,8,11+	.long	0x11EFD508+	.long	0x1210D508++	add	10,10,0++++	vxor	2,19,31+	vsrab	11,8,9+	vxor	19,8,23+	vaddubm	8,8,8+	.long	0x10E7DD08+	.long	0x118CDD08+	vsldoi	11,11,11,15+	.long	0x11ADDD08+	.long	0x11CEDD08+	vand	11,11,10+	.long	0x11EFDD08+	.long	0x1210DD08++	addi	7,1,64+15+	vxor	8,8,11+	.long	0x10E7E508+	.long	0x118CE508+	vxor	3,20,31+	vsrab	11,8,9+	vxor	20,8,23+	.long	0x11ADE508+	.long	0x11CEE508+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x11EFE508+	.long	0x1210E508+	lvx	24,0,7+	vand	11,11,10++	.long	0x10E7ED08+	.long	0x118CED08+	vxor	8,8,11+	.long	0x11ADED08+	.long	0x11CEED08+	vxor	4,21,31+	vsrab	11,8,9+	vxor	21,8,23+	.long	0x11EFED08+	.long	0x1210ED08+	lvx	25,3,7+	vaddubm	8,8,8+	vsldoi	11,11,11,15++	.long	0x10E7F508+	.long	0x118CF508+	vand	11,11,10+	.long	0x11ADF508+	.long	0x11CEF508+	vxor	8,8,11+	.long	0x11EFF508+	.long	0x1210F508+	vxor	5,22,31+	vsrab	11,8,9+	vxor	22,8,23++	.long	0x10E70509+	.long	0x7C005699+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x118C0D09+	.long	0x7C235699+	.long	0x11AD1509+	vperm	0,0,0,6+	.long	0x7C5A5699+	vand	11,11,10+	.long	0x11CE1D09+	vperm	1,1,1,6+	.long	0x7C7B5699+	.long	0x11EF2509+	vperm	2,2,2,6+	.long	0x7C9C5699+	vxor	8,8,11+	.long	0x11702D09++	vperm	3,3,3,6+	.long	0x7CBD5699+	addi	10,10,0x60+	vperm	4,4,4,6+	vperm	5,5,5,6++	vperm	7,7,7,6+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,17+	vperm	13,13,13,6+	.long	0x7D832799+	vxor	12,1,18+	vperm	14,14,14,6+	.long	0x7DBA2799+	vxor	13,2,19+	vperm	15,15,15,6+	.long	0x7DDB2799+	vxor	14,3,20+	vperm	16,11,11,6+	.long	0x7DFC2799+	vxor	15,4,21+	.long	0x7E1D2799++	vxor	16,5,22+	addi	4,4,0x60++	mtctr	9+	beq	.Loop_xts_enc6x++	addic.	5,5,0x60+	beq	.Lxts_enc6x_zero+	cmpwi	5,0x20+	blt	.Lxts_enc6x_one+	nop	+	beq	.Lxts_enc6x_two+	cmpwi	5,0x40+	blt	.Lxts_enc6x_three+	nop	+	beq	.Lxts_enc6x_four++.Lxts_enc6x_five:+	vxor	7,1,17+	vxor	12,2,18+	vxor	13,3,19+	vxor	14,4,20+	vxor	15,5,21++	bl	_aesp8_xts_enc5x++	vperm	7,7,7,6+	vor	17,22,22+	vperm	12,12,12,6+	.long	0x7CE02799+	vperm	13,13,13,6+	.long	0x7D832799+	vperm	14,14,14,6+	.long	0x7DBA2799+	vxor	11,15,22+	vperm	15,15,15,6+	.long	0x7DDB2799+	.long	0x7DFC2799+	addi	4,4,0x50+	bne	.Lxts_enc6x_steal+	b	.Lxts_enc6x_done++.align	4+.Lxts_enc6x_four:+	vxor	7,2,17+	vxor	12,3,18+	vxor	13,4,19+	vxor	14,5,20+	vxor	15,15,15++	bl	_aesp8_xts_enc5x++	vperm	7,7,7,6+	vor	17,21,21+	vperm	12,12,12,6+	.long	0x7CE02799+	vperm	13,13,13,6+	.long	0x7D832799+	vxor	11,14,21+	vperm	14,14,14,6+	.long	0x7DBA2799+	.long	0x7DDB2799+	addi	4,4,0x40+	bne	.Lxts_enc6x_steal+	b	.Lxts_enc6x_done++.align	4+.Lxts_enc6x_three:+	vxor	7,3,17+	vxor	12,4,18+	vxor	13,5,19+	vxor	14,14,14+	vxor	15,15,15++	bl	_aesp8_xts_enc5x++	vperm	7,7,7,6+	vor	17,20,20+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	11,13,20+	vperm	13,13,13,6+	.long	0x7D832799+	.long	0x7DBA2799+	addi	4,4,0x30+	bne	.Lxts_enc6x_steal+	b	.Lxts_enc6x_done++.align	4+.Lxts_enc6x_two:+	vxor	7,4,17+	vxor	12,5,18+	vxor	13,13,13+	vxor	14,14,14+	vxor	15,15,15++	bl	_aesp8_xts_enc5x++	vperm	7,7,7,6+	vor	17,19,19+	vxor	11,12,19+	vperm	12,12,12,6+	.long	0x7CE02799+	.long	0x7D832799+	addi	4,4,0x20+	bne	.Lxts_enc6x_steal+	b	.Lxts_enc6x_done++.align	4+.Lxts_enc6x_one:+	vxor	7,5,17+	nop	+.Loop_xts_enc1x:+	.long	0x10E7C508+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD08+	lvx	25,3,7+	bdnz	.Loop_xts_enc1x++	add	10,10,31+	cmpwi	31,0+	.long	0x10E7C508++	subi	10,10,16+	.long	0x10E7CD08++	lvsr	5,0,31+	.long	0x10E7D508++	.long	0x7C005699+	.long	0x10E7DD08++	addi	7,1,64+15+	.long	0x10E7E508+	lvx	24,0,7++	.long	0x10E7ED08+	lvx	25,3,7+	vxor	17,17,31++	vperm	0,0,0,6+	.long	0x10E7F508++	vperm	0,0,0,5+	.long	0x10E78D09++	vor	17,18,18+	vxor	11,7,18+	vperm	7,7,7,6+	.long	0x7CE02799+	addi	4,4,0x10+	bne	.Lxts_enc6x_steal+	b	.Lxts_enc6x_done++.align	4+.Lxts_enc6x_zero:+	cmpwi	31,0+	beq	.Lxts_enc6x_done++	add	10,10,31+	subi	10,10,16+	.long	0x7C005699+	lvsr	5,0,31+	vperm	0,0,0,6+	vperm	0,0,0,5+	vxor	11,11,17+.Lxts_enc6x_steal:+	vxor	0,0,17+	vxor	7,7,7+	vspltisb	12,-1+	vperm	7,7,12,5+	vsel	7,0,11,7++	subi	30,4,17+	subi	4,4,16+	mtctr	31+.Loop_xts_enc6x_steal:+	lbzu	0,1(30)+	stb	0,16(30)+	bdnz	.Loop_xts_enc6x_steal++	li	31,0+	mtctr	9+	b	.Loop_xts_enc1x++.align	4+.Lxts_enc6x_done:+	cmpldi	8,0+	beq	.Lxts_enc6x_ret++	vxor	8,17,23+	vperm	8,8,8,6+	.long	0x7D004799++.Lxts_enc6x_ret:+	mtlr	11+	li	10,79+	li	11,95+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32++	or	12,12,12+	lvx	20,10,1+	addi	10,10,32+	lvx	21,11,1+	addi	11,11,32+	lvx	22,10,1+	addi	10,10,32+	lvx	23,11,1+	addi	11,11,32+	lvx	24,10,1+	addi	10,10,32+	lvx	25,11,1+	addi	11,11,32+	lvx	26,10,1+	addi	10,10,32+	lvx	27,11,1+	addi	11,11,32+	lvx	28,10,1+	addi	10,10,32+	lvx	29,11,1+	addi	11,11,32+	lvx	30,10,1+	lvx	31,11,1+	ld	26,400(1)+	ld	27,408(1)+	ld	28,416(1)+	ld	29,424(1)+	ld	30,432(1)+	ld	31,440(1)+	addi	1,1,448+	blr	+.long	0+.byte	0,12,0x04,1,0x80,6,6,0+.long	0++.align	5+_aesp8_xts_enc5x:+	.long	0x10E7C508+	.long	0x118CC508+	.long	0x11ADC508+	.long	0x11CEC508+	.long	0x11EFC508+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD08+	.long	0x118CCD08+	.long	0x11ADCD08+	.long	0x11CECD08+	.long	0x11EFCD08+	lvx	25,3,7+	bdnz	_aesp8_xts_enc5x++	add	10,10,31+	cmpwi	31,0+	.long	0x10E7C508+	.long	0x118CC508+	.long	0x11ADC508+	.long	0x11CEC508+	.long	0x11EFC508++	subi	10,10,16+	.long	0x10E7CD08+	.long	0x118CCD08+	.long	0x11ADCD08+	.long	0x11CECD08+	.long	0x11EFCD08+	vxor	17,17,31++	.long	0x10E7D508+	lvsr	5,0,31+	.long	0x118CD508+	.long	0x11ADD508+	.long	0x11CED508+	.long	0x11EFD508+	vxor	1,18,31++	.long	0x10E7DD08+	.long	0x7C005699+	.long	0x118CDD08+	.long	0x11ADDD08+	.long	0x11CEDD08+	.long	0x11EFDD08+	vxor	2,19,31++	addi	7,1,64+15+	.long	0x10E7E508+	.long	0x118CE508+	.long	0x11ADE508+	.long	0x11CEE508+	.long	0x11EFE508+	lvx	24,0,7+	vxor	3,20,31++	.long	0x10E7ED08+	vperm	0,0,0,6+	.long	0x118CED08+	.long	0x11ADED08+	.long	0x11CEED08+	.long	0x11EFED08+	lvx	25,3,7+	vxor	4,21,31++	.long	0x10E7F508+	vperm	0,0,0,5+	.long	0x118CF508+	.long	0x11ADF508+	.long	0x11CEF508+	.long	0x11EFF508++	.long	0x10E78D09+	.long	0x118C0D09+	.long	0x11AD1509+	.long	0x11CE1D09+	.long	0x11EF2509+	blr	+.long	0+.byte	0,12,0x14,0,0,0,0,0++.align	5+_aesp8_xts_decrypt6x:+	stdu	1,-448(1)+	mflr	11+	li	7,207+	li	3,223+	std	11,464(1)+	stvx	20,7,1+	addi	7,7,32+	stvx	21,3,1+	addi	3,3,32+	stvx	22,7,1+	addi	7,7,32+	stvx	23,3,1+	addi	3,3,32+	stvx	24,7,1+	addi	7,7,32+	stvx	25,3,1+	addi	3,3,32+	stvx	26,7,1+	addi	7,7,32+	stvx	27,3,1+	addi	3,3,32+	stvx	28,7,1+	addi	7,7,32+	stvx	29,3,1+	addi	3,3,32+	stvx	30,7,1+	stvx	31,3,1+	li	0,-1+	stw	12,396(1)+	li	3,0x10+	std	26,400(1)+	li	26,0x20+	std	27,408(1)+	li	27,0x30+	std	28,416(1)+	li	28,0x40+	std	29,424(1)+	li	29,0x50+	std	30,432(1)+	li	30,0x60+	std	31,440(1)+	li	31,0x70+	or	0,0,0++	subi	9,9,3++	lvx	23,0,6+	lvx	30,3,6+	addi	6,6,0x20+	lvx	31,0,6+	vperm	23,30,23,7+	addi	7,1,64+15+	mtctr	9++.Load_xts_dec_key:+	vperm	24,31,30,7+	lvx	30,3,6+	addi	6,6,0x20+	stvx	24,0,7+	vperm	25,30,31,7+	lvx	31,0,6+	stvx	25,3,7+	addi	7,7,0x20+	bdnz	.Load_xts_dec_key++	lvx	26,3,6+	vperm	24,31,30,7+	lvx	27,26,6+	stvx	24,0,7+	vperm	25,26,31,7+	lvx	28,27,6+	stvx	25,3,7+	addi	7,1,64+15+	vperm	26,27,26,7+	lvx	29,28,6+	vperm	27,28,27,7+	lvx	30,29,6+	vperm	28,29,28,7+	lvx	31,30,6+	vperm	29,30,29,7+	lvx	22,31,6+	vperm	30,31,30,7+	lvx	24,0,7+	vperm	31,22,31,7+	lvx	25,3,7++	vperm	0,2,4,5+	subi	10,10,31+	vxor	17,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vand	11,11,10+	vxor	7,0,17+	vxor	8,8,11++	.long	0x7C235699+	vxor	18,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	1,1,1,6+	vand	11,11,10+	vxor	12,1,18+	vxor	8,8,11++	.long	0x7C5A5699+	andi.	31,5,15+	vxor	19,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	2,2,2,6+	vand	11,11,10+	vxor	13,2,19+	vxor	8,8,11++	.long	0x7C7B5699+	sub	5,5,31+	vxor	20,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	3,3,3,6+	vand	11,11,10+	vxor	14,3,20+	vxor	8,8,11++	.long	0x7C9C5699+	subi	5,5,0x60+	vxor	21,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	4,4,4,6+	vand	11,11,10+	vxor	15,4,21+	vxor	8,8,11++	.long	0x7CBD5699+	addi	10,10,0x60+	vxor	22,8,23+	vsrab	11,8,9+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	vperm	5,5,5,6+	vand	11,11,10+	vxor	16,5,22+	vxor	8,8,11++	vxor	31,31,23+	mtctr	9+	b	.Loop_xts_dec6x++.align	5+.Loop_xts_dec6x:+	.long	0x10E7C548+	.long	0x118CC548+	.long	0x11ADC548+	.long	0x11CEC548+	.long	0x11EFC548+	.long	0x1210C548+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD48+	.long	0x118CCD48+	.long	0x11ADCD48+	.long	0x11CECD48+	.long	0x11EFCD48+	.long	0x1210CD48+	lvx	25,3,7+	bdnz	.Loop_xts_dec6x++	subic	5,5,96+	vxor	0,17,31+	.long	0x10E7C548+	.long	0x118CC548+	vsrab	11,8,9+	vxor	17,8,23+	vaddubm	8,8,8+	.long	0x11ADC548+	.long	0x11CEC548+	vsldoi	11,11,11,15+	.long	0x11EFC548+	.long	0x1210C548++	subfe.	0,0,0+	vand	11,11,10+	.long	0x10E7CD48+	.long	0x118CCD48+	vxor	8,8,11+	.long	0x11ADCD48+	.long	0x11CECD48+	vxor	1,18,31+	vsrab	11,8,9+	vxor	18,8,23+	.long	0x11EFCD48+	.long	0x1210CD48++	and	0,0,5+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x10E7D548+	.long	0x118CD548+	vand	11,11,10+	.long	0x11ADD548+	.long	0x11CED548+	vxor	8,8,11+	.long	0x11EFD548+	.long	0x1210D548++	add	10,10,0++++	vxor	2,19,31+	vsrab	11,8,9+	vxor	19,8,23+	vaddubm	8,8,8+	.long	0x10E7DD48+	.long	0x118CDD48+	vsldoi	11,11,11,15+	.long	0x11ADDD48+	.long	0x11CEDD48+	vand	11,11,10+	.long	0x11EFDD48+	.long	0x1210DD48++	addi	7,1,64+15+	vxor	8,8,11+	.long	0x10E7E548+	.long	0x118CE548+	vxor	3,20,31+	vsrab	11,8,9+	vxor	20,8,23+	.long	0x11ADE548+	.long	0x11CEE548+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x11EFE548+	.long	0x1210E548+	lvx	24,0,7+	vand	11,11,10++	.long	0x10E7ED48+	.long	0x118CED48+	vxor	8,8,11+	.long	0x11ADED48+	.long	0x11CEED48+	vxor	4,21,31+	vsrab	11,8,9+	vxor	21,8,23+	.long	0x11EFED48+	.long	0x1210ED48+	lvx	25,3,7+	vaddubm	8,8,8+	vsldoi	11,11,11,15++	.long	0x10E7F548+	.long	0x118CF548+	vand	11,11,10+	.long	0x11ADF548+	.long	0x11CEF548+	vxor	8,8,11+	.long	0x11EFF548+	.long	0x1210F548+	vxor	5,22,31+	vsrab	11,8,9+	vxor	22,8,23++	.long	0x10E70549+	.long	0x7C005699+	vaddubm	8,8,8+	vsldoi	11,11,11,15+	.long	0x118C0D49+	.long	0x7C235699+	.long	0x11AD1549+	vperm	0,0,0,6+	.long	0x7C5A5699+	vand	11,11,10+	.long	0x11CE1D49+	vperm	1,1,1,6+	.long	0x7C7B5699+	.long	0x11EF2549+	vperm	2,2,2,6+	.long	0x7C9C5699+	vxor	8,8,11+	.long	0x12102D49+	vperm	3,3,3,6+	.long	0x7CBD5699+	addi	10,10,0x60+	vperm	4,4,4,6+	vperm	5,5,5,6++	vperm	7,7,7,6+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,17+	vperm	13,13,13,6+	.long	0x7D832799+	vxor	12,1,18+	vperm	14,14,14,6+	.long	0x7DBA2799+	vxor	13,2,19+	vperm	15,15,15,6+	.long	0x7DDB2799+	vxor	14,3,20+	vperm	16,16,16,6+	.long	0x7DFC2799+	vxor	15,4,21+	.long	0x7E1D2799+	vxor	16,5,22+	addi	4,4,0x60++	mtctr	9+	beq	.Loop_xts_dec6x++	addic.	5,5,0x60+	beq	.Lxts_dec6x_zero+	cmpwi	5,0x20+	blt	.Lxts_dec6x_one+	nop	+	beq	.Lxts_dec6x_two+	cmpwi	5,0x40+	blt	.Lxts_dec6x_three+	nop	+	beq	.Lxts_dec6x_four++.Lxts_dec6x_five:+	vxor	7,1,17+	vxor	12,2,18+	vxor	13,3,19+	vxor	14,4,20+	vxor	15,5,21++	bl	_aesp8_xts_dec5x++	vperm	7,7,7,6+	vor	17,22,22+	vxor	18,8,23+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,18+	vperm	13,13,13,6+	.long	0x7D832799+	vperm	14,14,14,6+	.long	0x7DBA2799+	vperm	15,15,15,6+	.long	0x7DDB2799+	.long	0x7DFC2799+	addi	4,4,0x50+	bne	.Lxts_dec6x_steal+	b	.Lxts_dec6x_done++.align	4+.Lxts_dec6x_four:+	vxor	7,2,17+	vxor	12,3,18+	vxor	13,4,19+	vxor	14,5,20+	vxor	15,15,15++	bl	_aesp8_xts_dec5x++	vperm	7,7,7,6+	vor	17,21,21+	vor	18,22,22+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,22+	vperm	13,13,13,6+	.long	0x7D832799+	vperm	14,14,14,6+	.long	0x7DBA2799+	.long	0x7DDB2799+	addi	4,4,0x40+	bne	.Lxts_dec6x_steal+	b	.Lxts_dec6x_done++.align	4+.Lxts_dec6x_three:+	vxor	7,3,17+	vxor	12,4,18+	vxor	13,5,19+	vxor	14,14,14+	vxor	15,15,15++	bl	_aesp8_xts_dec5x++	vperm	7,7,7,6+	vor	17,20,20+	vor	18,21,21+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,21+	vperm	13,13,13,6+	.long	0x7D832799+	.long	0x7DBA2799+	addi	4,4,0x30+	bne	.Lxts_dec6x_steal+	b	.Lxts_dec6x_done++.align	4+.Lxts_dec6x_two:+	vxor	7,4,17+	vxor	12,5,18+	vxor	13,13,13+	vxor	14,14,14+	vxor	15,15,15++	bl	_aesp8_xts_dec5x++	vperm	7,7,7,6+	vor	17,19,19+	vor	18,20,20+	vperm	12,12,12,6+	.long	0x7CE02799+	vxor	7,0,20+	.long	0x7D832799+	addi	4,4,0x20+	bne	.Lxts_dec6x_steal+	b	.Lxts_dec6x_done++.align	4+.Lxts_dec6x_one:+	vxor	7,5,17+	nop	+.Loop_xts_dec1x:+	.long	0x10E7C548+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD48+	lvx	25,3,7+	bdnz	.Loop_xts_dec1x++	subi	0,31,1+	.long	0x10E7C548++	andi.	0,0,16+	cmpwi	31,0+	.long	0x10E7CD48++	sub	10,10,0+	.long	0x10E7D548++	.long	0x7C005699+	.long	0x10E7DD48++	addi	7,1,64+15+	.long	0x10E7E548+	lvx	24,0,7++	.long	0x10E7ED48+	lvx	25,3,7+	vxor	17,17,31++	vperm	0,0,0,6+	.long	0x10E7F548++	mtctr	9+	.long	0x10E78D49++	vor	17,18,18+	vor	18,19,19+	vperm	7,7,7,6+	.long	0x7CE02799+	addi	4,4,0x10+	vxor	7,0,19+	bne	.Lxts_dec6x_steal+	b	.Lxts_dec6x_done++.align	4+.Lxts_dec6x_zero:+	cmpwi	31,0+	beq	.Lxts_dec6x_done++	.long	0x7C005699+	vperm	0,0,0,6+	vxor	7,0,18+.Lxts_dec6x_steal:+	.long	0x10E7C548+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD48+	lvx	25,3,7+	bdnz	.Lxts_dec6x_steal++	add	10,10,31+	.long	0x10E7C548++	cmpwi	31,0+	.long	0x10E7CD48++	.long	0x7C005699+	.long	0x10E7D548++	lvsr	5,0,31+	.long	0x10E7DD48++	addi	7,1,64+15+	.long	0x10E7E548+	lvx	24,0,7++	.long	0x10E7ED48+	lvx	25,3,7+	vxor	18,18,31++	vperm	0,0,0,6+	.long	0x10E7F548++	vperm	0,0,0,5+	.long	0x11679549++	vperm	7,11,11,6+	.long	0x7CE02799+++	vxor	7,7,7+	vspltisb	12,-1+	vperm	7,7,12,5+	vsel	7,0,11,7+	vxor	7,7,17++	subi	30,4,1+	mtctr	31+.Loop_xts_dec6x_steal:+	lbzu	0,1(30)+	stb	0,16(30)+	bdnz	.Loop_xts_dec6x_steal++	li	31,0+	mtctr	9+	b	.Loop_xts_dec1x++.align	4+.Lxts_dec6x_done:+	cmpldi	8,0+	beq	.Lxts_dec6x_ret++	vxor	8,17,23+	vperm	8,8,8,6+	.long	0x7D004799++.Lxts_dec6x_ret:+	mtlr	11+	li	10,79+	li	11,95+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32+	stvx	9,10,1+	addi	10,10,32+	stvx	9,11,1+	addi	11,11,32++	or	12,12,12+	lvx	20,10,1+	addi	10,10,32+	lvx	21,11,1+	addi	11,11,32+	lvx	22,10,1+	addi	10,10,32+	lvx	23,11,1+	addi	11,11,32+	lvx	24,10,1+	addi	10,10,32+	lvx	25,11,1+	addi	11,11,32+	lvx	26,10,1+	addi	10,10,32+	lvx	27,11,1+	addi	11,11,32+	lvx	28,10,1+	addi	10,10,32+	lvx	29,11,1+	addi	11,11,32+	lvx	30,10,1+	lvx	31,11,1+	ld	26,400(1)+	ld	27,408(1)+	ld	28,416(1)+	ld	29,424(1)+	ld	30,432(1)+	ld	31,440(1)+	addi	1,1,448+	blr	+.long	0+.byte	0,12,0x04,1,0x80,6,6,0+.long	0++.align	5+_aesp8_xts_dec5x:+	.long	0x10E7C548+	.long	0x118CC548+	.long	0x11ADC548+	.long	0x11CEC548+	.long	0x11EFC548+	lvx	24,26,7+	addi	7,7,0x20++	.long	0x10E7CD48+	.long	0x118CCD48+	.long	0x11ADCD48+	.long	0x11CECD48+	.long	0x11EFCD48+	lvx	25,3,7+	bdnz	_aesp8_xts_dec5x++	subi	0,31,1+	.long	0x10E7C548+	.long	0x118CC548+	.long	0x11ADC548+	.long	0x11CEC548+	.long	0x11EFC548++	andi.	0,0,16+	cmpwi	31,0+	.long	0x10E7CD48+	.long	0x118CCD48+	.long	0x11ADCD48+	.long	0x11CECD48+	.long	0x11EFCD48+	vxor	17,17,31++	sub	10,10,0+	.long	0x10E7D548+	.long	0x118CD548+	.long	0x11ADD548+	.long	0x11CED548+	.long	0x11EFD548+	vxor	1,18,31++	.long	0x10E7DD48+	.long	0x7C005699+	.long	0x118CDD48+	.long	0x11ADDD48+	.long	0x11CEDD48+	.long	0x11EFDD48+	vxor	2,19,31++	addi	7,1,64+15+	.long	0x10E7E548+	.long	0x118CE548+	.long	0x11ADE548+	.long	0x11CEE548+	.long	0x11EFE548+	lvx	24,0,7+	vxor	3,20,31++	.long	0x10E7ED48+	vperm	0,0,0,6+	.long	0x118CED48+	.long	0x11ADED48+	.long	0x11CEED48+	.long	0x11EFED48+	lvx	25,3,7+	vxor	4,21,31++	.long	0x10E7F548+	.long	0x118CF548+	.long	0x11ADF548+	.long	0x11CEF548+	.long	0x11EFF548++	.long	0x10E78D49+	.long	0x118C0D49+	.long	0x11AD1549+	.long	0x11CE1D49+	.long	0x11EF2549+	mtctr	9+	blr	+.long	0+.byte	0,12,0x14,0,0,0,0,0

@@ -0,0 +1,3801 @@+#!/usr/bin/env perl+#+# ====================================================================+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL+# project. The module is, however, dual licensed under OpenSSL and+# CRYPTOGAMS licenses depending on where you obtain it. For further+# details see http://www.openssl.org/~appro/cryptogams/.+# ====================================================================+#+# This module implements support for AES instructions as per PowerISA+# specification version 2.07, first implemented by POWER8 processor.+# The module is endian-agnostic in sense that it supports both big-+# and little-endian cases. Data alignment in parallelizable modes is+# handled with VSX loads and stores, which implies MSR.VSX flag being+# set. It should also be noted that ISA specification doesn't prohibit+# alignment exceptions for these instructions on page boundaries.+# Initially alignment was handled in pure AltiVec/VMX way [when data+# is aligned programmatically, which in turn guarantees exception-+# free execution], but it turned to hamper performance when vcipher+# instructions are interleaved. It's reckoned that eventual+# misalignment penalties at page boundaries are in average lower+# than additional overhead in pure AltiVec approach.+#+# May 2016+#+# Add XTS subroutine, 9x on little- and 12x improvement on big-endian+# systems were measured.+#+######################################################################+# Current large-block performance in cycles per byte processed with+# 128-bit key (less is better).+#+#		CBC en-/decrypt	CTR	XTS+# POWER8[le]	3.96/0.72	0.74	1.1+# POWER8[be]	3.75/0.65	0.66	1.0+# POWER9[le]	4.02/0.86	0.84	1.05+# POWER9[be]	3.99/0.78	0.79	0.97+++$flavour = shift;++if ($flavour =~ /64/) {+	$SIZE_T	=8;+	$LRSAVE	=2*$SIZE_T;+	$STU	="stdu";+	$POP	="ld";+	$PUSH	="std";+	$UCMP	="cmpld";+	$SHL	="sldi";+} elsif ($flavour =~ /32/) {+	$SIZE_T	=4;+	$LRSAVE	=$SIZE_T;+	$STU	="stwu";+	$POP	="lwz";+	$PUSH	="stw";+	$UCMP	="cmplw";+	$SHL	="slwi";+} else { die "nonsense $flavour"; }++$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;++$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or+die "can't locate ppc-xlate.pl";++open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";++$FRAME=8*$SIZE_T;+$prefix="aes_p8";++$sp="r1";+$vrsave="r12";++#########################################################################+{{{	# Key setup procedures						#+my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));+my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));+my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));++$code.=<<___;+.machine	"any"++.text++.align	7+rcon:+.long	0x01000000, 0x01000000, 0x01000000, 0x01000000	?rev+.long	0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000	?rev+.long	0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c	?rev+.long	0,0,0,0						?asis+Lconsts:+	mflr	r0+	bcl	20,31,\$+4+	mflr	$ptr	 #vvvvv "distance between . and rcon+	addi	$ptr,$ptr,-0x48+	mtlr	r0+	blr+	.long	0+	.byte	0,12,0x14,0,0,0,0,0+.asciz	"AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"++.globl	.${prefix}_set_encrypt_key+.align	5+.${prefix}_set_encrypt_key:+Lset_encrypt_key:+	mflr		r11+	$PUSH		r11,$LRSAVE($sp)++	li		$ptr,-1+	${UCMP}i	$inp,0+	beq-		Lenc_key_abort		# if ($inp==0) return -1;+	${UCMP}i	$out,0+	beq-		Lenc_key_abort		# if ($out==0) return -1;+	li		$ptr,-2+	cmpwi		$bits,128+	blt-		Lenc_key_abort+	cmpwi		$bits,256+	bgt-		Lenc_key_abort+	andi.		r0,$bits,0x3f+	bne-		Lenc_key_abort++	lis		r0,0xfff0+	mfspr		$vrsave,256+	mtspr		256,r0++	bl		Lconsts+	mtlr		r11++	neg		r9,$inp+	lvx		$in0,0,$inp+	addi		$inp,$inp,15		# 15 is not typo+	lvsr		$key,0,r9		# borrow $key+	li		r8,0x20+	cmpwi		$bits,192+	lvx		$in1,0,$inp+	le?vspltisb	$mask,0x0f		# borrow $mask+	lvx		$rcon,0,$ptr+	le?vxor		$key,$key,$mask		# adjust for byte swap+	lvx		$mask,r8,$ptr+	addi		$ptr,$ptr,0x10+	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]+	li		$cnt,8+	vxor		$zero,$zero,$zero+	mtctr		$cnt++	?lvsr		$outperm,0,$out+	vspltisb	$outmask,-1+	lvx		$outhead,0,$out+	?vperm		$outmask,$zero,$outmask,$outperm++	blt		Loop128+	addi		$inp,$inp,8+	beq		L192+	addi		$inp,$inp,8+	b		L256++.align	4+Loop128:+	vperm		$key,$in0,$in0,$mask	# rotate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vcipherlast	$key,$key,$rcon+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	 vadduwm	$rcon,$rcon,$rcon+	vxor		$in0,$in0,$key+	bdnz		Loop128++	lvx		$rcon,0,$ptr		# last two round keys++	vperm		$key,$in0,$in0,$mask	# rotate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vcipherlast	$key,$key,$rcon+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	 vadduwm	$rcon,$rcon,$rcon+	vxor		$in0,$in0,$key++	vperm		$key,$in0,$in0,$mask	# rotate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vcipherlast	$key,$key,$rcon+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vxor		$in0,$in0,$key+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	 stvx		$stage,0,$out++	addi		$inp,$out,15		# 15 is not typo+	addi		$out,$out,0x50++	li		$rounds,10+	b		Ldone++.align	4+L192:+	lvx		$tmp,0,$inp+	li		$cnt,4+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	 stvx		$stage,0,$out+	 addi		$out,$out,16+	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]+	vspltisb	$key,8			# borrow $key+	mtctr		$cnt+	vsububm		$mask,$mask,$key	# adjust the mask++Loop192:+	vperm		$key,$in1,$in1,$mask	# roate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	vcipherlast	$key,$key,$rcon++	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp++	 vsldoi		$stage,$zero,$in1,8+	vspltw		$tmp,$in0,3+	vxor		$tmp,$tmp,$in1+	vsldoi		$in1,$zero,$in1,12	# >>32+	 vadduwm	$rcon,$rcon,$rcon+	vxor		$in1,$in1,$tmp+	vxor		$in0,$in0,$key+	vxor		$in1,$in1,$key+	 vsldoi		$stage,$stage,$in0,8++	vperm		$key,$in1,$in1,$mask	# rotate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	 vperm		$outtail,$stage,$stage,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vcipherlast	$key,$key,$rcon+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	 vsldoi		$stage,$in0,$in1,8+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	 vperm		$outtail,$stage,$stage,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	vspltw		$tmp,$in0,3+	vxor		$tmp,$tmp,$in1+	vsldoi		$in1,$zero,$in1,12	# >>32+	 vadduwm	$rcon,$rcon,$rcon+	vxor		$in1,$in1,$tmp+	vxor		$in0,$in0,$key+	vxor		$in1,$in1,$key+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	 stvx		$stage,0,$out+	 addi		$inp,$out,15		# 15 is not typo+	 addi		$out,$out,16+	bdnz		Loop192++	li		$rounds,12+	addi		$out,$out,0x20+	b		Ldone++.align	4+L256:+	lvx		$tmp,0,$inp+	li		$cnt,7+	li		$rounds,14+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	 stvx		$stage,0,$out+	 addi		$out,$out,16+	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]+	mtctr		$cnt++Loop256:+	vperm		$key,$in1,$in1,$mask	# rotate-n-splat+	vsldoi		$tmp,$zero,$in0,12	# >>32+	 vperm		$outtail,$in1,$in1,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	vcipherlast	$key,$key,$rcon+	 stvx		$stage,0,$out+	 addi		$out,$out,16++	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in0,$in0,$tmp+	 vadduwm	$rcon,$rcon,$rcon+	vxor		$in0,$in0,$key+	 vperm		$outtail,$in0,$in0,$outperm	# rotate+	 vsel		$stage,$outhead,$outtail,$outmask+	 vmr		$outhead,$outtail+	 stvx		$stage,0,$out+	 addi		$inp,$out,15		# 15 is not typo+	 addi		$out,$out,16+	bdz		Ldone++	vspltw		$key,$in0,3		# just splat+	vsldoi		$tmp,$zero,$in1,12	# >>32+	vsbox		$key,$key++	vxor		$in1,$in1,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in1,$in1,$tmp+	vsldoi		$tmp,$zero,$tmp,12	# >>32+	vxor		$in1,$in1,$tmp++	vxor		$in1,$in1,$key+	b		Loop256++.align	4+Ldone:+	lvx		$in1,0,$inp		# redundant in aligned case+	vsel		$in1,$outhead,$in1,$outmask+	stvx		$in1,0,$inp+	li		$ptr,0+	mtspr		256,$vrsave+	stw		$rounds,0($out)++Lenc_key_abort:+	mr		r3,$ptr+	blr+	.long		0+	.byte		0,12,0x14,1,0,0,3,0+	.long		0+.size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key++.globl	.${prefix}_set_decrypt_key+.align	5+.${prefix}_set_decrypt_key:+	$STU		$sp,-$FRAME($sp)+	mflr		r10+	$PUSH		r10,$FRAME+$LRSAVE($sp)+	bl		Lset_encrypt_key+	mtlr		r10++	cmpwi		r3,0+	bne-		Ldec_key_abort++	slwi		$cnt,$rounds,4+	subi		$inp,$out,240		# first round key+	srwi		$rounds,$rounds,1+	add		$out,$inp,$cnt		# last round key+	mtctr		$rounds++Ldeckey:+	lwz		r0, 0($inp)+	lwz		r6, 4($inp)+	lwz		r7, 8($inp)+	lwz		r8, 12($inp)+	addi		$inp,$inp,16+	lwz		r9, 0($out)+	lwz		r10,4($out)+	lwz		r11,8($out)+	lwz		r12,12($out)+	stw		r0, 0($out)+	stw		r6, 4($out)+	stw		r7, 8($out)+	stw		r8, 12($out)+	subi		$out,$out,16+	stw		r9, -16($inp)+	stw		r10,-12($inp)+	stw		r11,-8($inp)+	stw		r12,-4($inp)+	bdnz		Ldeckey++	xor		r3,r3,r3		# return value+Ldec_key_abort:+	addi		$sp,$sp,$FRAME+	blr+	.long		0+	.byte		0,12,4,1,0x80,0,3,0+	.long		0+.size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key+___+}}}+#########################################################################+{{{	# Single block en- and decrypt procedures			#+sub gen_block () {+my $dir = shift;+my $n   = $dir eq "de" ? "n" : "";+my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));++$code.=<<___;+.globl	.${prefix}_${dir}crypt+.align	5+.${prefix}_${dir}crypt:+	lwz		$rounds,240($key)+	lis		r0,0xfc00+	mfspr		$vrsave,256+	li		$idx,15			# 15 is not typo+	mtspr		256,r0++	lvx		v0,0,$inp+	neg		r11,$out+	lvx		v1,$idx,$inp+	lvsl		v2,0,$inp		# inpperm+	le?vspltisb	v4,0x0f+	?lvsl		v3,0,r11		# outperm+	le?vxor		v2,v2,v4+	li		$idx,16+	vperm		v0,v0,v1,v2		# align [and byte swap in LE]+	lvx		v1,0,$key+	?lvsl		v5,0,$key		# keyperm+	srwi		$rounds,$rounds,1+	lvx		v2,$idx,$key+	addi		$idx,$idx,16+	subi		$rounds,$rounds,1+	?vperm		v1,v1,v2,v5		# align round key++	vxor		v0,v0,v1+	lvx		v1,$idx,$key+	addi		$idx,$idx,16+	mtctr		$rounds++Loop_${dir}c:+	?vperm		v2,v2,v1,v5+	v${n}cipher	v0,v0,v2+	lvx		v2,$idx,$key+	addi		$idx,$idx,16+	?vperm		v1,v1,v2,v5+	v${n}cipher	v0,v0,v1+	lvx		v1,$idx,$key+	addi		$idx,$idx,16+	bdnz		Loop_${dir}c++	?vperm		v2,v2,v1,v5+	v${n}cipher	v0,v0,v2+	lvx		v2,$idx,$key+	?vperm		v1,v1,v2,v5+	v${n}cipherlast	v0,v0,v1++	vspltisb	v2,-1+	vxor		v1,v1,v1+	li		$idx,15			# 15 is not typo+	?vperm		v2,v1,v2,v3		# outmask+	le?vxor		v3,v3,v4+	lvx		v1,0,$out		# outhead+	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]+	vsel		v1,v1,v0,v2+	lvx		v4,$idx,$out+	stvx		v1,0,$out+	vsel		v0,v0,v4,v2+	stvx		v0,$idx,$out++	mtspr		256,$vrsave+	blr+	.long		0+	.byte		0,12,0x14,0,0,0,3,0+	.long		0+.size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt+___+}+&gen_block("en");+&gen_block("de");+}}}+#########################################################################+{{{	# CBC en- and decrypt procedures				#+my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));+my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));+my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=+						map("v$_",(4..10));+$code.=<<___;+.globl	.${prefix}_cbc_encrypt+.align	5+.${prefix}_cbc_encrypt:+	${UCMP}i	$len,16+	bltlr-++	cmpwi		$enc,0			# test direction+	lis		r0,0xffe0+	mfspr		$vrsave,256+	mtspr		256,r0++	li		$idx,15+	vxor		$rndkey0,$rndkey0,$rndkey0+	le?vspltisb	$tmp,0x0f++	lvx		$ivec,0,$ivp		# load [unaligned] iv+	lvsl		$inpperm,0,$ivp+	lvx		$inptail,$idx,$ivp+	le?vxor		$inpperm,$inpperm,$tmp+	vperm		$ivec,$ivec,$inptail,$inpperm++	neg		r11,$inp+	?lvsl		$keyperm,0,$key		# prepare for unaligned key+	lwz		$rounds,240($key)++	lvsr		$inpperm,0,r11		# prepare for unaligned load+	lvx		$inptail,0,$inp+	addi		$inp,$inp,15		# 15 is not typo+	le?vxor		$inpperm,$inpperm,$tmp++	?lvsr		$outperm,0,$out		# prepare for unaligned store+	vspltisb	$outmask,-1+	lvx		$outhead,0,$out+	?vperm		$outmask,$rndkey0,$outmask,$outperm+	le?vxor		$outperm,$outperm,$tmp++	srwi		$rounds,$rounds,1+	li		$idx,16+	subi		$rounds,$rounds,1+	beq		Lcbc_dec++Lcbc_enc:+	vmr		$inout,$inptail+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16+	mtctr		$rounds+	subi		$len,$len,16		# len-=16++	lvx		$rndkey0,0,$key+	 vperm		$inout,$inout,$inptail,$inpperm+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16+	vxor		$inout,$inout,$ivec++Loop_cbc_enc:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipher		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16+	bdnz		Loop_cbc_enc++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	li		$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipherlast	$ivec,$inout,$rndkey0+	${UCMP}i	$len,16++	vperm		$tmp,$ivec,$ivec,$outperm+	vsel		$inout,$outhead,$tmp,$outmask+	vmr		$outhead,$tmp+	stvx		$inout,0,$out+	addi		$out,$out,16+	bge		Lcbc_enc++	b		Lcbc_done++.align	4+Lcbc_dec:+	${UCMP}i	$len,128+	bge		_aesp8_cbc_decrypt8x+	vmr		$tmp,$inptail+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16+	mtctr		$rounds+	subi		$len,$len,16		# len-=16++	lvx		$rndkey0,0,$key+	 vperm		$tmp,$tmp,$inptail,$inpperm+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$tmp,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16++Loop_cbc_dec:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vncipher	$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16+	bdnz		Loop_cbc_dec++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	li		$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vncipherlast	$inout,$inout,$rndkey0+	${UCMP}i	$len,16++	vxor		$inout,$inout,$ivec+	vmr		$ivec,$tmp+	vperm		$tmp,$inout,$inout,$outperm+	vsel		$inout,$outhead,$tmp,$outmask+	vmr		$outhead,$tmp+	stvx		$inout,0,$out+	addi		$out,$out,16+	bge		Lcbc_dec++Lcbc_done:+	addi		$out,$out,-1+	lvx		$inout,0,$out		# redundant in aligned case+	vsel		$inout,$outhead,$inout,$outmask+	stvx		$inout,0,$out++	neg		$enc,$ivp		# write [unaligned] iv+	li		$idx,15			# 15 is not typo+	vxor		$rndkey0,$rndkey0,$rndkey0+	vspltisb	$outmask,-1+	le?vspltisb	$tmp,0x0f+	?lvsl		$outperm,0,$enc+	?vperm		$outmask,$rndkey0,$outmask,$outperm+	le?vxor		$outperm,$outperm,$tmp+	lvx		$outhead,0,$ivp+	vperm		$ivec,$ivec,$ivec,$outperm+	vsel		$inout,$outhead,$ivec,$outmask+	lvx		$inptail,$idx,$ivp+	stvx		$inout,0,$ivp+	vsel		$inout,$ivec,$inptail,$outmask+	stvx		$inout,$idx,$ivp++	mtspr		256,$vrsave+	blr+	.long		0+	.byte		0,12,0x14,0,0,0,6,0+	.long		0+___+#########################################################################+{{	# Optimized CBC decrypt procedure				#+my $key_="r11";+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));+    $x00=0 if ($flavour =~ /osx/);+my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));+my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys+			# v26-v31 last 6 round keys+my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment++$code.=<<___;+.align	5+_aesp8_cbc_decrypt8x:+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)+	li		r10,`$FRAME+8*16+15`+	li		r11,`$FRAME+8*16+31`+	stvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	stvx		v21,r11,$sp+	addi		r11,r11,32+	stvx		v22,r10,$sp+	addi		r10,r10,32+	stvx		v23,r11,$sp+	addi		r11,r11,32+	stvx		v24,r10,$sp+	addi		r10,r10,32+	stvx		v25,r11,$sp+	addi		r11,r11,32+	stvx		v26,r10,$sp+	addi		r10,r10,32+	stvx		v27,r11,$sp+	addi		r11,r11,32+	stvx		v28,r10,$sp+	addi		r10,r10,32+	stvx		v29,r11,$sp+	addi		r11,r11,32+	stvx		v30,r10,$sp+	stvx		v31,r11,$sp+	li		r0,-1+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave+	li		$x10,0x10+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	li		$x20,0x20+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	li		$x30,0x30+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	li		$x40,0x40+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	li		$x50,0x50+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	li		$x60,0x60+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	li		$x70,0x70+	mtspr		256,r0++	subi		$rounds,$rounds,3	# -4 in total+	subi		$len,$len,128		# bias++	lvx		$rndkey0,$x00,$key	# load key schedule+	lvx		v30,$x10,$key+	addi		$key,$key,0x20+	lvx		v31,$x00,$key+	?vperm		$rndkey0,$rndkey0,v30,$keyperm+	addi		$key_,$sp,$FRAME+15+	mtctr		$rounds++Load_cbc_dec_key:+	?vperm		v24,v30,v31,$keyperm+	lvx		v30,$x10,$key+	addi		$key,$key,0x20+	stvx		v24,$x00,$key_		# off-load round[1]+	?vperm		v25,v31,v30,$keyperm+	lvx		v31,$x00,$key+	stvx		v25,$x10,$key_		# off-load round[2]+	addi		$key_,$key_,0x20+	bdnz		Load_cbc_dec_key++	lvx		v26,$x10,$key+	?vperm		v24,v30,v31,$keyperm+	lvx		v27,$x20,$key+	stvx		v24,$x00,$key_		# off-load round[3]+	?vperm		v25,v31,v26,$keyperm+	lvx		v28,$x30,$key+	stvx		v25,$x10,$key_		# off-load round[4]+	addi		$key_,$sp,$FRAME+15	# rewind $key_+	?vperm		v26,v26,v27,$keyperm+	lvx		v29,$x40,$key+	?vperm		v27,v27,v28,$keyperm+	lvx		v30,$x50,$key+	?vperm		v28,v28,v29,$keyperm+	lvx		v31,$x60,$key+	?vperm		v29,v29,v30,$keyperm+	lvx		$out0,$x70,$key		# borrow $out0+	?vperm		v30,v30,v31,$keyperm+	lvx		v24,$x00,$key_		# pre-load round[1]+	?vperm		v31,v31,$out0,$keyperm+	lvx		v25,$x10,$key_		# pre-load round[2]++	#lvx		$inptail,0,$inp		# "caller" already did this+	#addi		$inp,$inp,15		# 15 is not typo+	subi		$inp,$inp,15		# undo "caller"++	 le?li		$idx,8+	lvx_u		$in0,$x00,$inp		# load first 8 "words"+	 le?lvsl	$inpperm,0,$idx+	 le?vspltisb	$tmp,0x0f+	lvx_u		$in1,$x10,$inp+	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u+	lvx_u		$in2,$x20,$inp+	 le?vperm	$in0,$in0,$in0,$inpperm+	lvx_u		$in3,$x30,$inp+	 le?vperm	$in1,$in1,$in1,$inpperm+	lvx_u		$in4,$x40,$inp+	 le?vperm	$in2,$in2,$in2,$inpperm+	vxor		$out0,$in0,$rndkey0+	lvx_u		$in5,$x50,$inp+	 le?vperm	$in3,$in3,$in3,$inpperm+	vxor		$out1,$in1,$rndkey0+	lvx_u		$in6,$x60,$inp+	 le?vperm	$in4,$in4,$in4,$inpperm+	vxor		$out2,$in2,$rndkey0+	lvx_u		$in7,$x70,$inp+	addi		$inp,$inp,0x80+	 le?vperm	$in5,$in5,$in5,$inpperm+	vxor		$out3,$in3,$rndkey0+	 le?vperm	$in6,$in6,$in6,$inpperm+	vxor		$out4,$in4,$rndkey0+	 le?vperm	$in7,$in7,$in7,$inpperm+	vxor		$out5,$in5,$rndkey0+	vxor		$out6,$in6,$rndkey0+	vxor		$out7,$in7,$rndkey0++	mtctr		$rounds+	b		Loop_cbc_dec8x+.align	5+Loop_cbc_dec8x:+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24+	vncipher	$out6,$out6,v24+	vncipher	$out7,$out7,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25+	vncipher	$out6,$out6,v25+	vncipher	$out7,$out7,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_cbc_dec8x++	subic		$len,$len,128		# $len-=128+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24+	vncipher	$out6,$out6,v24+	vncipher	$out7,$out7,v24++	subfe.		r0,r0,r0		# borrow?-1:0+	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25+	vncipher	$out6,$out6,v25+	vncipher	$out7,$out7,v25++	and		r0,r0,$len+	vncipher	$out0,$out0,v26+	vncipher	$out1,$out1,v26+	vncipher	$out2,$out2,v26+	vncipher	$out3,$out3,v26+	vncipher	$out4,$out4,v26+	vncipher	$out5,$out5,v26+	vncipher	$out6,$out6,v26+	vncipher	$out7,$out7,v26++	add		$inp,$inp,r0		# $inp is adjusted in such+						# way that at exit from the+						# loop inX-in7 are loaded+						# with last "words"+	vncipher	$out0,$out0,v27+	vncipher	$out1,$out1,v27+	vncipher	$out2,$out2,v27+	vncipher	$out3,$out3,v27+	vncipher	$out4,$out4,v27+	vncipher	$out5,$out5,v27+	vncipher	$out6,$out6,v27+	vncipher	$out7,$out7,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vncipher	$out0,$out0,v28+	vncipher	$out1,$out1,v28+	vncipher	$out2,$out2,v28+	vncipher	$out3,$out3,v28+	vncipher	$out4,$out4,v28+	vncipher	$out5,$out5,v28+	vncipher	$out6,$out6,v28+	vncipher	$out7,$out7,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]++	vncipher	$out0,$out0,v29+	vncipher	$out1,$out1,v29+	vncipher	$out2,$out2,v29+	vncipher	$out3,$out3,v29+	vncipher	$out4,$out4,v29+	vncipher	$out5,$out5,v29+	vncipher	$out6,$out6,v29+	vncipher	$out7,$out7,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]++	vncipher	$out0,$out0,v30+	 vxor		$ivec,$ivec,v31		# xor with last round key+	vncipher	$out1,$out1,v30+	 vxor		$in0,$in0,v31+	vncipher	$out2,$out2,v30+	 vxor		$in1,$in1,v31+	vncipher	$out3,$out3,v30+	 vxor		$in2,$in2,v31+	vncipher	$out4,$out4,v30+	 vxor		$in3,$in3,v31+	vncipher	$out5,$out5,v30+	 vxor		$in4,$in4,v31+	vncipher	$out6,$out6,v30+	 vxor		$in5,$in5,v31+	vncipher	$out7,$out7,v30+	 vxor		$in6,$in6,v31++	vncipherlast	$out0,$out0,$ivec+	vncipherlast	$out1,$out1,$in0+	 lvx_u		$in0,$x00,$inp		# load next input block+	vncipherlast	$out2,$out2,$in1+	 lvx_u		$in1,$x10,$inp+	vncipherlast	$out3,$out3,$in2+	 le?vperm	$in0,$in0,$in0,$inpperm+	 lvx_u		$in2,$x20,$inp+	vncipherlast	$out4,$out4,$in3+	 le?vperm	$in1,$in1,$in1,$inpperm+	 lvx_u		$in3,$x30,$inp+	vncipherlast	$out5,$out5,$in4+	 le?vperm	$in2,$in2,$in2,$inpperm+	 lvx_u		$in4,$x40,$inp+	vncipherlast	$out6,$out6,$in5+	 le?vperm	$in3,$in3,$in3,$inpperm+	 lvx_u		$in5,$x50,$inp+	vncipherlast	$out7,$out7,$in6+	 le?vperm	$in4,$in4,$in4,$inpperm+	 lvx_u		$in6,$x60,$inp+	vmr		$ivec,$in7+	 le?vperm	$in5,$in5,$in5,$inpperm+	 lvx_u		$in7,$x70,$inp+	 addi		$inp,$inp,0x80++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	 le?vperm	$in6,$in6,$in6,$inpperm+	 vxor		$out0,$in0,$rndkey0+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	 le?vperm	$in7,$in7,$in7,$inpperm+	 vxor		$out1,$in1,$rndkey0+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	 vxor		$out2,$in2,$rndkey0+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x30,$out+	 vxor		$out3,$in3,$rndkey0+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x40,$out+	 vxor		$out4,$in4,$rndkey0+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x50,$out+	 vxor		$out5,$in5,$rndkey0+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x60,$out+	 vxor		$out6,$in6,$rndkey0+	stvx_u		$out7,$x70,$out+	addi		$out,$out,0x80+	 vxor		$out7,$in7,$rndkey0++	mtctr		$rounds+	beq		Loop_cbc_dec8x		# did $len-=128 borrow?++	addic.		$len,$len,128+	beq		Lcbc_dec8x_done+	nop+	nop++Loop_cbc_dec8x_tail:				# up to 7 "words" tail...+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24+	vncipher	$out6,$out6,v24+	vncipher	$out7,$out7,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25+	vncipher	$out6,$out6,v25+	vncipher	$out7,$out7,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_cbc_dec8x_tail++	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24+	vncipher	$out6,$out6,v24+	vncipher	$out7,$out7,v24++	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25+	vncipher	$out6,$out6,v25+	vncipher	$out7,$out7,v25++	vncipher	$out1,$out1,v26+	vncipher	$out2,$out2,v26+	vncipher	$out3,$out3,v26+	vncipher	$out4,$out4,v26+	vncipher	$out5,$out5,v26+	vncipher	$out6,$out6,v26+	vncipher	$out7,$out7,v26++	vncipher	$out1,$out1,v27+	vncipher	$out2,$out2,v27+	vncipher	$out3,$out3,v27+	vncipher	$out4,$out4,v27+	vncipher	$out5,$out5,v27+	vncipher	$out6,$out6,v27+	vncipher	$out7,$out7,v27++	vncipher	$out1,$out1,v28+	vncipher	$out2,$out2,v28+	vncipher	$out3,$out3,v28+	vncipher	$out4,$out4,v28+	vncipher	$out5,$out5,v28+	vncipher	$out6,$out6,v28+	vncipher	$out7,$out7,v28++	vncipher	$out1,$out1,v29+	vncipher	$out2,$out2,v29+	vncipher	$out3,$out3,v29+	vncipher	$out4,$out4,v29+	vncipher	$out5,$out5,v29+	vncipher	$out6,$out6,v29+	vncipher	$out7,$out7,v29++	vncipher	$out1,$out1,v30+	 vxor		$ivec,$ivec,v31		# last round key+	vncipher	$out2,$out2,v30+	 vxor		$in1,$in1,v31+	vncipher	$out3,$out3,v30+	 vxor		$in2,$in2,v31+	vncipher	$out4,$out4,v30+	 vxor		$in3,$in3,v31+	vncipher	$out5,$out5,v30+	 vxor		$in4,$in4,v31+	vncipher	$out6,$out6,v30+	 vxor		$in5,$in5,v31+	vncipher	$out7,$out7,v30+	 vxor		$in6,$in6,v31++	cmplwi		$len,32			# switch($len)+	blt		Lcbc_dec8x_one+	nop+	beq		Lcbc_dec8x_two+	cmplwi		$len,64+	blt		Lcbc_dec8x_three+	nop+	beq		Lcbc_dec8x_four+	cmplwi		$len,96+	blt		Lcbc_dec8x_five+	nop+	beq		Lcbc_dec8x_six++Lcbc_dec8x_seven:+	vncipherlast	$out1,$out1,$ivec+	vncipherlast	$out2,$out2,$in1+	vncipherlast	$out3,$out3,$in2+	vncipherlast	$out4,$out4,$in3+	vncipherlast	$out5,$out5,$in4+	vncipherlast	$out6,$out6,$in5+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out1,$out1,$out1,$inpperm+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x00,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x10,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x20,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x30,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x40,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x50,$out+	stvx_u		$out7,$x60,$out+	addi		$out,$out,0x70+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_six:+	vncipherlast	$out2,$out2,$ivec+	vncipherlast	$out3,$out3,$in2+	vncipherlast	$out4,$out4,$in3+	vncipherlast	$out5,$out5,$in4+	vncipherlast	$out6,$out6,$in5+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out2,$out2,$out2,$inpperm+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x00,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x10,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x20,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x30,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x40,$out+	stvx_u		$out7,$x50,$out+	addi		$out,$out,0x60+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_five:+	vncipherlast	$out3,$out3,$ivec+	vncipherlast	$out4,$out4,$in3+	vncipherlast	$out5,$out5,$in4+	vncipherlast	$out6,$out6,$in5+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out3,$out3,$out3,$inpperm+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x00,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x10,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x20,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x30,$out+	stvx_u		$out7,$x40,$out+	addi		$out,$out,0x50+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_four:+	vncipherlast	$out4,$out4,$ivec+	vncipherlast	$out5,$out5,$in4+	vncipherlast	$out6,$out6,$in5+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out4,$out4,$out4,$inpperm+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x00,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x10,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x20,$out+	stvx_u		$out7,$x30,$out+	addi		$out,$out,0x40+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_three:+	vncipherlast	$out5,$out5,$ivec+	vncipherlast	$out6,$out6,$in5+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out5,$out5,$out5,$inpperm+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x00,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x10,$out+	stvx_u		$out7,$x20,$out+	addi		$out,$out,0x30+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_two:+	vncipherlast	$out6,$out6,$ivec+	vncipherlast	$out7,$out7,$in6+	vmr		$ivec,$in7++	le?vperm	$out6,$out6,$out6,$inpperm+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x00,$out+	stvx_u		$out7,$x10,$out+	addi		$out,$out,0x20+	b		Lcbc_dec8x_done++.align	5+Lcbc_dec8x_one:+	vncipherlast	$out7,$out7,$ivec+	vmr		$ivec,$in7++	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out7,0,$out+	addi		$out,$out,0x10++Lcbc_dec8x_done:+	le?vperm	$ivec,$ivec,$ivec,$inpperm+	stvx_u		$ivec,0,$ivp		# write [unaligned] iv++	li		r10,`$FRAME+15`+	li		r11,`$FRAME+31`+	stvx		$inpperm,r10,$sp	# wipe copies of round keys+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32++	mtspr		256,$vrsave+	lvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	lvx		v21,r11,$sp+	addi		r11,r11,32+	lvx		v22,r10,$sp+	addi		r10,r10,32+	lvx		v23,r11,$sp+	addi		r11,r11,32+	lvx		v24,r10,$sp+	addi		r10,r10,32+	lvx		v25,r11,$sp+	addi		r11,r11,32+	lvx		v26,r10,$sp+	addi		r10,r10,32+	lvx		v27,r11,$sp+	addi		r11,r11,32+	lvx		v28,r10,$sp+	addi		r10,r10,32+	lvx		v29,r11,$sp+	addi		r11,r11,32+	lvx		v30,r10,$sp+	lvx		v31,r11,$sp+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`+	blr+	.long		0+	.byte		0,12,0x04,0,0x80,6,6,0+	.long		0+.size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt+___+}}	}}}++#########################################################################+{{{	# CTR procedure[s]						#+my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));+my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));+my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=+						map("v$_",(4..11));+my $dat=$tmp;++$code.=<<___;+.globl	.${prefix}_ctr32_encrypt_blocks+.align	5+.${prefix}_ctr32_encrypt_blocks:+	${UCMP}i	$len,1+	bltlr-++	lis		r0,0xfff0+	mfspr		$vrsave,256+	mtspr		256,r0++	li		$idx,15+	vxor		$rndkey0,$rndkey0,$rndkey0+	le?vspltisb	$tmp,0x0f++	lvx		$ivec,0,$ivp		# load [unaligned] iv+	lvsl		$inpperm,0,$ivp+	lvx		$inptail,$idx,$ivp+	 vspltisb	$one,1+	le?vxor		$inpperm,$inpperm,$tmp+	vperm		$ivec,$ivec,$inptail,$inpperm+	 vsldoi		$one,$rndkey0,$one,1++	neg		r11,$inp+	?lvsl		$keyperm,0,$key		# prepare for unaligned key+	lwz		$rounds,240($key)++	lvsr		$inpperm,0,r11		# prepare for unaligned load+	lvx		$inptail,0,$inp+	addi		$inp,$inp,15		# 15 is not typo+	le?vxor		$inpperm,$inpperm,$tmp++	srwi		$rounds,$rounds,1+	li		$idx,16+	subi		$rounds,$rounds,1++	${UCMP}i	$len,8+	bge		_aesp8_ctr32_encrypt8x++	?lvsr		$outperm,0,$out		# prepare for unaligned store+	vspltisb	$outmask,-1+	lvx		$outhead,0,$out+	?vperm		$outmask,$rndkey0,$outmask,$outperm+	le?vxor		$outperm,$outperm,$tmp++	lvx		$rndkey0,0,$key+	mtctr		$rounds+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$ivec,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16+	b		Loop_ctr32_enc++.align	5+Loop_ctr32_enc:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipher		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key+	addi		$idx,$idx,16+	bdnz		Loop_ctr32_enc++	vadduwm		$ivec,$ivec,$one+	 vmr		$dat,$inptail+	 lvx		$inptail,0,$inp+	 addi		$inp,$inp,16+	 subic.		$len,$len,1		# blocks--++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key+	 vperm		$dat,$dat,$inptail,$inpperm+	 li		$idx,16+	?vperm		$rndkey1,$rndkey0,$rndkey1,$keyperm+	 lvx		$rndkey0,0,$key+	vxor		$dat,$dat,$rndkey1	# last round key+	vcipherlast	$inout,$inout,$dat++	 lvx		$rndkey1,$idx,$key+	 addi		$idx,$idx,16+	vperm		$inout,$inout,$inout,$outperm+	vsel		$dat,$outhead,$inout,$outmask+	 mtctr		$rounds+	 ?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vmr		$outhead,$inout+	 vxor		$inout,$ivec,$rndkey0+	 lvx		$rndkey0,$idx,$key+	 addi		$idx,$idx,16+	stvx		$dat,0,$out+	addi		$out,$out,16+	bne		Loop_ctr32_enc++	addi		$out,$out,-1+	lvx		$inout,0,$out		# redundant in aligned case+	vsel		$inout,$outhead,$inout,$outmask+	stvx		$inout,0,$out++	mtspr		256,$vrsave+	blr+	.long		0+	.byte		0,12,0x14,0,0,0,6,0+	.long		0+___+#########################################################################+{{	# Optimized CTR procedure					#+my $key_="r11";+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));+    $x00=0 if ($flavour =~ /osx/);+my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));+my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys+			# v26-v31 last 6 round keys+my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment+my ($two,$three,$four)=($outhead,$outperm,$outmask);++$code.=<<___;+.align	5+_aesp8_ctr32_encrypt8x:+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)+	li		r10,`$FRAME+8*16+15`+	li		r11,`$FRAME+8*16+31`+	stvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	stvx		v21,r11,$sp+	addi		r11,r11,32+	stvx		v22,r10,$sp+	addi		r10,r10,32+	stvx		v23,r11,$sp+	addi		r11,r11,32+	stvx		v24,r10,$sp+	addi		r10,r10,32+	stvx		v25,r11,$sp+	addi		r11,r11,32+	stvx		v26,r10,$sp+	addi		r10,r10,32+	stvx		v27,r11,$sp+	addi		r11,r11,32+	stvx		v28,r10,$sp+	addi		r10,r10,32+	stvx		v29,r11,$sp+	addi		r11,r11,32+	stvx		v30,r10,$sp+	stvx		v31,r11,$sp+	li		r0,-1+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave+	li		$x10,0x10+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	li		$x20,0x20+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	li		$x30,0x30+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	li		$x40,0x40+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	li		$x50,0x50+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	li		$x60,0x60+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	li		$x70,0x70+	mtspr		256,r0++	subi		$rounds,$rounds,3	# -4 in total++	lvx		$rndkey0,$x00,$key	# load key schedule+	lvx		v30,$x10,$key+	addi		$key,$key,0x20+	lvx		v31,$x00,$key+	?vperm		$rndkey0,$rndkey0,v30,$keyperm+	addi		$key_,$sp,$FRAME+15+	mtctr		$rounds++Load_ctr32_enc_key:+	?vperm		v24,v30,v31,$keyperm+	lvx		v30,$x10,$key+	addi		$key,$key,0x20+	stvx		v24,$x00,$key_		# off-load round[1]+	?vperm		v25,v31,v30,$keyperm+	lvx		v31,$x00,$key+	stvx		v25,$x10,$key_		# off-load round[2]+	addi		$key_,$key_,0x20+	bdnz		Load_ctr32_enc_key++	lvx		v26,$x10,$key+	?vperm		v24,v30,v31,$keyperm+	lvx		v27,$x20,$key+	stvx		v24,$x00,$key_		# off-load round[3]+	?vperm		v25,v31,v26,$keyperm+	lvx		v28,$x30,$key+	stvx		v25,$x10,$key_		# off-load round[4]+	addi		$key_,$sp,$FRAME+15	# rewind $key_+	?vperm		v26,v26,v27,$keyperm+	lvx		v29,$x40,$key+	?vperm		v27,v27,v28,$keyperm+	lvx		v30,$x50,$key+	?vperm		v28,v28,v29,$keyperm+	lvx		v31,$x60,$key+	?vperm		v29,v29,v30,$keyperm+	lvx		$out0,$x70,$key		# borrow $out0+	?vperm		v30,v30,v31,$keyperm+	lvx		v24,$x00,$key_		# pre-load round[1]+	?vperm		v31,v31,$out0,$keyperm+	lvx		v25,$x10,$key_		# pre-load round[2]++	vadduwm		$two,$one,$one+	subi		$inp,$inp,15		# undo "caller"+	$SHL		$len,$len,4++	vadduwm		$out1,$ivec,$one	# counter values ...+	vadduwm		$out2,$ivec,$two+	vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]+	 le?li		$idx,8+	vadduwm		$out3,$out1,$two+	vxor		$out1,$out1,$rndkey0+	 le?lvsl	$inpperm,0,$idx+	vadduwm		$out4,$out2,$two+	vxor		$out2,$out2,$rndkey0+	 le?vspltisb	$tmp,0x0f+	vadduwm		$out5,$out3,$two+	vxor		$out3,$out3,$rndkey0+	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u+	vadduwm		$out6,$out4,$two+	vxor		$out4,$out4,$rndkey0+	vadduwm		$out7,$out5,$two+	vxor		$out5,$out5,$rndkey0+	vadduwm		$ivec,$out6,$two	# next counter value+	vxor		$out6,$out6,$rndkey0+	vxor		$out7,$out7,$rndkey0++	mtctr		$rounds+	b		Loop_ctr32_enc8x+.align	5+Loop_ctr32_enc8x:+	vcipher 	$out0,$out0,v24+	vcipher 	$out1,$out1,v24+	vcipher 	$out2,$out2,v24+	vcipher 	$out3,$out3,v24+	vcipher 	$out4,$out4,v24+	vcipher 	$out5,$out5,v24+	vcipher 	$out6,$out6,v24+	vcipher 	$out7,$out7,v24+Loop_ctr32_enc8x_middle:+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vcipher 	$out0,$out0,v25+	vcipher 	$out1,$out1,v25+	vcipher 	$out2,$out2,v25+	vcipher 	$out3,$out3,v25+	vcipher 	$out4,$out4,v25+	vcipher 	$out5,$out5,v25+	vcipher 	$out6,$out6,v25+	vcipher 	$out7,$out7,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_ctr32_enc8x++	subic		r11,$len,256		# $len-256, borrow $key_+	vcipher 	$out0,$out0,v24+	vcipher 	$out1,$out1,v24+	vcipher 	$out2,$out2,v24+	vcipher 	$out3,$out3,v24+	vcipher 	$out4,$out4,v24+	vcipher 	$out5,$out5,v24+	vcipher 	$out6,$out6,v24+	vcipher 	$out7,$out7,v24++	subfe		r0,r0,r0		# borrow?-1:0+	vcipher 	$out0,$out0,v25+	vcipher 	$out1,$out1,v25+	vcipher 	$out2,$out2,v25+	vcipher 	$out3,$out3,v25+	vcipher 	$out4,$out4,v25+	vcipher		$out5,$out5,v25+	vcipher		$out6,$out6,v25+	vcipher		$out7,$out7,v25++	and		r0,r0,r11+	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vcipher		$out0,$out0,v26+	vcipher		$out1,$out1,v26+	vcipher		$out2,$out2,v26+	vcipher		$out3,$out3,v26+	vcipher		$out4,$out4,v26+	vcipher		$out5,$out5,v26+	vcipher		$out6,$out6,v26+	vcipher		$out7,$out7,v26+	lvx		v24,$x00,$key_		# re-pre-load round[1]++	subic		$len,$len,129		# $len-=129+	vcipher		$out0,$out0,v27+	addi		$len,$len,1		# $len-=128 really+	vcipher		$out1,$out1,v27+	vcipher		$out2,$out2,v27+	vcipher		$out3,$out3,v27+	vcipher		$out4,$out4,v27+	vcipher		$out5,$out5,v27+	vcipher		$out6,$out6,v27+	vcipher		$out7,$out7,v27+	lvx		v25,$x10,$key_		# re-pre-load round[2]++	vcipher		$out0,$out0,v28+	 lvx_u		$in0,$x00,$inp		# load input+	vcipher		$out1,$out1,v28+	 lvx_u		$in1,$x10,$inp+	vcipher		$out2,$out2,v28+	 lvx_u		$in2,$x20,$inp+	vcipher		$out3,$out3,v28+	 lvx_u		$in3,$x30,$inp+	vcipher		$out4,$out4,v28+	 lvx_u		$in4,$x40,$inp+	vcipher		$out5,$out5,v28+	 lvx_u		$in5,$x50,$inp+	vcipher		$out6,$out6,v28+	 lvx_u		$in6,$x60,$inp+	vcipher		$out7,$out7,v28+	 lvx_u		$in7,$x70,$inp+	 addi		$inp,$inp,0x80++	vcipher		$out0,$out0,v29+	 le?vperm	$in0,$in0,$in0,$inpperm+	vcipher		$out1,$out1,v29+	 le?vperm	$in1,$in1,$in1,$inpperm+	vcipher		$out2,$out2,v29+	 le?vperm	$in2,$in2,$in2,$inpperm+	vcipher		$out3,$out3,v29+	 le?vperm	$in3,$in3,$in3,$inpperm+	vcipher		$out4,$out4,v29+	 le?vperm	$in4,$in4,$in4,$inpperm+	vcipher		$out5,$out5,v29+	 le?vperm	$in5,$in5,$in5,$inpperm+	vcipher		$out6,$out6,v29+	 le?vperm	$in6,$in6,$in6,$inpperm+	vcipher		$out7,$out7,v29+	 le?vperm	$in7,$in7,$in7,$inpperm++	add		$inp,$inp,r0		# $inp is adjusted in such+						# way that at exit from the+						# loop inX-in7 are loaded+						# with last "words"+	subfe.		r0,r0,r0		# borrow?-1:0+	vcipher		$out0,$out0,v30+	 vxor		$in0,$in0,v31		# xor with last round key+	vcipher		$out1,$out1,v30+	 vxor		$in1,$in1,v31+	vcipher		$out2,$out2,v30+	 vxor		$in2,$in2,v31+	vcipher		$out3,$out3,v30+	 vxor		$in3,$in3,v31+	vcipher		$out4,$out4,v30+	 vxor		$in4,$in4,v31+	vcipher		$out5,$out5,v30+	 vxor		$in5,$in5,v31+	vcipher		$out6,$out6,v30+	 vxor		$in6,$in6,v31+	vcipher		$out7,$out7,v30+	 vxor		$in7,$in7,v31++	bne		Lctr32_enc8x_break	# did $len-129 borrow?++	vcipherlast	$in0,$out0,$in0+	vcipherlast	$in1,$out1,$in1+	 vadduwm	$out1,$ivec,$one	# counter values ...+	vcipherlast	$in2,$out2,$in2+	 vadduwm	$out2,$ivec,$two+	 vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]+	vcipherlast	$in3,$out3,$in3+	 vadduwm	$out3,$out1,$two+	 vxor		$out1,$out1,$rndkey0+	vcipherlast	$in4,$out4,$in4+	 vadduwm	$out4,$out2,$two+	 vxor		$out2,$out2,$rndkey0+	vcipherlast	$in5,$out5,$in5+	 vadduwm	$out5,$out3,$two+	 vxor		$out3,$out3,$rndkey0+	vcipherlast	$in6,$out6,$in6+	 vadduwm	$out6,$out4,$two+	 vxor		$out4,$out4,$rndkey0+	vcipherlast	$in7,$out7,$in7+	 vadduwm	$out7,$out5,$two+	 vxor		$out5,$out5,$rndkey0+	le?vperm	$in0,$in0,$in0,$inpperm+	 vadduwm	$ivec,$out6,$two	# next counter value+	 vxor		$out6,$out6,$rndkey0+	le?vperm	$in1,$in1,$in1,$inpperm+	 vxor		$out7,$out7,$rndkey0+	mtctr		$rounds++	 vcipher	$out0,$out0,v24+	stvx_u		$in0,$x00,$out+	le?vperm	$in2,$in2,$in2,$inpperm+	 vcipher	$out1,$out1,v24+	stvx_u		$in1,$x10,$out+	le?vperm	$in3,$in3,$in3,$inpperm+	 vcipher	$out2,$out2,v24+	stvx_u		$in2,$x20,$out+	le?vperm	$in4,$in4,$in4,$inpperm+	 vcipher	$out3,$out3,v24+	stvx_u		$in3,$x30,$out+	le?vperm	$in5,$in5,$in5,$inpperm+	 vcipher	$out4,$out4,v24+	stvx_u		$in4,$x40,$out+	le?vperm	$in6,$in6,$in6,$inpperm+	 vcipher	$out5,$out5,v24+	stvx_u		$in5,$x50,$out+	le?vperm	$in7,$in7,$in7,$inpperm+	 vcipher	$out6,$out6,v24+	stvx_u		$in6,$x60,$out+	 vcipher	$out7,$out7,v24+	stvx_u		$in7,$x70,$out+	addi		$out,$out,0x80++	b		Loop_ctr32_enc8x_middle++.align	5+Lctr32_enc8x_break:+	cmpwi		$len,-0x60+	blt		Lctr32_enc8x_one+	nop+	beq		Lctr32_enc8x_two+	cmpwi		$len,-0x40+	blt		Lctr32_enc8x_three+	nop+	beq		Lctr32_enc8x_four+	cmpwi		$len,-0x20+	blt		Lctr32_enc8x_five+	nop+	beq		Lctr32_enc8x_six+	cmpwi		$len,0x00+	blt		Lctr32_enc8x_seven++Lctr32_enc8x_eight:+	vcipherlast	$out0,$out0,$in0+	vcipherlast	$out1,$out1,$in1+	vcipherlast	$out2,$out2,$in2+	vcipherlast	$out3,$out3,$in3+	vcipherlast	$out4,$out4,$in4+	vcipherlast	$out5,$out5,$in5+	vcipherlast	$out6,$out6,$in6+	vcipherlast	$out7,$out7,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x30,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x40,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x50,$out+	le?vperm	$out7,$out7,$out7,$inpperm+	stvx_u		$out6,$x60,$out+	stvx_u		$out7,$x70,$out+	addi		$out,$out,0x80+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_seven:+	vcipherlast	$out0,$out0,$in1+	vcipherlast	$out1,$out1,$in2+	vcipherlast	$out2,$out2,$in3+	vcipherlast	$out3,$out3,$in4+	vcipherlast	$out4,$out4,$in5+	vcipherlast	$out5,$out5,$in6+	vcipherlast	$out6,$out6,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x30,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x40,$out+	le?vperm	$out6,$out6,$out6,$inpperm+	stvx_u		$out5,$x50,$out+	stvx_u		$out6,$x60,$out+	addi		$out,$out,0x70+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_six:+	vcipherlast	$out0,$out0,$in2+	vcipherlast	$out1,$out1,$in3+	vcipherlast	$out2,$out2,$in4+	vcipherlast	$out3,$out3,$in5+	vcipherlast	$out4,$out4,$in6+	vcipherlast	$out5,$out5,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x30,$out+	le?vperm	$out5,$out5,$out5,$inpperm+	stvx_u		$out4,$x40,$out+	stvx_u		$out5,$x50,$out+	addi		$out,$out,0x60+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_five:+	vcipherlast	$out0,$out0,$in3+	vcipherlast	$out1,$out1,$in4+	vcipherlast	$out2,$out2,$in5+	vcipherlast	$out3,$out3,$in6+	vcipherlast	$out4,$out4,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	le?vperm	$out4,$out4,$out4,$inpperm+	stvx_u		$out3,$x30,$out+	stvx_u		$out4,$x40,$out+	addi		$out,$out,0x50+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_four:+	vcipherlast	$out0,$out0,$in4+	vcipherlast	$out1,$out1,$in5+	vcipherlast	$out2,$out2,$in6+	vcipherlast	$out3,$out3,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$inpperm+	stvx_u		$out2,$x20,$out+	stvx_u		$out3,$x30,$out+	addi		$out,$out,0x40+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_three:+	vcipherlast	$out0,$out0,$in5+	vcipherlast	$out1,$out1,$in6+	vcipherlast	$out2,$out2,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	le?vperm	$out2,$out2,$out2,$inpperm+	stvx_u		$out1,$x10,$out+	stvx_u		$out2,$x20,$out+	addi		$out,$out,0x30+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_two:+	vcipherlast	$out0,$out0,$in6+	vcipherlast	$out1,$out1,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	le?vperm	$out1,$out1,$out1,$inpperm+	stvx_u		$out0,$x00,$out+	stvx_u		$out1,$x10,$out+	addi		$out,$out,0x20+	b		Lctr32_enc8x_done++.align	5+Lctr32_enc8x_one:+	vcipherlast	$out0,$out0,$in7++	le?vperm	$out0,$out0,$out0,$inpperm+	stvx_u		$out0,0,$out+	addi		$out,$out,0x10++Lctr32_enc8x_done:+	li		r10,`$FRAME+15`+	li		r11,`$FRAME+31`+	stvx		$inpperm,r10,$sp	# wipe copies of round keys+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32+	stvx		$inpperm,r10,$sp+	addi		r10,r10,32+	stvx		$inpperm,r11,$sp+	addi		r11,r11,32++	mtspr		256,$vrsave+	lvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	lvx		v21,r11,$sp+	addi		r11,r11,32+	lvx		v22,r10,$sp+	addi		r10,r10,32+	lvx		v23,r11,$sp+	addi		r11,r11,32+	lvx		v24,r10,$sp+	addi		r10,r10,32+	lvx		v25,r11,$sp+	addi		r11,r11,32+	lvx		v26,r10,$sp+	addi		r10,r10,32+	lvx		v27,r11,$sp+	addi		r11,r11,32+	lvx		v28,r10,$sp+	addi		r10,r10,32+	lvx		v29,r11,$sp+	addi		r11,r11,32+	lvx		v30,r10,$sp+	lvx		v31,r11,$sp+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`+	blr+	.long		0+	.byte		0,12,0x04,0,0x80,6,6,0+	.long		0+.size	.${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks+___+}}	}}}++#########################################################################+{{{	# XTS procedures						#+# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#+#                             const AES_KEY *key1, const AES_KEY *key2,	#+#                             [const] unsigned char iv[16]);		#+# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#+# input tweak value is assumed to be encrypted already, and last tweak	#+# value, one suitable for consecutive call on same chunk of data, is	#+# written back to original buffer. In addition, in "tweak chaining"	#+# mode only complete input blocks are processed.			#++my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));+my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));+my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));+my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));+my $taillen = $key2;++   ($inp,$idx) = ($idx,$inp);				# reassign++$code.=<<___;+.globl	.${prefix}_xts_encrypt+.align	5+.${prefix}_xts_encrypt:+	mr		$inp,r3				# reassign+	li		r3,-1+	${UCMP}i	$len,16+	bltlr-++	lis		r0,0xfff0+	mfspr		r12,256				# save vrsave+	li		r11,0+	mtspr		256,r0++	vspltisb	$seven,0x07			# 0x070707..07+	le?lvsl		$leperm,r11,r11+	le?vspltisb	$tmp,0x0f+	le?vxor		$leperm,$leperm,$seven++	li		$idx,15+	lvx		$tweak,0,$ivp			# load [unaligned] iv+	lvsl		$inpperm,0,$ivp+	lvx		$inptail,$idx,$ivp+	le?vxor		$inpperm,$inpperm,$tmp+	vperm		$tweak,$tweak,$inptail,$inpperm++	neg		r11,$inp+	lvsr		$inpperm,0,r11			# prepare for unaligned load+	lvx		$inout,0,$inp+	addi		$inp,$inp,15			# 15 is not typo+	le?vxor		$inpperm,$inpperm,$tmp++	${UCMP}i	$key2,0				# key2==NULL?+	beq		Lxts_enc_no_key2++	?lvsl		$keyperm,0,$key2		# prepare for unaligned key+	lwz		$rounds,240($key2)+	srwi		$rounds,$rounds,1+	subi		$rounds,$rounds,1+	li		$idx,16++	lvx		$rndkey0,0,$key2+	lvx		$rndkey1,$idx,$key2+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$tweak,$tweak,$rndkey0+	lvx		$rndkey0,$idx,$key2+	addi		$idx,$idx,16+	mtctr		$rounds++Ltweak_xts_enc:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$tweak,$tweak,$rndkey1+	lvx		$rndkey1,$idx,$key2+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipher		$tweak,$tweak,$rndkey0+	lvx		$rndkey0,$idx,$key2+	addi		$idx,$idx,16+	bdnz		Ltweak_xts_enc++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$tweak,$tweak,$rndkey1+	lvx		$rndkey1,$idx,$key2+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipherlast	$tweak,$tweak,$rndkey0++	li		$ivp,0				# don't chain the tweak+	b		Lxts_enc++Lxts_enc_no_key2:+	li		$idx,-16+	and		$len,$len,$idx			# in "tweak chaining"+							# mode only complete+							# blocks are processed+Lxts_enc:+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16++	?lvsl		$keyperm,0,$key1		# prepare for unaligned key+	lwz		$rounds,240($key1)+	srwi		$rounds,$rounds,1+	subi		$rounds,$rounds,1+	li		$idx,16++	vslb		$eighty7,$seven,$seven		# 0x808080..80+	vor		$eighty7,$eighty7,$seven	# 0x878787..87+	vspltisb	$tmp,1				# 0x010101..01+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01++	${UCMP}i	$len,96+	bge		_aesp8_xts_encrypt6x++	andi.		$taillen,$len,15+	subic		r0,$len,32+	subi		$taillen,$taillen,16+	subfe		r0,r0,r0+	and		r0,r0,$taillen+	add		$inp,$inp,r0++	lvx		$rndkey0,0,$key1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	vperm		$inout,$inout,$inptail,$inpperm+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$inout,$tweak+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16+	mtctr		$rounds+	b		Loop_xts_enc++.align	5+Loop_xts_enc:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipher		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16+	bdnz		Loop_xts_enc++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	li		$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$rndkey0,$rndkey0,$tweak+	vcipherlast	$output,$inout,$rndkey0++	le?vperm	$tmp,$output,$output,$leperm+	be?nop+	le?stvx_u	$tmp,0,$out+	be?stvx_u	$output,0,$out+	addi		$out,$out,16++	subic.		$len,$len,16+	beq		Lxts_enc_done++	vmr		$inout,$inptail+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16+	lvx		$rndkey0,0,$key1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16++	subic		r0,$len,32+	subfe		r0,r0,r0+	and		r0,r0,$taillen+	add		$inp,$inp,r0++	vsrab		$tmp,$tweak,$seven		# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	vxor		$tweak,$tweak,$tmp++	vperm		$inout,$inout,$inptail,$inpperm+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$inout,$tweak+	vxor		$output,$output,$rndkey0	# just in case $len<16+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16++	mtctr		$rounds+	${UCMP}i	$len,16+	bge		Loop_xts_enc++	vxor		$output,$output,$tweak+	lvsr		$inpperm,0,$len			# $inpperm is no longer needed+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed+	vspltisb	$tmp,-1+	vperm		$inptail,$inptail,$tmp,$inpperm+	vsel		$inout,$inout,$output,$inptail++	subi		r11,$out,17+	subi		$out,$out,16+	mtctr		$len+	li		$len,16+Loop_xts_enc_steal:+	lbzu		r0,1(r11)+	stb		r0,16(r11)+	bdnz		Loop_xts_enc_steal++	mtctr		$rounds+	b		Loop_xts_enc			# one more time...++Lxts_enc_done:+	${UCMP}i	$ivp,0+	beq		Lxts_enc_ret++	vsrab		$tmp,$tweak,$seven		# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	vxor		$tweak,$tweak,$tmp++	le?vperm	$tweak,$tweak,$tweak,$leperm+	stvx_u		$tweak,0,$ivp++Lxts_enc_ret:+	mtspr		256,r12				# restore vrsave+	li		r3,0+	blr+	.long		0+	.byte		0,12,0x04,0,0x80,6,6,0+	.long		0+.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt++.globl	.${prefix}_xts_decrypt+.align	5+.${prefix}_xts_decrypt:+	mr		$inp,r3				# reassign+	li		r3,-1+	${UCMP}i	$len,16+	bltlr-++	lis		r0,0xfff8+	mfspr		r12,256				# save vrsave+	li		r11,0+	mtspr		256,r0++	andi.		r0,$len,15+	neg		r0,r0+	andi.		r0,r0,16+	sub		$len,$len,r0++	vspltisb	$seven,0x07			# 0x070707..07+	le?lvsl		$leperm,r11,r11+	le?vspltisb	$tmp,0x0f+	le?vxor		$leperm,$leperm,$seven++	li		$idx,15+	lvx		$tweak,0,$ivp			# load [unaligned] iv+	lvsl		$inpperm,0,$ivp+	lvx		$inptail,$idx,$ivp+	le?vxor		$inpperm,$inpperm,$tmp+	vperm		$tweak,$tweak,$inptail,$inpperm++	neg		r11,$inp+	lvsr		$inpperm,0,r11			# prepare for unaligned load+	lvx		$inout,0,$inp+	addi		$inp,$inp,15			# 15 is not typo+	le?vxor		$inpperm,$inpperm,$tmp++	${UCMP}i	$key2,0				# key2==NULL?+	beq		Lxts_dec_no_key2++	?lvsl		$keyperm,0,$key2		# prepare for unaligned key+	lwz		$rounds,240($key2)+	srwi		$rounds,$rounds,1+	subi		$rounds,$rounds,1+	li		$idx,16++	lvx		$rndkey0,0,$key2+	lvx		$rndkey1,$idx,$key2+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$tweak,$tweak,$rndkey0+	lvx		$rndkey0,$idx,$key2+	addi		$idx,$idx,16+	mtctr		$rounds++Ltweak_xts_dec:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$tweak,$tweak,$rndkey1+	lvx		$rndkey1,$idx,$key2+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipher		$tweak,$tweak,$rndkey0+	lvx		$rndkey0,$idx,$key2+	addi		$idx,$idx,16+	bdnz		Ltweak_xts_dec++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vcipher		$tweak,$tweak,$rndkey1+	lvx		$rndkey1,$idx,$key2+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vcipherlast	$tweak,$tweak,$rndkey0++	li		$ivp,0				# don't chain the tweak+	b		Lxts_dec++Lxts_dec_no_key2:+	neg		$idx,$len+	andi.		$idx,$idx,15+	add		$len,$len,$idx			# in "tweak chaining"+							# mode only complete+							# blocks are processed+Lxts_dec:+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16++	?lvsl		$keyperm,0,$key1		# prepare for unaligned key+	lwz		$rounds,240($key1)+	srwi		$rounds,$rounds,1+	subi		$rounds,$rounds,1+	li		$idx,16++	vslb		$eighty7,$seven,$seven		# 0x808080..80+	vor		$eighty7,$eighty7,$seven	# 0x878787..87+	vspltisb	$tmp,1				# 0x010101..01+	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01++	${UCMP}i	$len,96+	bge		_aesp8_xts_decrypt6x++	lvx		$rndkey0,0,$key1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	vperm		$inout,$inout,$inptail,$inpperm+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$inout,$tweak+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16+	mtctr		$rounds++	${UCMP}i	$len,16+	blt		Ltail_xts_dec+	be?b		Loop_xts_dec++.align	5+Loop_xts_dec:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vncipher	$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16+	bdnz		Loop_xts_dec++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	li		$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$rndkey0,$rndkey0,$tweak+	vncipherlast	$output,$inout,$rndkey0++	le?vperm	$tmp,$output,$output,$leperm+	be?nop+	le?stvx_u	$tmp,0,$out+	be?stvx_u	$output,0,$out+	addi		$out,$out,16++	subic.		$len,$len,16+	beq		Lxts_dec_done++	vmr		$inout,$inptail+	lvx		$inptail,0,$inp+	addi		$inp,$inp,16+	lvx		$rndkey0,0,$key1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16++	vsrab		$tmp,$tweak,$seven		# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	vxor		$tweak,$tweak,$tmp++	vperm		$inout,$inout,$inptail,$inpperm+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$inout,$inout,$tweak+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16++	mtctr		$rounds+	${UCMP}i	$len,16+	bge		Loop_xts_dec++Ltail_xts_dec:+	vsrab		$tmp,$tweak,$seven		# next tweak value+	vaddubm		$tweak1,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	vxor		$tweak1,$tweak1,$tmp++	subi		$inp,$inp,16+	add		$inp,$inp,$len++	vxor		$inout,$inout,$tweak		# :-(+	vxor		$inout,$inout,$tweak1		# :-)++Loop_xts_dec_short:+	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vncipher	$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16+	bdnz		Loop_xts_dec_short++	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm+	vncipher	$inout,$inout,$rndkey1+	lvx		$rndkey1,$idx,$key1+	li		$idx,16+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm+	vxor		$rndkey0,$rndkey0,$tweak1+	vncipherlast	$output,$inout,$rndkey0++	le?vperm	$tmp,$output,$output,$leperm+	be?nop+	le?stvx_u	$tmp,0,$out+	be?stvx_u	$output,0,$out++	vmr		$inout,$inptail+	lvx		$inptail,0,$inp+	#addi		$inp,$inp,16+	lvx		$rndkey0,0,$key1+	lvx		$rndkey1,$idx,$key1+	addi		$idx,$idx,16+	vperm		$inout,$inout,$inptail,$inpperm+	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm++	lvsr		$inpperm,0,$len			# $inpperm is no longer needed+	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed+	vspltisb	$tmp,-1+	vperm		$inptail,$inptail,$tmp,$inpperm+	vsel		$inout,$inout,$output,$inptail++	vxor		$rndkey0,$rndkey0,$tweak+	vxor		$inout,$inout,$rndkey0+	lvx		$rndkey0,$idx,$key1+	addi		$idx,$idx,16++	subi		r11,$out,1+	mtctr		$len+	li		$len,16+Loop_xts_dec_steal:+	lbzu		r0,1(r11)+	stb		r0,16(r11)+	bdnz		Loop_xts_dec_steal++	mtctr		$rounds+	b		Loop_xts_dec			# one more time...++Lxts_dec_done:+	${UCMP}i	$ivp,0+	beq		Lxts_dec_ret++	vsrab		$tmp,$tweak,$seven		# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	vxor		$tweak,$tweak,$tmp++	le?vperm	$tweak,$tweak,$tweak,$leperm+	stvx_u		$tweak,0,$ivp++Lxts_dec_ret:+	mtspr		256,r12				# restore vrsave+	li		r3,0+	blr+	.long		0+	.byte		0,12,0x04,0,0x80,6,6,0+	.long		0+.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt+___+#########################################################################+{{	# Optimized XTS procedures					#+my $key_=$key2;+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));+    $x00=0 if ($flavour =~ /osx/);+my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));+my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));+my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));+my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys+			# v26-v31 last 6 round keys+my ($keyperm)=($out0);	# aliases with "caller", redundant assignment+my $taillen=$x70;++$code.=<<___;+.align	5+_aesp8_xts_encrypt6x:+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)+	mflr		r11+	li		r7,`$FRAME+8*16+15`+	li		r3,`$FRAME+8*16+31`+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)+	stvx		v20,r7,$sp		# ABI says so+	addi		r7,r7,32+	stvx		v21,r3,$sp+	addi		r3,r3,32+	stvx		v22,r7,$sp+	addi		r7,r7,32+	stvx		v23,r3,$sp+	addi		r3,r3,32+	stvx		v24,r7,$sp+	addi		r7,r7,32+	stvx		v25,r3,$sp+	addi		r3,r3,32+	stvx		v26,r7,$sp+	addi		r7,r7,32+	stvx		v27,r3,$sp+	addi		r3,r3,32+	stvx		v28,r7,$sp+	addi		r7,r7,32+	stvx		v29,r3,$sp+	addi		r3,r3,32+	stvx		v30,r7,$sp+	stvx		v31,r3,$sp+	li		r0,-1+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave+	li		$x10,0x10+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	li		$x20,0x20+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	li		$x30,0x30+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	li		$x40,0x40+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	li		$x50,0x50+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	li		$x60,0x60+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	li		$x70,0x70+	mtspr		256,r0++	subi		$rounds,$rounds,3	# -4 in total++	lvx		$rndkey0,$x00,$key1	# load key schedule+	lvx		v30,$x10,$key1+	addi		$key1,$key1,0x20+	lvx		v31,$x00,$key1+	?vperm		$rndkey0,$rndkey0,v30,$keyperm+	addi		$key_,$sp,$FRAME+15+	mtctr		$rounds++Load_xts_enc_key:+	?vperm		v24,v30,v31,$keyperm+	lvx		v30,$x10,$key1+	addi		$key1,$key1,0x20+	stvx		v24,$x00,$key_		# off-load round[1]+	?vperm		v25,v31,v30,$keyperm+	lvx		v31,$x00,$key1+	stvx		v25,$x10,$key_		# off-load round[2]+	addi		$key_,$key_,0x20+	bdnz		Load_xts_enc_key++	lvx		v26,$x10,$key1+	?vperm		v24,v30,v31,$keyperm+	lvx		v27,$x20,$key1+	stvx		v24,$x00,$key_		# off-load round[3]+	?vperm		v25,v31,v26,$keyperm+	lvx		v28,$x30,$key1+	stvx		v25,$x10,$key_		# off-load round[4]+	addi		$key_,$sp,$FRAME+15	# rewind $key_+	?vperm		v26,v26,v27,$keyperm+	lvx		v29,$x40,$key1+	?vperm		v27,v27,v28,$keyperm+	lvx		v30,$x50,$key1+	?vperm		v28,v28,v29,$keyperm+	lvx		v31,$x60,$key1+	?vperm		v29,v29,v30,$keyperm+	lvx		$twk5,$x70,$key1	# borrow $twk5+	?vperm		v30,v30,v31,$keyperm+	lvx		v24,$x00,$key_		# pre-load round[1]+	?vperm		v31,v31,$twk5,$keyperm+	lvx		v25,$x10,$key_		# pre-load round[2]++	 vperm		$in0,$inout,$inptail,$inpperm+	 subi		$inp,$inp,31		# undo "caller"+	vxor		$twk0,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	 vxor		$out0,$in0,$twk0+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in1,$x10,$inp+	vxor		$twk1,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in1,$in1,$in1,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out1,$in1,$twk1+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in2,$x20,$inp+	 andi.		$taillen,$len,15+	vxor		$twk2,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in2,$in2,$in2,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out2,$in2,$twk2+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in3,$x30,$inp+	 sub		$len,$len,$taillen+	vxor		$twk3,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in3,$in3,$in3,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out3,$in3,$twk3+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in4,$x40,$inp+	 subi		$len,$len,0x60+	vxor		$twk4,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in4,$in4,$in4,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out4,$in4,$twk4+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in5,$x50,$inp+	 addi		$inp,$inp,0x60+	vxor		$twk5,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in5,$in5,$in5,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out5,$in5,$twk5+	vxor		$tweak,$tweak,$tmp++	vxor		v31,v31,$rndkey0+	mtctr		$rounds+	b		Loop_xts_enc6x++.align	5+Loop_xts_enc6x:+	vcipher		$out0,$out0,v24+	vcipher		$out1,$out1,v24+	vcipher		$out2,$out2,v24+	vcipher		$out3,$out3,v24+	vcipher		$out4,$out4,v24+	vcipher		$out5,$out5,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vcipher		$out0,$out0,v25+	vcipher		$out1,$out1,v25+	vcipher		$out2,$out2,v25+	vcipher		$out3,$out3,v25+	vcipher		$out4,$out4,v25+	vcipher		$out5,$out5,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_xts_enc6x++	subic		$len,$len,96		# $len-=96+	 vxor		$in0,$twk0,v31		# xor with last round key+	vcipher		$out0,$out0,v24+	vcipher		$out1,$out1,v24+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk0,$tweak,$rndkey0+	 vaddubm	$tweak,$tweak,$tweak+	vcipher		$out2,$out2,v24+	vcipher		$out3,$out3,v24+	 vsldoi		$tmp,$tmp,$tmp,15+	vcipher		$out4,$out4,v24+	vcipher		$out5,$out5,v24++	subfe.		r0,r0,r0		# borrow?-1:0+	 vand		$tmp,$tmp,$eighty7+	vcipher		$out0,$out0,v25+	vcipher		$out1,$out1,v25+	 vxor		$tweak,$tweak,$tmp+	vcipher		$out2,$out2,v25+	vcipher		$out3,$out3,v25+	 vxor		$in1,$twk1,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk1,$tweak,$rndkey0+	vcipher		$out4,$out4,v25+	vcipher		$out5,$out5,v25++	and		r0,r0,$len+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vcipher		$out0,$out0,v26+	vcipher		$out1,$out1,v26+	 vand		$tmp,$tmp,$eighty7+	vcipher		$out2,$out2,v26+	vcipher		$out3,$out3,v26+	 vxor		$tweak,$tweak,$tmp+	vcipher		$out4,$out4,v26+	vcipher		$out5,$out5,v26++	add		$inp,$inp,r0		# $inp is adjusted in such+						# way that at exit from the+						# loop inX-in5 are loaded+						# with last "words"+	 vxor		$in2,$twk2,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk2,$tweak,$rndkey0+	 vaddubm	$tweak,$tweak,$tweak+	vcipher		$out0,$out0,v27+	vcipher		$out1,$out1,v27+	 vsldoi		$tmp,$tmp,$tmp,15+	vcipher		$out2,$out2,v27+	vcipher		$out3,$out3,v27+	 vand		$tmp,$tmp,$eighty7+	vcipher		$out4,$out4,v27+	vcipher		$out5,$out5,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	 vxor		$tweak,$tweak,$tmp+	vcipher		$out0,$out0,v28+	vcipher		$out1,$out1,v28+	 vxor		$in3,$twk3,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk3,$tweak,$rndkey0+	vcipher		$out2,$out2,v28+	vcipher		$out3,$out3,v28+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vcipher		$out4,$out4,v28+	vcipher		$out5,$out5,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]+	 vand		$tmp,$tmp,$eighty7++	vcipher		$out0,$out0,v29+	vcipher		$out1,$out1,v29+	 vxor		$tweak,$tweak,$tmp+	vcipher		$out2,$out2,v29+	vcipher		$out3,$out3,v29+	 vxor		$in4,$twk4,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk4,$tweak,$rndkey0+	vcipher		$out4,$out4,v29+	vcipher		$out5,$out5,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15++	vcipher		$out0,$out0,v30+	vcipher		$out1,$out1,v30+	 vand		$tmp,$tmp,$eighty7+	vcipher		$out2,$out2,v30+	vcipher		$out3,$out3,v30+	 vxor		$tweak,$tweak,$tmp+	vcipher		$out4,$out4,v30+	vcipher		$out5,$out5,v30+	 vxor		$in5,$twk5,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk5,$tweak,$rndkey0++	vcipherlast	$out0,$out0,$in0+	 lvx_u		$in0,$x00,$inp		# load next input block+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vcipherlast	$out1,$out1,$in1+	 lvx_u		$in1,$x10,$inp+	vcipherlast	$out2,$out2,$in2+	 le?vperm	$in0,$in0,$in0,$leperm+	 lvx_u		$in2,$x20,$inp+	 vand		$tmp,$tmp,$eighty7+	vcipherlast	$out3,$out3,$in3+	 le?vperm	$in1,$in1,$in1,$leperm+	 lvx_u		$in3,$x30,$inp+	vcipherlast	$out4,$out4,$in4+	 le?vperm	$in2,$in2,$in2,$leperm+	 lvx_u		$in4,$x40,$inp+	 vxor		$tweak,$tweak,$tmp+	vcipherlast	$tmp,$out5,$in5		# last block might be needed+						# in stealing mode+	 le?vperm	$in3,$in3,$in3,$leperm+	 lvx_u		$in5,$x50,$inp+	 addi		$inp,$inp,0x60+	 le?vperm	$in4,$in4,$in4,$leperm+	 le?vperm	$in5,$in5,$in5,$leperm++	le?vperm	$out0,$out0,$out0,$leperm+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	 vxor		$out0,$in0,$twk0+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	 vxor		$out1,$in1,$twk1+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	 vxor		$out2,$in2,$twk2+	le?vperm	$out4,$out4,$out4,$leperm+	stvx_u		$out3,$x30,$out+	 vxor		$out3,$in3,$twk3+	le?vperm	$out5,$tmp,$tmp,$leperm+	stvx_u		$out4,$x40,$out+	 vxor		$out4,$in4,$twk4+	le?stvx_u	$out5,$x50,$out+	be?stvx_u	$tmp, $x50,$out+	 vxor		$out5,$in5,$twk5+	addi		$out,$out,0x60++	mtctr		$rounds+	beq		Loop_xts_enc6x		# did $len-=96 borrow?++	addic.		$len,$len,0x60+	beq		Lxts_enc6x_zero+	cmpwi		$len,0x20+	blt		Lxts_enc6x_one+	nop+	beq		Lxts_enc6x_two+	cmpwi		$len,0x40+	blt		Lxts_enc6x_three+	nop+	beq		Lxts_enc6x_four++Lxts_enc6x_five:+	vxor		$out0,$in1,$twk0+	vxor		$out1,$in2,$twk1+	vxor		$out2,$in3,$twk2+	vxor		$out3,$in4,$twk3+	vxor		$out4,$in5,$twk4++	bl		_aesp8_xts_enc5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk5		# unused tweak+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	vxor		$tmp,$out4,$twk5	# last block prep for stealing+	le?vperm	$out4,$out4,$out4,$leperm+	stvx_u		$out3,$x30,$out+	stvx_u		$out4,$x40,$out+	addi		$out,$out,0x50+	bne		Lxts_enc6x_steal+	b		Lxts_enc6x_done++.align	4+Lxts_enc6x_four:+	vxor		$out0,$in2,$twk0+	vxor		$out1,$in3,$twk1+	vxor		$out2,$in4,$twk2+	vxor		$out3,$in5,$twk3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_enc5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk4		# unused tweak+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	vxor		$tmp,$out3,$twk4	# last block prep for stealing+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	stvx_u		$out3,$x30,$out+	addi		$out,$out,0x40+	bne		Lxts_enc6x_steal+	b		Lxts_enc6x_done++.align	4+Lxts_enc6x_three:+	vxor		$out0,$in3,$twk0+	vxor		$out1,$in4,$twk1+	vxor		$out2,$in5,$twk2+	vxor		$out3,$out3,$out3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_enc5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk3		# unused tweak+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	vxor		$tmp,$out2,$twk3	# last block prep for stealing+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	stvx_u		$out2,$x20,$out+	addi		$out,$out,0x30+	bne		Lxts_enc6x_steal+	b		Lxts_enc6x_done++.align	4+Lxts_enc6x_two:+	vxor		$out0,$in4,$twk0+	vxor		$out1,$in5,$twk1+	vxor		$out2,$out2,$out2+	vxor		$out3,$out3,$out3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_enc5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk2		# unused tweak+	vxor		$tmp,$out1,$twk2	# last block prep for stealing+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	stvx_u		$out1,$x10,$out+	addi		$out,$out,0x20+	bne		Lxts_enc6x_steal+	b		Lxts_enc6x_done++.align	4+Lxts_enc6x_one:+	vxor		$out0,$in5,$twk0+	nop+Loop_xts_enc1x:+	vcipher		$out0,$out0,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vcipher		$out0,$out0,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_xts_enc1x++	add		$inp,$inp,$taillen+	cmpwi		$taillen,0+	vcipher		$out0,$out0,v24++	subi		$inp,$inp,16+	vcipher		$out0,$out0,v25++	lvsr		$inpperm,0,$taillen+	vcipher		$out0,$out0,v26++	lvx_u		$in0,0,$inp+	vcipher		$out0,$out0,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vcipher		$out0,$out0,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]++	vcipher		$out0,$out0,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vxor		$twk0,$twk0,v31++	le?vperm	$in0,$in0,$in0,$leperm+	vcipher		$out0,$out0,v30++	vperm		$in0,$in0,$in0,$inpperm+	vcipherlast	$out0,$out0,$twk0++	vmr		$twk0,$twk1		# unused tweak+	vxor		$tmp,$out0,$twk1	# last block prep for stealing+	le?vperm	$out0,$out0,$out0,$leperm+	stvx_u		$out0,$x00,$out		# store output+	addi		$out,$out,0x10+	bne		Lxts_enc6x_steal+	b		Lxts_enc6x_done++.align	4+Lxts_enc6x_zero:+	cmpwi		$taillen,0+	beq		Lxts_enc6x_done++	add		$inp,$inp,$taillen+	subi		$inp,$inp,16+	lvx_u		$in0,0,$inp+	lvsr		$inpperm,0,$taillen	# $in5 is no more+	le?vperm	$in0,$in0,$in0,$leperm+	vperm		$in0,$in0,$in0,$inpperm+	vxor		$tmp,$tmp,$twk0+Lxts_enc6x_steal:+	vxor		$in0,$in0,$twk0+	vxor		$out0,$out0,$out0+	vspltisb	$out1,-1+	vperm		$out0,$out0,$out1,$inpperm+	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?++	subi		r30,$out,17+	subi		$out,$out,16+	mtctr		$taillen+Loop_xts_enc6x_steal:+	lbzu		r0,1(r30)+	stb		r0,16(r30)+	bdnz		Loop_xts_enc6x_steal++	li		$taillen,0+	mtctr		$rounds+	b		Loop_xts_enc1x		# one more time...++.align	4+Lxts_enc6x_done:+	${UCMP}i	$ivp,0+	beq		Lxts_enc6x_ret++	vxor		$tweak,$twk0,$rndkey0+	le?vperm	$tweak,$tweak,$tweak,$leperm+	stvx_u		$tweak,0,$ivp++Lxts_enc6x_ret:+	mtlr		r11+	li		r10,`$FRAME+15`+	li		r11,`$FRAME+31`+	stvx		$seven,r10,$sp		# wipe copies of round keys+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32++	mtspr		256,$vrsave+	lvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	lvx		v21,r11,$sp+	addi		r11,r11,32+	lvx		v22,r10,$sp+	addi		r10,r10,32+	lvx		v23,r11,$sp+	addi		r11,r11,32+	lvx		v24,r10,$sp+	addi		r10,r10,32+	lvx		v25,r11,$sp+	addi		r11,r11,32+	lvx		v26,r10,$sp+	addi		r10,r10,32+	lvx		v27,r11,$sp+	addi		r11,r11,32+	lvx		v28,r10,$sp+	addi		r10,r10,32+	lvx		v29,r11,$sp+	addi		r11,r11,32+	lvx		v30,r10,$sp+	lvx		v31,r11,$sp+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`+	blr+	.long		0+	.byte		0,12,0x04,1,0x80,6,6,0+	.long		0++.align	5+_aesp8_xts_enc5x:+	vcipher		$out0,$out0,v24+	vcipher		$out1,$out1,v24+	vcipher		$out2,$out2,v24+	vcipher		$out3,$out3,v24+	vcipher		$out4,$out4,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vcipher		$out0,$out0,v25+	vcipher		$out1,$out1,v25+	vcipher		$out2,$out2,v25+	vcipher		$out3,$out3,v25+	vcipher		$out4,$out4,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		_aesp8_xts_enc5x++	add		$inp,$inp,$taillen+	cmpwi		$taillen,0+	vcipher		$out0,$out0,v24+	vcipher		$out1,$out1,v24+	vcipher		$out2,$out2,v24+	vcipher		$out3,$out3,v24+	vcipher		$out4,$out4,v24++	subi		$inp,$inp,16+	vcipher		$out0,$out0,v25+	vcipher		$out1,$out1,v25+	vcipher		$out2,$out2,v25+	vcipher		$out3,$out3,v25+	vcipher		$out4,$out4,v25+	 vxor		$twk0,$twk0,v31++	vcipher		$out0,$out0,v26+	lvsr		$inpperm,0,$taillen	# $in5 is no more+	vcipher		$out1,$out1,v26+	vcipher		$out2,$out2,v26+	vcipher		$out3,$out3,v26+	vcipher		$out4,$out4,v26+	 vxor		$in1,$twk1,v31++	vcipher		$out0,$out0,v27+	lvx_u		$in0,0,$inp+	vcipher		$out1,$out1,v27+	vcipher		$out2,$out2,v27+	vcipher		$out3,$out3,v27+	vcipher		$out4,$out4,v27+	 vxor		$in2,$twk2,v31++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vcipher		$out0,$out0,v28+	vcipher		$out1,$out1,v28+	vcipher		$out2,$out2,v28+	vcipher		$out3,$out3,v28+	vcipher		$out4,$out4,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]+	 vxor		$in3,$twk3,v31++	vcipher		$out0,$out0,v29+	le?vperm	$in0,$in0,$in0,$leperm+	vcipher		$out1,$out1,v29+	vcipher		$out2,$out2,v29+	vcipher		$out3,$out3,v29+	vcipher		$out4,$out4,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vxor		$in4,$twk4,v31++	vcipher		$out0,$out0,v30+	vperm		$in0,$in0,$in0,$inpperm+	vcipher		$out1,$out1,v30+	vcipher		$out2,$out2,v30+	vcipher		$out3,$out3,v30+	vcipher		$out4,$out4,v30++	vcipherlast	$out0,$out0,$twk0+	vcipherlast	$out1,$out1,$in1+	vcipherlast	$out2,$out2,$in2+	vcipherlast	$out3,$out3,$in3+	vcipherlast	$out4,$out4,$in4+	blr+        .long   	0+        .byte   	0,12,0x14,0,0,0,0,0++.align	5+_aesp8_xts_decrypt6x:+	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)+	mflr		r11+	li		r7,`$FRAME+8*16+15`+	li		r3,`$FRAME+8*16+31`+	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)+	stvx		v20,r7,$sp		# ABI says so+	addi		r7,r7,32+	stvx		v21,r3,$sp+	addi		r3,r3,32+	stvx		v22,r7,$sp+	addi		r7,r7,32+	stvx		v23,r3,$sp+	addi		r3,r3,32+	stvx		v24,r7,$sp+	addi		r7,r7,32+	stvx		v25,r3,$sp+	addi		r3,r3,32+	stvx		v26,r7,$sp+	addi		r7,r7,32+	stvx		v27,r3,$sp+	addi		r3,r3,32+	stvx		v28,r7,$sp+	addi		r7,r7,32+	stvx		v29,r3,$sp+	addi		r3,r3,32+	stvx		v30,r7,$sp+	stvx		v31,r3,$sp+	li		r0,-1+	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave+	li		$x10,0x10+	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	li		$x20,0x20+	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	li		$x30,0x30+	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	li		$x40,0x40+	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	li		$x50,0x50+	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	li		$x60,0x60+	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	li		$x70,0x70+	mtspr		256,r0++	subi		$rounds,$rounds,3	# -4 in total++	lvx		$rndkey0,$x00,$key1	# load key schedule+	lvx		v30,$x10,$key1+	addi		$key1,$key1,0x20+	lvx		v31,$x00,$key1+	?vperm		$rndkey0,$rndkey0,v30,$keyperm+	addi		$key_,$sp,$FRAME+15+	mtctr		$rounds++Load_xts_dec_key:+	?vperm		v24,v30,v31,$keyperm+	lvx		v30,$x10,$key1+	addi		$key1,$key1,0x20+	stvx		v24,$x00,$key_		# off-load round[1]+	?vperm		v25,v31,v30,$keyperm+	lvx		v31,$x00,$key1+	stvx		v25,$x10,$key_		# off-load round[2]+	addi		$key_,$key_,0x20+	bdnz		Load_xts_dec_key++	lvx		v26,$x10,$key1+	?vperm		v24,v30,v31,$keyperm+	lvx		v27,$x20,$key1+	stvx		v24,$x00,$key_		# off-load round[3]+	?vperm		v25,v31,v26,$keyperm+	lvx		v28,$x30,$key1+	stvx		v25,$x10,$key_		# off-load round[4]+	addi		$key_,$sp,$FRAME+15	# rewind $key_+	?vperm		v26,v26,v27,$keyperm+	lvx		v29,$x40,$key1+	?vperm		v27,v27,v28,$keyperm+	lvx		v30,$x50,$key1+	?vperm		v28,v28,v29,$keyperm+	lvx		v31,$x60,$key1+	?vperm		v29,v29,v30,$keyperm+	lvx		$twk5,$x70,$key1	# borrow $twk5+	?vperm		v30,v30,v31,$keyperm+	lvx		v24,$x00,$key_		# pre-load round[1]+	?vperm		v31,v31,$twk5,$keyperm+	lvx		v25,$x10,$key_		# pre-load round[2]++	 vperm		$in0,$inout,$inptail,$inpperm+	 subi		$inp,$inp,31		# undo "caller"+	vxor		$twk0,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	vand		$tmp,$tmp,$eighty7+	 vxor		$out0,$in0,$twk0+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in1,$x10,$inp+	vxor		$twk1,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in1,$in1,$in1,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out1,$in1,$twk1+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in2,$x20,$inp+	 andi.		$taillen,$len,15+	vxor		$twk2,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in2,$in2,$in2,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out2,$in2,$twk2+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in3,$x30,$inp+	 sub		$len,$len,$taillen+	vxor		$twk3,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in3,$in3,$in3,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out3,$in3,$twk3+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in4,$x40,$inp+	 subi		$len,$len,0x60+	vxor		$twk4,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in4,$in4,$in4,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out4,$in4,$twk4+	vxor		$tweak,$tweak,$tmp++	 lvx_u		$in5,$x50,$inp+	 addi		$inp,$inp,0x60+	vxor		$twk5,$tweak,$rndkey0+	vsrab		$tmp,$tweak,$seven	# next tweak value+	vaddubm		$tweak,$tweak,$tweak+	vsldoi		$tmp,$tmp,$tmp,15+	 le?vperm	$in5,$in5,$in5,$leperm+	vand		$tmp,$tmp,$eighty7+	 vxor		$out5,$in5,$twk5+	vxor		$tweak,$tweak,$tmp++	vxor		v31,v31,$rndkey0+	mtctr		$rounds+	b		Loop_xts_dec6x++.align	5+Loop_xts_dec6x:+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_xts_dec6x++	subic		$len,$len,96		# $len-=96+	 vxor		$in0,$twk0,v31		# xor with last round key+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk0,$tweak,$rndkey0+	 vaddubm	$tweak,$tweak,$tweak+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	 vsldoi		$tmp,$tmp,$tmp,15+	vncipher	$out4,$out4,v24+	vncipher	$out5,$out5,v24++	subfe.		r0,r0,r0		# borrow?-1:0+	 vand		$tmp,$tmp,$eighty7+	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	 vxor		$tweak,$tweak,$tmp+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	 vxor		$in1,$twk1,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk1,$tweak,$rndkey0+	vncipher	$out4,$out4,v25+	vncipher	$out5,$out5,v25++	and		r0,r0,$len+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vncipher	$out0,$out0,v26+	vncipher	$out1,$out1,v26+	 vand		$tmp,$tmp,$eighty7+	vncipher	$out2,$out2,v26+	vncipher	$out3,$out3,v26+	 vxor		$tweak,$tweak,$tmp+	vncipher	$out4,$out4,v26+	vncipher	$out5,$out5,v26++	add		$inp,$inp,r0		# $inp is adjusted in such+						# way that at exit from the+						# loop inX-in5 are loaded+						# with last "words"+	 vxor		$in2,$twk2,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk2,$tweak,$rndkey0+	 vaddubm	$tweak,$tweak,$tweak+	vncipher	$out0,$out0,v27+	vncipher	$out1,$out1,v27+	 vsldoi		$tmp,$tmp,$tmp,15+	vncipher	$out2,$out2,v27+	vncipher	$out3,$out3,v27+	 vand		$tmp,$tmp,$eighty7+	vncipher	$out4,$out4,v27+	vncipher	$out5,$out5,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	 vxor		$tweak,$tweak,$tmp+	vncipher	$out0,$out0,v28+	vncipher	$out1,$out1,v28+	 vxor		$in3,$twk3,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk3,$tweak,$rndkey0+	vncipher	$out2,$out2,v28+	vncipher	$out3,$out3,v28+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vncipher	$out4,$out4,v28+	vncipher	$out5,$out5,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]+	 vand		$tmp,$tmp,$eighty7++	vncipher	$out0,$out0,v29+	vncipher	$out1,$out1,v29+	 vxor		$tweak,$tweak,$tmp+	vncipher	$out2,$out2,v29+	vncipher	$out3,$out3,v29+	 vxor		$in4,$twk4,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk4,$tweak,$rndkey0+	vncipher	$out4,$out4,v29+	vncipher	$out5,$out5,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15++	vncipher	$out0,$out0,v30+	vncipher	$out1,$out1,v30+	 vand		$tmp,$tmp,$eighty7+	vncipher	$out2,$out2,v30+	vncipher	$out3,$out3,v30+	 vxor		$tweak,$tweak,$tmp+	vncipher	$out4,$out4,v30+	vncipher	$out5,$out5,v30+	 vxor		$in5,$twk5,v31+	 vsrab		$tmp,$tweak,$seven	# next tweak value+	 vxor		$twk5,$tweak,$rndkey0++	vncipherlast	$out0,$out0,$in0+	 lvx_u		$in0,$x00,$inp		# load next input block+	 vaddubm	$tweak,$tweak,$tweak+	 vsldoi		$tmp,$tmp,$tmp,15+	vncipherlast	$out1,$out1,$in1+	 lvx_u		$in1,$x10,$inp+	vncipherlast	$out2,$out2,$in2+	 le?vperm	$in0,$in0,$in0,$leperm+	 lvx_u		$in2,$x20,$inp+	 vand		$tmp,$tmp,$eighty7+	vncipherlast	$out3,$out3,$in3+	 le?vperm	$in1,$in1,$in1,$leperm+	 lvx_u		$in3,$x30,$inp+	vncipherlast	$out4,$out4,$in4+	 le?vperm	$in2,$in2,$in2,$leperm+	 lvx_u		$in4,$x40,$inp+	 vxor		$tweak,$tweak,$tmp+	vncipherlast	$out5,$out5,$in5+	 le?vperm	$in3,$in3,$in3,$leperm+	 lvx_u		$in5,$x50,$inp+	 addi		$inp,$inp,0x60+	 le?vperm	$in4,$in4,$in4,$leperm+	 le?vperm	$in5,$in5,$in5,$leperm++	le?vperm	$out0,$out0,$out0,$leperm+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	 vxor		$out0,$in0,$twk0+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	 vxor		$out1,$in1,$twk1+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	 vxor		$out2,$in2,$twk2+	le?vperm	$out4,$out4,$out4,$leperm+	stvx_u		$out3,$x30,$out+	 vxor		$out3,$in3,$twk3+	le?vperm	$out5,$out5,$out5,$leperm+	stvx_u		$out4,$x40,$out+	 vxor		$out4,$in4,$twk4+	stvx_u		$out5,$x50,$out+	 vxor		$out5,$in5,$twk5+	addi		$out,$out,0x60++	mtctr		$rounds+	beq		Loop_xts_dec6x		# did $len-=96 borrow?++	addic.		$len,$len,0x60+	beq		Lxts_dec6x_zero+	cmpwi		$len,0x20+	blt		Lxts_dec6x_one+	nop+	beq		Lxts_dec6x_two+	cmpwi		$len,0x40+	blt		Lxts_dec6x_three+	nop+	beq		Lxts_dec6x_four++Lxts_dec6x_five:+	vxor		$out0,$in1,$twk0+	vxor		$out1,$in2,$twk1+	vxor		$out2,$in3,$twk2+	vxor		$out3,$in4,$twk3+	vxor		$out4,$in5,$twk4++	bl		_aesp8_xts_dec5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk5		# unused tweak+	vxor		$twk1,$tweak,$rndkey0+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	vxor		$out0,$in0,$twk1+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	le?vperm	$out4,$out4,$out4,$leperm+	stvx_u		$out3,$x30,$out+	stvx_u		$out4,$x40,$out+	addi		$out,$out,0x50+	bne		Lxts_dec6x_steal+	b		Lxts_dec6x_done++.align	4+Lxts_dec6x_four:+	vxor		$out0,$in2,$twk0+	vxor		$out1,$in3,$twk1+	vxor		$out2,$in4,$twk2+	vxor		$out3,$in5,$twk3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_dec5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk4		# unused tweak+	vmr		$twk1,$twk5+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	vxor		$out0,$in0,$twk5+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	le?vperm	$out3,$out3,$out3,$leperm+	stvx_u		$out2,$x20,$out+	stvx_u		$out3,$x30,$out+	addi		$out,$out,0x40+	bne		Lxts_dec6x_steal+	b		Lxts_dec6x_done++.align	4+Lxts_dec6x_three:+	vxor		$out0,$in3,$twk0+	vxor		$out1,$in4,$twk1+	vxor		$out2,$in5,$twk2+	vxor		$out3,$out3,$out3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_dec5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk3		# unused tweak+	vmr		$twk1,$twk4+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	vxor		$out0,$in0,$twk4+	le?vperm	$out2,$out2,$out2,$leperm+	stvx_u		$out1,$x10,$out+	stvx_u		$out2,$x20,$out+	addi		$out,$out,0x30+	bne		Lxts_dec6x_steal+	b		Lxts_dec6x_done++.align	4+Lxts_dec6x_two:+	vxor		$out0,$in4,$twk0+	vxor		$out1,$in5,$twk1+	vxor		$out2,$out2,$out2+	vxor		$out3,$out3,$out3+	vxor		$out4,$out4,$out4++	bl		_aesp8_xts_dec5x++	le?vperm	$out0,$out0,$out0,$leperm+	vmr		$twk0,$twk2		# unused tweak+	vmr		$twk1,$twk3+	le?vperm	$out1,$out1,$out1,$leperm+	stvx_u		$out0,$x00,$out		# store output+	vxor		$out0,$in0,$twk3+	stvx_u		$out1,$x10,$out+	addi		$out,$out,0x20+	bne		Lxts_dec6x_steal+	b		Lxts_dec6x_done++.align	4+Lxts_dec6x_one:+	vxor		$out0,$in5,$twk0+	nop+Loop_xts_dec1x:+	vncipher	$out0,$out0,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out0,$out0,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Loop_xts_dec1x++	subi		r0,$taillen,1+	vncipher	$out0,$out0,v24++	andi.		r0,r0,16+	cmpwi		$taillen,0+	vncipher	$out0,$out0,v25++	sub		$inp,$inp,r0+	vncipher	$out0,$out0,v26++	lvx_u		$in0,0,$inp+	vncipher	$out0,$out0,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vncipher	$out0,$out0,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]++	vncipher	$out0,$out0,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vxor		$twk0,$twk0,v31++	le?vperm	$in0,$in0,$in0,$leperm+	vncipher	$out0,$out0,v30++	mtctr		$rounds+	vncipherlast	$out0,$out0,$twk0++	vmr		$twk0,$twk1		# unused tweak+	vmr		$twk1,$twk2+	le?vperm	$out0,$out0,$out0,$leperm+	stvx_u		$out0,$x00,$out		# store output+	addi		$out,$out,0x10+	vxor		$out0,$in0,$twk2+	bne		Lxts_dec6x_steal+	b		Lxts_dec6x_done++.align	4+Lxts_dec6x_zero:+	cmpwi		$taillen,0+	beq		Lxts_dec6x_done++	lvx_u		$in0,0,$inp+	le?vperm	$in0,$in0,$in0,$leperm+	vxor		$out0,$in0,$twk1+Lxts_dec6x_steal:+	vncipher	$out0,$out0,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out0,$out0,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		Lxts_dec6x_steal++	add		$inp,$inp,$taillen+	vncipher	$out0,$out0,v24++	cmpwi		$taillen,0+	vncipher	$out0,$out0,v25++	lvx_u		$in0,0,$inp+	vncipher	$out0,$out0,v26++	lvsr		$inpperm,0,$taillen	# $in5 is no more+	vncipher	$out0,$out0,v27++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vncipher	$out0,$out0,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]++	vncipher	$out0,$out0,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vxor		$twk1,$twk1,v31++	le?vperm	$in0,$in0,$in0,$leperm+	vncipher	$out0,$out0,v30++	vperm		$in0,$in0,$in0,$inpperm+	vncipherlast	$tmp,$out0,$twk1++	le?vperm	$out0,$tmp,$tmp,$leperm+	le?stvx_u	$out0,0,$out+	be?stvx_u	$tmp,0,$out++	vxor		$out0,$out0,$out0+	vspltisb	$out1,-1+	vperm		$out0,$out0,$out1,$inpperm+	vsel		$out0,$in0,$tmp,$out0+	vxor		$out0,$out0,$twk0++	subi		r30,$out,1+	mtctr		$taillen+Loop_xts_dec6x_steal:+	lbzu		r0,1(r30)+	stb		r0,16(r30)+	bdnz		Loop_xts_dec6x_steal++	li		$taillen,0+	mtctr		$rounds+	b		Loop_xts_dec1x		# one more time...++.align	4+Lxts_dec6x_done:+	${UCMP}i	$ivp,0+	beq		Lxts_dec6x_ret++	vxor		$tweak,$twk0,$rndkey0+	le?vperm	$tweak,$tweak,$tweak,$leperm+	stvx_u		$tweak,0,$ivp++Lxts_dec6x_ret:+	mtlr		r11+	li		r10,`$FRAME+15`+	li		r11,`$FRAME+31`+	stvx		$seven,r10,$sp		# wipe copies of round keys+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32+	stvx		$seven,r10,$sp+	addi		r10,r10,32+	stvx		$seven,r11,$sp+	addi		r11,r11,32++	mtspr		256,$vrsave+	lvx		v20,r10,$sp		# ABI says so+	addi		r10,r10,32+	lvx		v21,r11,$sp+	addi		r11,r11,32+	lvx		v22,r10,$sp+	addi		r10,r10,32+	lvx		v23,r11,$sp+	addi		r11,r11,32+	lvx		v24,r10,$sp+	addi		r10,r10,32+	lvx		v25,r11,$sp+	addi		r11,r11,32+	lvx		v26,r10,$sp+	addi		r10,r10,32+	lvx		v27,r11,$sp+	addi		r11,r11,32+	lvx		v28,r10,$sp+	addi		r10,r10,32+	lvx		v29,r11,$sp+	addi		r11,r11,32+	lvx		v30,r10,$sp+	lvx		v31,r11,$sp+	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)+	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)+	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)+	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)+	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)+	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)+	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`+	blr+	.long		0+	.byte		0,12,0x04,1,0x80,6,6,0+	.long		0++.align	5+_aesp8_xts_dec5x:+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24+	lvx		v24,$x20,$key_		# round[3]+	addi		$key_,$key_,0x20++	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	lvx		v25,$x10,$key_		# round[4]+	bdnz		_aesp8_xts_dec5x++	subi		r0,$taillen,1+	vncipher	$out0,$out0,v24+	vncipher	$out1,$out1,v24+	vncipher	$out2,$out2,v24+	vncipher	$out3,$out3,v24+	vncipher	$out4,$out4,v24++	andi.		r0,r0,16+	cmpwi		$taillen,0+	vncipher	$out0,$out0,v25+	vncipher	$out1,$out1,v25+	vncipher	$out2,$out2,v25+	vncipher	$out3,$out3,v25+	vncipher	$out4,$out4,v25+	 vxor		$twk0,$twk0,v31++	sub		$inp,$inp,r0+	vncipher	$out0,$out0,v26+	vncipher	$out1,$out1,v26+	vncipher	$out2,$out2,v26+	vncipher	$out3,$out3,v26+	vncipher	$out4,$out4,v26+	 vxor		$in1,$twk1,v31++	vncipher	$out0,$out0,v27+	lvx_u		$in0,0,$inp+	vncipher	$out1,$out1,v27+	vncipher	$out2,$out2,v27+	vncipher	$out3,$out3,v27+	vncipher	$out4,$out4,v27+	 vxor		$in2,$twk2,v31++	addi		$key_,$sp,$FRAME+15	# rewind $key_+	vncipher	$out0,$out0,v28+	vncipher	$out1,$out1,v28+	vncipher	$out2,$out2,v28+	vncipher	$out3,$out3,v28+	vncipher	$out4,$out4,v28+	lvx		v24,$x00,$key_		# re-pre-load round[1]+	 vxor		$in3,$twk3,v31++	vncipher	$out0,$out0,v29+	le?vperm	$in0,$in0,$in0,$leperm+	vncipher	$out1,$out1,v29+	vncipher	$out2,$out2,v29+	vncipher	$out3,$out3,v29+	vncipher	$out4,$out4,v29+	lvx		v25,$x10,$key_		# re-pre-load round[2]+	 vxor		$in4,$twk4,v31++	vncipher	$out0,$out0,v30+	vncipher	$out1,$out1,v30+	vncipher	$out2,$out2,v30+	vncipher	$out3,$out3,v30+	vncipher	$out4,$out4,v30++	vncipherlast	$out0,$out0,$twk0+	vncipherlast	$out1,$out1,$in1+	vncipherlast	$out2,$out2,$in2+	vncipherlast	$out3,$out3,$in3+	vncipherlast	$out4,$out4,$in4+	mtctr		$rounds+	blr+        .long   	0+        .byte   	0,12,0x14,0,0,0,0,0+___+}}	}}}++my $consts=1;+foreach(split("\n",$code)) {+        s/\`([^\`]*)\`/eval($1)/geo;++	# constants table endian-specific conversion+	if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {+	    my $conv=$3;+	    my @bytes=();++	    # convert to endian-agnostic format+	    if ($1 eq "long") {+	      foreach (split(/,\s*/,$2)) {+		my $l = /^0/?oct:int;+		push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;+	      }+	    } else {+		@bytes = map(/^0/?oct:int,split(/,\s*/,$2));+	    }++	    # little-endian conversion+	    if ($flavour =~ /le$/o) {+		SWITCH: for($conv)  {+		    /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };+		    /\?rev/ && do   { @bytes=reverse(@bytes);    last; };+		}+	    }++	    #emit+	    print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";+	    next;+	}+	$consts=0 if (m/Lconsts:/o);	# end of table++	# instructions prefixed with '?' are endian-specific and need+	# to be adjusted accordingly...+	if ($flavour =~ /le$/o) {	# little-endian+	    s/le\?//o		or+	    s/be\?/#be#/o	or+	    s/\?lvsr/lvsl/o	or+	    s/\?lvsl/lvsr/o	or+	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or+	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or+	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;+	} else {			# big-endian+	    s/le\?/#le#/o	or+	    s/be\?//o		or+	    s/\?([a-z]+)/$1/o;+	}++        print $_,"\n";+}++close STDOUT;