diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # CHANGELOG for crypton +## 2.2.4 + +**New module `Crypto.PubKey.RSA.Blinded`.** A `BlindedKey` carries RSA +blinding from one signature or decryption to the next. The existing +`*Safer` functions use it through a small process-wide table, and +verification keeps R^2 per modulus in another. + +* perf(rsa): keep the blinding between operations + [#335](https://github.com/kazu-yamamoto/crypton/pull/335) +* perf(rsa): a generated Montgomery multiplication for AArch64 + [#336](https://github.com/kazu-yamamoto/crypton/pull/336) +* fix: the C warnings CI reports + [#337](https://github.com/kazu-yamamoto/crypton/pull/337) +* perf(rsa): both CRT halves at once with AVX-512 IFMA + [#338](https://github.com/kazu-yamamoto/crypton/pull/338) +* perf(rsa): keep R^2 per modulus for verification + [#339](https://github.com/kazu-yamamoto/crypton/pull/339) + ## 2.2.3 **crypton builds on Windows again.** 2.2.2's tarball lacked a header that diff --git a/Crypto/Number/ModArithmetic.hs b/Crypto/Number/ModArithmetic.hs --- a/Crypto/Number/ModArithmetic.hs +++ b/Crypto/Number/ModArithmetic.hs @@ -49,6 +49,9 @@ import Foreign.C.Types (CInt (..)) import Foreign.Marshal.Alloc (allocaBytes) import Foreign.Ptr (Ptr, plusPtr) +import Data.IORef (IORef, atomicModifyIORef', newIORef) +import System.IO.Unsafe (unsafePerformIO) +import Crypto.Internal.ByteArray (Bytes, allocRet, withByteArray) -- | Raised when two numbers are supposed to be coprimes but are not. data CoprimesAssertionError = CoprimesAssertionError @@ -141,19 +144,6 @@ -- what GMP's own secure exponentiation lets slip !eLen = 8 * ((numBytes e + 7) `div` 8) --- safe, as the secure one below is: the call is tens of microseconds at RSA --- sizes, and the hundred nanoseconds this costs over an unsafe call is --- nothing beside holding a capability for that long in a server -foreign import ccall safe "crypton_powm_pub" - c_powm_pub - :: Ptr Word8 - -> Ptr Word8 - -> Word32 - -> Ptr Word8 - -> Word32 - -> Ptr Word8 - -> Word32 - -> IO CInt foreign import ccall safe "crypton_powm_sec" c_powm_sec @@ -203,31 +193,80 @@ -- optimisation and never the only path. The base has to be reduced and -- the exponent positive, which the guard above sees to. expPub :: Integer -> Integer -> Integer -> Maybe Integer -expPub b e m = unsafeDoIO $ - allocaBytes (sum widths) $ \start -> case scanl plusPtr start widths of - (out : base : expo : modu : _) -> do - _ <- Internal.i2ospOf b base mLen - _ <- Internal.i2ospOf e expo eLen - _ <- Internal.i2ospOf m modu mLen - r <- - c_powm_pub - out - base - (fromIntegral mLen) - expo - (fromIntegral eLen) - modu - (fromIntegral mLen) - if r == 0 - then do - !v <- Internal.os2ip out mLen - return (Just v) - else return Nothing - _ -> return Nothing +expPub b e m = unsafeDoIO $ do + known <- r2For m + case known of + Nothing -> return Nothing + Just r2 -> + allocaBytes (sum widths) $ \start -> case scanl plusPtr start widths of + (out : base : expo : modu : _) -> do + _ <- Internal.i2ospOf b base mLen + _ <- Internal.i2ospOf e expo eLen + _ <- Internal.i2ospOf m modu mLen + r <- withByteArray r2 $ \pr2 -> + c_powm_pub_r2 + out + base + (fromIntegral mLen) + expo + (fromIntegral eLen) + modu + (fromIntegral mLen) + pr2 + if r == 0 + then do + !v <- Internal.os2ip out mLen + return (Just v) + else return Nothing + _ -> return Nothing where !mLen = numBytes m !eLen = numBytes e widths = [mLen, mLen, eLen, mLen] + +-- | R^2 mod m for the public path, kept for the last few moduli, since a +-- program verifies under the same keys again and again. 'Nothing' is kept +-- too: it means the C declines this modulus, which is then not asked again. +r2For :: Integer -> IO (Maybe Bytes) +r2For m = do + found <- atomicModifyIORef' r2Table $ \ks -> case break ((== m) . fst) ks of + (before, hit : after) -> (hit : before ++ after, Just (snd hit)) + _ -> (ks, Nothing) + case found of + Just r2 -> return r2 + Nothing -> do + let mLen = numBytes m + r2 <- allocaBytes mLen $ \modu -> do + _ <- Internal.i2ospOf m modu mLen + (r, out) <- allocRet mLen $ \pout -> c_powm_r2 pout modu (fromIntegral mLen) + return (if r == 0 then Just out else Nothing) + atomicModifyIORef' r2Table $ \ks -> (take r2TableSize ((m, r2) : ks), ()) + return r2 + +r2TableSize :: Int +r2TableSize = 8 + +r2Table :: IORef [(Integer, Maybe Bytes)] +r2Table = unsafePerformIO (newIORef []) +{-# NOINLINE r2Table #-} + +-- safe, as the secure one below is: the call is tens of microseconds at RSA +-- sizes, and the hundred nanoseconds this costs over an unsafe call is +-- nothing beside holding a capability for that long in a server +foreign import ccall safe "crypton_powm_pub_r2" + c_powm_pub_r2 + :: Ptr Word8 + -> Ptr Word8 + -> Word32 + -> Ptr Word8 + -> Word32 + -> Ptr Word8 + -> Word32 + -> Ptr Word8 + -> IO CInt + +foreign import ccall unsafe "crypton_powm_r2" + c_powm_r2 :: Ptr Word8 -> Ptr Word8 -> Word32 -> IO CInt -- | @exponentiation@ computes modular exponentiation as /b^e mod m/ -- using repetitive squaring. diff --git a/Crypto/PubKey/RSA/Blinded.hs b/Crypto/PubKey/RSA/Blinded.hs new file mode 100644 --- /dev/null +++ b/Crypto/PubKey/RSA/Blinded.hs @@ -0,0 +1,29 @@ +-- | +-- Module : Crypto.PubKey.RSA.Blinded +-- License : BSD-style +-- Stability : experimental +-- Portability : Good +-- +-- An RSA private key that keeps its blinding between operations. +-- +-- Blinding hides the private-key operation's input from timing: it is +-- multiplied by @r^e@ for a random @r@ first, and the result by @r^-1@ +-- after. Drawing a new @r@ costs a modular inverse and an exponentiation, +-- about a tenth of an RSA-2048 signature. A t'BlindedKey' draws one and +-- squares the pair for each use after it, which is what OpenSSL does, and +-- draws a new one every 32 uses, from the 'Crypto.Random.Types.MonadRandom' +-- the operation runs in. +-- +-- The key is threaded through the operations as a hash context is: each +-- function in "Crypto.PubKey.RSA.PKCS15", "Crypto.PubKey.RSA.PSS" and +-- "Crypto.PubKey.RSA.OAEP" whose name ends in @Blinded@ returns the key for +-- the next one. The @Safer@ functions there keep a t'BlindedKey' per key +-- themselves, for the last few keys used, so they get the same saving +-- without one being passed. +module Crypto.PubKey.RSA.Blinded ( + BlindedKey, + blindedKey, + blindedPrivateKey, +) where + +import Crypto.PubKey.RSA.Blinded.Internal diff --git a/Crypto/PubKey/RSA/Blinded/Internal.hs b/Crypto/PubKey/RSA/Blinded/Internal.hs new file mode 100644 --- /dev/null +++ b/Crypto/PubKey/RSA/Blinded/Internal.hs @@ -0,0 +1,136 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# OPTIONS_GHC -fno-cse -fno-full-laziness #-} + +-- | +-- Module : Crypto.PubKey.RSA.Blinded.Internal +-- License : BSD-style +-- Stability : experimental +-- Portability : Good +-- +-- A private key with a blinding pair that is reused by squaring, as OpenSSL +-- does, rather than generated afresh for every operation. The key is a value +-- threaded through the operations, as a hash 'Crypto.Hash.Context' is. +module Crypto.PubKey.RSA.Blinded.Internal ( + BlindedKey (..), + blindedKey, + dpBlinded, + Draw, + drawFor, + withBlindingFor, + dpWithPair, +) where + +import Data.IORef +import System.IO.Unsafe (unsafePerformIO) + +import Crypto.Internal.ByteArray (ByteArray, ScrubbedBytes) +import Crypto.Number.Generate (generateMax) +import Crypto.Number.ModArithmetic (expFast, inverseCoprimes) +import Crypto.Number.Serialize (i2ospOf_, os2ip) +import Crypto.PubKey.RSA.Prim (dp) +import Crypto.PubKey.RSA.Types +import Crypto.Random.Types + +-- | A private key together with the blinding its next operation uses. +-- +-- Each operation multiplies the input by @r^e@ and the output by @r^-1@, and +-- returns the key with both squared for the next one, so a pair costs two +-- multiplications after the first. A new @r@ is drawn every 32 operations. +-- Use the key an operation returns: handing the same one to two operations +-- gives both the same pair. +data BlindedKey = BlindedKey + { blindedPrivateKey :: PrivateKey + -- ^ the key it was made from + , blindedPair :: Pair + } + +-- | @r^e@ and @r^-1@ modulo n, and how many more uses they have after this. +data Pair = Pair !Int !Integer !Integer | Spent + +-- | Uses of one @r@ before another is drawn, OpenSSL's number. +refresh :: Int +refresh = 32 + +-- | A t'BlindedKey' for a private key. The first pair is drawn when it is +-- first used. +blindedKey :: PrivateKey -> BlindedKey +blindedKey pk = BlindedKey pk Spent + +-- | The random numbers a new pair is made from: drawing them is cheap, and +-- making the pair, an inverse and an exponentiation, is done only if needed. +data Draw = Draw !Integer !Integer + +-- | Draw for a key, as 'Crypto.PubKey.RSA.generateBlinder' does. +drawFor :: forall m. MonadRandom m => PrivateKey -> m Draw +drawFor pk = do + r <- generateMax (private_n pk) + u <- os2ip <$> (getRandomBytes 16 :: m ScrubbedBytes) + return (Draw r u) + +-- | @r^e@ and @r^-1@. The inverse is taken of @r*u@ and multiplied by @u@, +-- so the extended Euclidean algorithm never sees @r@, as in +-- 'Crypto.PubKey.RSA.generateBlinder'. +makePair :: PrivateKey -> Draw -> (Integer, Integer) +makePair pk (Draw r u) = + (expFast r (private_e pk) n, (inverseCoprimes ((r * u) `mod` n) n * u) `mod` n) + where + n = private_n pk + +-- | The pair for this operation, and the key holding the next one. The +-- draw is used only when a new pair is due. +takePair :: Draw -> BlindedKey -> ((Integer, Integer), BlindedKey) +takePair d (BlindedKey pk pair) = case pair of + Pair k a ai -> ((a, ai), BlindedKey pk (next k a ai)) + Spent -> case makePair pk d of + (a, ai) -> ((a, ai), BlindedKey pk (next refresh a ai)) + where + n = private_n pk + next k a ai + | k > 1 = Pair (k - 1) ((a * a) `mod` n) ((ai * ai) `mod` n) + | otherwise = Spent + +-- | The decryption primitive, blinded with a given pair. +dpWithPair :: ByteArray ba => (Integer, Integer) -> PrivateKey -> ba -> ba +dpWithPair (a, ai) pk c = i2ospOf_ k ((ai * os2ip m) `mod` n) + where + n = private_n pk + k = private_size pk + m = dp Nothing pk (i2ospOf_ k ((a * os2ip c) `mod` n) `asTypeOf` c) + +-- | The decryption primitive, blinded with the key's pair, and the key for +-- the next operation. Randomness is drawn only when a new pair is due. +dpBlinded + :: (MonadRandom m, ByteArray ba) => BlindedKey -> ba -> m (ba, BlindedKey) +dpBlinded bk c = do + d <- case blindedPair bk of + Pair {} -> return (Draw 0 0) + Spent -> drawFor (blindedPrivateKey bk) + let (pair, bk') = takePair d bk + return (dpWithPair pair (blindedPrivateKey bk) c, bk') + +-- | Run an operation with the pair kept for a key, for the functions that +-- take a bare t'PrivateKey'. One t'BlindedKey' is kept per public key, for +-- the last few keys used; taking its pair and leaving the next is atomic, so +-- no two operations get the same pair. The caller draws every time, since +-- whether a new pair is due is known only in here. +withBlindingFor :: PrivateKey -> Draw -> ((Integer, Integer) -> a) -> a +withBlindingFor pk d f = unsafePerformIO $ do + pair <- atomicModifyIORef' table step + return (f pair) + where + pub = private_pub pk + step ks = case break ((== pub) . private_pub . blindedPrivateKey) ks of + (before, bk : after) -> found bk (before ++ after) + _ -> found (blindedKey pk) ks + found bk rest = case takePair d bk of + (pair@(!_, !_), bk') -> (take tableSize (bk' : rest), pair) +{-# NOINLINE withBlindingFor #-} + +-- | How many keys 'withBlindingFor' remembers. +tableSize :: Int +tableSize = 4 + +table :: IORef [BlindedKey] +table = unsafePerformIO (newIORef []) +{-# NOINLINE table #-} diff --git a/Crypto/PubKey/RSA/OAEP.hs b/Crypto/PubKey/RSA/OAEP.hs --- a/Crypto/PubKey/RSA/OAEP.hs +++ b/Crypto/PubKey/RSA/OAEP.hs @@ -18,13 +18,14 @@ -- * OAEP decryption decrypt, decryptSafer, + decryptBlinded, ) where import Crypto.Hash import Crypto.Number.Serialize (os2ip) import Crypto.PubKey.Internal (and') import Crypto.PubKey.MaskGenFunction -import Crypto.PubKey.RSA (generateBlinder) +import Crypto.PubKey.RSA.Blinded.Internal import Crypto.PubKey.RSA.Prim import Crypto.PubKey.RSA.Types import Crypto.Random.Types @@ -212,7 +213,8 @@ k = private_size pk hashLen = hashDigestSize (oaepHash oaep) --- | Decrypt a ciphertext using OAEP and by automatically generating a blinder. +-- | Decrypt a ciphertext using OAEP, blinded with the pair kept for this key; +-- see "Crypto.PubKey.RSA.Blinded". decryptSafer :: (HashAlgorithm hash, MonadRandom m) => OAEPParams hash ByteString ByteString @@ -222,6 +224,34 @@ -> ByteString -- ^ Cipher text -> m (Either Error ByteString) -decryptSafer oaep pk cipher = do - blinder <- generateBlinder (private_n pk) - return (decrypt (Just blinder) oaep pk cipher) +decryptSafer oaep pk cipher + | B.length cipher /= k = return (Left MessageSizeIncorrect) + | os2ip cipher >= private_n pk = return (Left MessageSizeIncorrect) + | k < 2 * hashLen + 2 = return (Left InvalidParameters) + | otherwise = do + d <- drawFor pk + return $ withBlindingFor pk d $ \pair -> unpad oaep k (dpWithPair pair pk cipher) + where + k = private_size pk + hashLen = hashDigestSize (oaepHash oaep) + +-- | 'decrypt' with the blinding a t'BlindedKey' keeps, and the key for the +-- next operation. +decryptBlinded + :: (HashAlgorithm hash, MonadRandom m) + => OAEPParams hash ByteString ByteString + -> BlindedKey + -> ByteString + -- ^ Cipher text + -> m (Either Error ByteString, BlindedKey) +decryptBlinded oaep bk cipher + | B.length cipher /= k = return (Left MessageSizeIncorrect, bk) + | os2ip cipher >= private_n pk = return (Left MessageSizeIncorrect, bk) + | k < 2 * hashLen + 2 = return (Left InvalidParameters, bk) + | otherwise = do + (m, bk') <- dpBlinded bk cipher + return (unpad oaep k m, bk') + where + pk = blindedPrivateKey bk + k = private_size pk + hashLen = hashDigestSize (oaepHash oaep) diff --git a/Crypto/PubKey/RSA/PKCS15.hs b/Crypto/PubKey/RSA/PKCS15.hs --- a/Crypto/PubKey/RSA/PKCS15.hs +++ b/Crypto/PubKey/RSA/PKCS15.hs @@ -20,6 +20,11 @@ signDigestInfo, signSaferDigestInfo, + -- * Private key operations with a kept blinding + decryptBlinded, + signDigestBlinded, + signDigestInfoBlinded, + -- * Public key operations encrypt, verify, @@ -33,7 +38,7 @@ import Crypto.Hash import Crypto.Number.Serialize (os2ip) import Crypto.PubKey.Internal (and') -import Crypto.PubKey.RSA (generateBlinder) +import Crypto.PubKey.RSA.Blinded.Internal import Crypto.PubKey.RSA.Prim import Crypto.PubKey.RSA.Types import Crypto.Random.Types @@ -486,7 +491,8 @@ -- "convert" must be apply to "c". | otherwise = unpad $ dp blinder pk $ B.convert c --- | decrypt message using the private key and by automatically generating a blinder. +-- | decrypt message using the private key, blinded with the pair kept for +-- this key; see "Crypto.PubKey.RSA.Blinded". decryptSafer :: (MonadRandom m, ByteArray ba) => PrivateKey @@ -494,10 +500,30 @@ -> ByteString -- ^ cipher text -> m (Either Error ba) -decryptSafer pk b = do - blinder <- generateBlinder (private_n pk) - return (decrypt (Just blinder) pk b) +decryptSafer pk b + | B.length b /= private_size pk = return (Left MessageSizeIncorrect) + | os2ip b >= private_n pk = return (Left MessageSizeIncorrect) + | otherwise = do + d <- drawFor pk + return $ withBlindingFor pk d $ \pair -> unpad (dpWithPair pair pk (B.convert b)) +-- | 'decrypt' with the blinding a t'BlindedKey' keeps, and the key for the +-- next operation. +decryptBlinded + :: (MonadRandom m, ByteArray ba) + => BlindedKey + -> ByteString + -- ^ cipher text + -> m (Either Error ba, BlindedKey) +decryptBlinded bk c + | B.length c /= private_size pk = return (Left MessageSizeIncorrect, bk) + | os2ip c >= private_n pk = return (Left MessageSizeIncorrect, bk) + | otherwise = do + (m, bk') <- dpBlinded bk (B.convert c) + return (unpad m, bk') + where + pk = blindedPrivateKey bk + -- | encrypt a bytestring using the public key. -- -- The message needs to be smaller than the key size - 11. @@ -538,7 +564,8 @@ "Use signDigest, or signDigestInfo when the message is already a DigestInfo" #-} --- | sign message using the private key and by automatically generating a blinder. +-- | sign message using the private key, blinded with the pair kept for this +-- key; see "Crypto.PubKey.RSA.Blinded". -- -- __Deprecated.__ Use 'signSaferDigest', or 'signSaferDigestInfo'. signSafer @@ -551,8 +578,8 @@ -- ^ message to sign -> m (Either Error ByteString) signSafer hashAlg pk m = do - blinder <- generateBlinder (private_n pk) - return (sign (Just blinder) hashAlg pk m) + d <- drawFor pk + return $ signWithBlindingFor pk d (makeSignature hashAlg (private_size pk) m) {-# DEPRECATED signSafer "Use signSaferDigest, or signSaferDigestInfo when the message is already a DigestInfo" @@ -632,8 +659,8 @@ signDigest blinder pk digest = dp blinder pk `fmap` padSignature (private_size pk) (hashDigestASN1 digest) --- | 'signDigest' with a blinder generated for the occasion, as 'signSafer' --- is to 'sign'. +-- | 'signDigest' blinded with the pair kept for this key, as 'signSafer' is +-- to 'sign'; see "Crypto.PubKey.RSA.Blinded". signSaferDigest :: (HashAlgorithmASN1 hashAlg, MonadRandom m) => PrivateKey @@ -642,9 +669,22 @@ -- ^ digest of the message to sign -> m (Either Error ByteString) signSaferDigest pk digest = do - blinder <- generateBlinder (private_n pk) - return (signDigest (Just blinder) pk digest) + d <- drawFor pk + return $ + signWithBlindingFor pk d (padSignature (private_size pk) (hashDigestASN1 digest)) +-- | 'signDigest' with the blinding a t'BlindedKey' keeps, and the key for the +-- next operation. +signDigestBlinded + :: (HashAlgorithmASN1 hashAlg, MonadRandom m) + => BlindedKey + -> Digest hashAlg + -- ^ digest of the message to sign + -> m (Either Error ByteString, BlindedKey) +signDigestBlinded bk digest = + signBlindedEncoded bk $ + padSignature (private_size (blindedPrivateKey bk)) (hashDigestASN1 digest) + -- | Sign something that is already a DigestInfo, the ASN.1 structure -- naming a hash algorithm and carrying a digest under it. -- @@ -666,7 +706,7 @@ signDigestInfo blinder pk di = dp blinder pk `fmap` padSignature (private_size pk) di --- | 'signDigestInfo' with a blinder generated for the occasion. +-- | 'signDigestInfo' blinded with the pair kept for this key. signSaferDigestInfo :: MonadRandom m => PrivateKey @@ -675,8 +715,37 @@ -- ^ a DigestInfo, encoded -> m (Either Error ByteString) signSaferDigestInfo pk di = do - blinder <- generateBlinder (private_n pk) - return (signDigestInfo (Just blinder) pk di) + d <- drawFor pk + return $ signWithBlindingFor pk d (padSignature (private_size pk) di) + +-- | 'signDigestInfo' with the blinding a t'BlindedKey' keeps, and the key for +-- the next operation. +signDigestInfoBlinded + :: MonadRandom m + => BlindedKey + -> ByteString + -- ^ a DigestInfo, encoded + -> m (Either Error ByteString, BlindedKey) +signDigestInfoBlinded bk di = + signBlindedEncoded bk $ padSignature (private_size (blindedPrivateKey bk)) di + +-- | The private-key operation on an encoded signature, if there is one; the +-- key is used up only if there is. +signBlindedEncoded + :: MonadRandom m + => BlindedKey + -> Either Error ByteString + -> m (Either Error ByteString, BlindedKey) +signBlindedEncoded bk (Left err) = return (Left err, bk) +signBlindedEncoded bk (Right em) = do + (s, bk') <- dpBlinded bk em + return (Right s, bk') + +-- | The same, with the pair the @Safer@ functions keep for this key. +signWithBlindingFor + :: PrivateKey -> Draw -> Either Error ByteString -> Either Error ByteString +signWithBlindingFor _ _ (Left err) = Left err +signWithBlindingFor pk d (Right em) = Right (withBlindingFor pk d (\pair -> dpWithPair pair pk em)) -- | Verify a signature over a digest. The checks are 'verify's. verifyDigest diff --git a/Crypto/PubKey/RSA/PSS.hs b/Crypto/PubKey/RSA/PSS.hs --- a/Crypto/PubKey/RSA/PSS.hs +++ b/Crypto/PubKey/RSA/PSS.hs @@ -16,6 +16,8 @@ signDigest, signSafer, signDigestSafer, + signBlinded, + signDigestBlinded, verify, verifyDigest, ) where @@ -24,7 +26,7 @@ import Crypto.Number.Basic (numBits) import Crypto.Number.Serialize (os2ip) import Crypto.PubKey.MaskGenFunction -import Crypto.PubKey.RSA (generateBlinder) +import Crypto.PubKey.RSA.Blinded.Internal import Crypto.PubKey.RSA.Prim import Crypto.PubKey.RSA.Types import Crypto.Random.Types @@ -85,9 +87,20 @@ -> Digest hash -- ^ Message digest -> Either Error ByteString -signDigestWithSalt salt blinder params pk digest +signDigestWithSalt salt blinder params pk digest = + dp blinder pk <$> encodeDigest salt params pk digest + +-- | The encoded message the private-key operation is applied to. +encodeDigest + :: HashAlgorithm hash + => ByteString + -> PSSParams hash ByteString ByteString + -> PrivateKey + -> Digest hash + -> Either Error ByteString +encodeDigest salt params pk digest | emLen < hashLen + saltLen + 2 = Left InvalidParameters - | otherwise = Right $ dp blinder pk em + | otherwise = Right em where k = private_size pk emLen = if emTruncate pubBits then k - 1 else k @@ -164,7 +177,8 @@ salt <- getRandomBytes (pssSaltLength params) return (signDigestWithSalt salt blinder params pk digest) --- | Sign using the PSS Parameters and an automatically generated blinder. +-- | Sign using the PSS Parameters, blinded with the pair kept for this key; +-- see "Crypto.PubKey.RSA.Blinded". signSafer :: (HashAlgorithm hash, MonadRandom m) => PSSParams hash ByteString ByteString @@ -174,11 +188,10 @@ -> ByteString -- ^ message to sign -> m (Either Error ByteString) -signSafer params pk m = do - blinder <- generateBlinder (private_n pk) - sign (Just blinder) params pk m +signSafer params pk m = signDigestSafer params pk (hashWith (pssHash params) m) --- | Sign using the PSS Parameters and an automatically generated blinder. +-- | Sign using the PSS Parameters, blinded with the pair kept for this key; +-- see "Crypto.PubKey.RSA.Blinded". signDigestSafer :: (HashAlgorithm hash, MonadRandom m) => PSSParams hash ByteString ByteString @@ -189,8 +202,39 @@ -- ^ message digst -> m (Either Error ByteString) signDigestSafer params pk digest = do - blinder <- generateBlinder (private_n pk) - signDigest (Just blinder) params pk digest + salt <- getRandomBytes (pssSaltLength params) + d <- drawFor pk + return $ case encodeDigest salt params pk digest of + Left err -> Left err + Right em -> Right (withBlindingFor pk d (\pair -> dpWithPair pair pk em)) + +-- | 'signSafer' with the blinding a t'BlindedKey' keeps, and the key for the +-- next operation. +signBlinded + :: (HashAlgorithm hash, MonadRandom m) + => PSSParams hash ByteString ByteString + -> BlindedKey + -> ByteString + -- ^ message to sign + -> m (Either Error ByteString, BlindedKey) +signBlinded params bk m = signDigestBlinded params bk (hashWith (pssHash params) m) + +-- | 'signDigestSafer' with the blinding a t'BlindedKey' keeps, and the key +-- for the next operation. +signDigestBlinded + :: (HashAlgorithm hash, MonadRandom m) + => PSSParams hash ByteString ByteString + -> BlindedKey + -> Digest hash + -- ^ message digest + -> m (Either Error ByteString, BlindedKey) +signDigestBlinded params bk digest = do + salt <- getRandomBytes (pssSaltLength params) + case encodeDigest salt params (blindedPrivateKey bk) digest of + Left err -> return (Left err, bk) + Right em -> do + (s, bk') <- dpBlinded bk em + return (Right s, bk') -- | Verify a signature using the PSS Parameters verify diff --git a/Crypto/PubKey/RSA/Prim.hs b/Crypto/PubKey/RSA/Prim.hs --- a/Crypto/PubKey/RSA/Prim.hs +++ b/Crypto/PubKey/RSA/Prim.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE BangPatterns #-} + -- | -- Module : Crypto.PubKey.RSA.Prim -- License : BSD-style @@ -12,9 +14,18 @@ ep, ) where +import Data.Memory.PtrMethods (memSet) +import Data.Word (Word32, Word8) +import Foreign.C.Types (CInt (..)) +import Foreign.Marshal.Alloc (allocaBytes) +import Foreign.Ptr (Ptr, plusPtr) + import Crypto.Internal.ByteArray (ByteArray) +import Crypto.Internal.Compat (unsafeDoIO) +import Crypto.Number.Basic (numBytes) import Crypto.Number.ModArithmetic (expFast, expSafe) import Crypto.Number.Serialize (i2ospOf_, os2ip) +import qualified Crypto.Number.Serialize.Internal as Internal import Crypto.PubKey.RSA.Types {- dpSlow computes the decrypted message not using any precomputed cache value. @@ -33,17 +44,62 @@ where re = expFast r (public_e $ private_pub pk) (private_n pk) iC = multiplication re (os2ip c) (private_n pk) - m1 = expSafe iC (private_dP pk) (private_p pk) - m2 = expSafe iC (private_dQ pk) (private_q pk) + (m1, m2) = crtHalves iC pk h = ((private_qinv pk) * (m1 - m2)) `mod` (private_p pk) dpFastNoBlinder :: ByteArray ba => PrivateKey -> ba -> ba dpFastNoBlinder pk c = i2ospOf_ (private_size pk) (m2 + h * (private_q pk)) where iC = os2ip c - m1 = expSafe iC (private_dP pk) (private_p pk) - m2 = expSafe iC (private_dQ pk) (private_q pk) + (m1, m2) = crtHalves iC pk h = ((private_qinv pk) * (m1 - m2)) `mod` (private_p pk) + +-- | c^dP mod p and c^dQ mod q, the halves of the private-key operation: both +-- at once where the C can (AVX-512 IFMA, and the sizes of RSA-2048, -3072 and +-- -4096), one after the other where it declines. +crtHalves :: Integer -> PrivateKey -> (Integer, Integer) +crtHalves c pk = case expSafeX2 (c `mod` p) dP p (c `mod` q) dQ q of + Just r -> r + Nothing -> (expSafe c dP p, expSafe c dQ q) + where + p = private_p pk + q = private_q pk + dP = private_dP pk + dQ = private_dQ pk + +-- | Two secure exponentiations in one call. The bases are reduced already +-- and the moduli are the same length; both exponents go in at that length, +-- so the length the C sees is the modulus', as 'expSafe' pads to whole words. +expSafeX2 :: Integer -> Integer -> Integer -> Integer -> Integer -> Integer -> Maybe (Integer, Integer) +expSafeX2 b1 e1 m1 b2 e2 m2 + | numBytes m2 /= len || not (odd m1 && odd m2) = Nothing + | e1 < 0 || e2 < 0 || numBytes e1 > len || numBytes e2 > len = Nothing + | otherwise = unsafeDoIO $ + allocaBytes (8 * len) $ \start -> case take 8 (iterate (`plusPtr` len) start) of + [o1, pb1, pe1, pm1, o2, pb2, pe2, pm2] -> do + mapM_ + (\(v, ptr) -> Internal.i2ospOf v ptr len) + [(b1, pb1), (e1, pe1), (m1, pm1), (b2, pb2), (e2, pe2), (m2, pm2)] + r <- c_powm_sec_x2 o1 pb1 l pe1 l pm1 l o2 pb2 l pe2 l pm2 l + -- the exponents are the caller's secrets + memSet pe1 0 len + memSet pe2 0 len + if r /= 0 + then return Nothing + else do + !v1 <- Internal.os2ip o1 len + !v2 <- Internal.os2ip o2 len + return (Just (v1, v2)) + _ -> return Nothing + where + !len = numBytes m1 + l = fromIntegral len :: Word32 + +foreign import ccall safe "crypton_powm_sec_x2" + c_powm_sec_x2 + :: Ptr Word8 -> Ptr Word8 -> Word32 -> Ptr Word8 -> Word32 -> Ptr Word8 -> Word32 + -> Ptr Word8 -> Ptr Word8 -> Word32 -> Ptr Word8 -> Word32 -> Ptr Word8 -> Word32 + -> IO CInt -- | Compute the RSA decrypt primitive. -- if the p and q numbers are available, then dpFast is used diff --git a/Crypto/System/CPU.hs b/Crypto/System/CPU.hs --- a/Crypto/System/CPU.hs +++ b/Crypto/System/CPU.hs @@ -26,6 +26,7 @@ ADX, VAES, VAES512, + AVX512IFMA, -- AArch64 NEON, ARMAES, @@ -112,6 +113,11 @@ pattern VAES512 :: ProcessorOption pattern VAES512 = ProcessorOption 10 +-- | The AVX-512 52-bit integer multiply-adds, with AVX-512 F and VL, and the +-- operating system saving their state: RSA's private-key operation uses them. +pattern AVX512IFMA :: ProcessorOption +pattern AVX512IFMA = ProcessorOption 19 + -- | Advanced SIMD, which is not optional on AArch64. pattern NEON :: ProcessorOption pattern NEON = ProcessorOption 11 @@ -161,6 +167,7 @@ show ADX = "ADX" show VAES = "VAES" show VAES512 = "VAES512" + show AVX512IFMA = "AVX512IFMA" show NEON = "NEON" show ARMAES = "ARMAES" show ARMPMULL = "ARMPMULL" @@ -193,7 +200,7 @@ -- 2 is not asked for and has no name: it was RDRAND, which crypton no -- longer dispatches on. The number is left out rather than reused, so -- that the others keep the values they had. - allOptions = [ProcessorOption n | n <- [0 .. 18], n /= 2] + allOptions = [ProcessorOption n | n <- [0 .. 19], n /= 2] askC (ProcessorOption n) = (/= 0) <$> crypton_cpu_option (fromIntegral n) {-# NOINLINE processorOptions #-} diff --git a/cbits/crypton_bitfn.h b/cbits/crypton_bitfn.h --- a/cbits/crypton_bitfn.h +++ b/cbits/crypton_bitfn.h @@ -127,23 +127,6 @@ } #endif -#ifndef ARCH_HAS_MEMORY_ZERO -static inline void memory_zero(void *ptr, uint32_t len) -{ - uint32_t *ptr32 = ptr; - uint8_t *ptr8; - int i; - - for (i = 0; i < len / 4; i++) - *ptr32++ = 0; - if (len % 4) { - ptr8 = (uint8_t *) ptr32; - for (i = len % 4; i >= 0; i--) - ptr8[i] = 0; - } -} -#endif - #ifndef ARCH_HAS_ARRAY_COPY32 static inline void array_copy32(uint32_t *d, uint32_t *s, uint32_t nb) { diff --git a/cbits/crypton_chacha.c b/cbits/crypton_chacha.c --- a/cbits/crypton_chacha.c +++ b/cbits/crypton_chacha.c @@ -109,8 +109,9 @@ #define ALIGNED64(PTR) \ (((uintptr_t)(const void *)(PTR)) % 8 == 0) -static const uint8_t sigma[16] = "expand 32-byte k"; -static const uint8_t tau[16] = "expand 16-byte k"; +/* sixteen bytes are read; the seventeenth is the string's terminator */ +static const uint8_t sigma[17] = "expand 32-byte k"; +static const uint8_t tau[17] = "expand 16-byte k"; static void chacha_core(int rounds, crypton_chacha_block *out, const crypton_chacha_state *in) { @@ -384,7 +385,7 @@ } /* xor as much as needed */ - for (i = 0; i < bytes; i++) + for (i = 0; i < (int) bytes; i++) dst[i] = src[i] ^ out.b[i]; /* copy the left over in the buffer */ @@ -510,7 +511,7 @@ } /* xor as much as needed */ - for (i = 0; i < bytes; i++) + for (i = 0; i < (int) bytes; i++) dst[i] = out.b[i]; /* copy the left over in the buffer */ diff --git a/cbits/crypton_cpu.c b/cbits/crypton_cpu.c --- a/cbits/crypton_cpu.c +++ b/cbits/crypton_cpu.c @@ -512,6 +512,14 @@ * AVX-512 at all and never reaches here. */ && !(amd && family == 0x19)) f |= CRYPTON_X86_VAES512; + /* IFMA (bit 21) with F (16) and VL (31), and the same + * AVX-512 state. Zen 4 is not left out here: RSA's + * multiplication uses the 256-bit form. */ + if ((f & CRYPTON_X86_AVX2) + && (ebx & (1 << 16)) && (ebx & (1 << 21)) + && (ebx & (1u << 31)) + && ((xcr0() & 0xe6) == 0xe6)) + f |= CRYPTON_X86_IFMA; } #ifdef CRYPTON_CPU_TESTING_ADX f |= CRYPTON_X86_ADX; @@ -602,6 +610,7 @@ #define OPT_ARMSHA512 16 #define OPT_PPCAES 17 #define OPT_PPCVPMSUM 18 +#define OPT_IFMA 19 int crypton_cpu_option(unsigned int option) { @@ -619,6 +628,7 @@ case OPT_ADX: return (f & CRYPTON_X86_ADX) != 0; case OPT_VAES: return (f & CRYPTON_X86_VAES) != 0; case OPT_VAES512: return (f & CRYPTON_X86_VAES512) != 0; + case OPT_IFMA: return (f & CRYPTON_X86_IFMA) != 0; default: return 0; } #else diff --git a/cbits/crypton_cpu.h b/cbits/crypton_cpu.h --- a/cbits/crypton_cpu.h +++ b/cbits/crypton_cpu.h @@ -65,6 +65,9 @@ * bits of XCR0 than AVX2 wants: the mask registers and the two upper halves * of the vector registers. */ #define CRYPTON_X86_VAES512 256 +/* AVX-512 IFMA, the 52-bit multiply-adds, with F and VL for the 256-bit form + * RSA uses them in: EVEX-encoded, so they want the AVX-512 state saved too. */ +#define CRYPTON_X86_IFMA 512 #ifdef ARCH_X86 uint32_t crypton_x86_simd_features(void); #endif diff --git a/cbits/crypton_mont.h b/cbits/crypton_mont.h new file mode 100644 --- /dev/null +++ b/cbits/crypton_mont.h @@ -0,0 +1,9015 @@ +/* Generated by cbits/crypton_mont.py; do not edit. + * + * Montgomery multiplication and squaring for AArch64 at 16, 24 and 32 limbs, + * the halves of RSA-2048, -3072 and -4096. Each answers what mont_mul and + * mont_sqr answer. */ +#ifndef CRYPTON_MONT_H +#define CRYPTON_MONT_H + +#include "crypton_bignum.h" + +/* (t2:t1:t0) += x * y */ +#define MONT_MAC(t0, t1, t2, x, y) \ + do { \ + limb_t lo_, hi_; \ + __asm__("mul\t%0, %5, %6\n\t" \ + "umulh\t%1, %5, %6\n\t" \ + "adds\t%2, %2, %0\n\t" \ + "adcs\t%3, %3, %1\n\t" \ + "adc\t%4, %4, xzr" \ + : "=&r"(lo_), "=&r"(hi_), "+r"(t0), "+r"(t1), "+r"(t2) \ + : "r"(x), "r"(y) \ + : "cc"); \ + } while (0) + +/* (s2:s1:s0) = x * y */ +#define MONT_MUL(s0, s1, s2, x, y) \ + __asm__("mul\t%0, %3, %4\n\t" \ + "umulh\t%1, %3, %4\n\t" \ + "mov\t%2, xzr" \ + : "=&r"(s0), "=&r"(s1), "=r"(s2) \ + : "r"(x), "r"(y)) + +/* (t2:t1:t0) += (s2:s1:s0) */ +#define MONT_ADD(t0, t1, t2, s0, s1, s2) \ + __asm__("adds\t%0, %0, %3\n\t" \ + "adcs\t%1, %1, %4\n\t" \ + "adc\t%2, %2, %5" \ + : "+r"(t0), "+r"(t1), "+r"(t2) \ + : "r"(s0), "r"(s1), "r"(s2) \ + : "cc") + +/* (s2:s1:s0) *= 2 */ +#define MONT_DBL(s0, s1, s2) \ + do { \ + s2 = (s2 << 1) | (s1 >> 63); \ + s1 = (s1 << 1) | (s0 >> 63); \ + s0 <<= 1; \ + } while (0) + +/* r = a * b * R^-1 mod m, for a and b under m; t is 2n limbs of scratch */ +static void crypton_mont_mul16(limb_t *r, const limb_t *a, const limb_t *b, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 16; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], b[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MAC(t0, t1, t2, a[0], b[1]); + MONT_MUL(v0, v1, v2, a[1], b[0]); + MONT_MUL(w0, w1, w2, u[0], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MAC(t0, t1, t2, a[0], b[2]); + MONT_MUL(v0, v1, v2, a[1], b[1]); + MONT_MUL(w0, w1, w2, a[2], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[2]); + MONT_MAC(v0, v1, v2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MAC(t0, t1, t2, a[0], b[3]); + MONT_MUL(v0, v1, v2, a[1], b[2]); + MONT_MUL(w0, w1, w2, a[2], b[1]); + MONT_MAC(t0, t1, t2, a[3], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[3]); + MONT_MAC(w0, w1, w2, u[1], m[2]); + MONT_MAC(t0, t1, t2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MAC(t0, t1, t2, a[0], b[4]); + MONT_MUL(v0, v1, v2, a[1], b[3]); + MONT_MUL(w0, w1, w2, a[2], b[2]); + MONT_MAC(t0, t1, t2, a[3], b[1]); + MONT_MAC(v0, v1, v2, a[4], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[4]); + MONT_MAC(t0, t1, t2, u[1], m[3]); + MONT_MAC(v0, v1, v2, u[2], m[2]); + MONT_MAC(w0, w1, w2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MAC(t0, t1, t2, a[0], b[5]); + MONT_MUL(v0, v1, v2, a[1], b[4]); + MONT_MUL(w0, w1, w2, a[2], b[3]); + MONT_MAC(t0, t1, t2, a[3], b[2]); + MONT_MAC(v0, v1, v2, a[4], b[1]); + MONT_MAC(w0, w1, w2, a[5], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MAC(v0, v1, v2, u[1], m[4]); + MONT_MAC(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MAC(t0, t1, t2, a[0], b[6]); + MONT_MUL(v0, v1, v2, a[1], b[5]); + MONT_MUL(w0, w1, w2, a[2], b[4]); + MONT_MAC(t0, t1, t2, a[3], b[3]); + MONT_MAC(v0, v1, v2, a[4], b[2]); + MONT_MAC(w0, w1, w2, a[5], b[1]); + MONT_MAC(t0, t1, t2, a[6], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[6]); + MONT_MAC(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MAC(t0, t1, t2, a[0], b[7]); + MONT_MUL(v0, v1, v2, a[1], b[6]); + MONT_MUL(w0, w1, w2, a[2], b[5]); + MONT_MAC(t0, t1, t2, a[3], b[4]); + MONT_MAC(v0, v1, v2, a[4], b[3]); + MONT_MAC(w0, w1, w2, a[5], b[2]); + MONT_MAC(t0, t1, t2, a[6], b[1]); + MONT_MAC(v0, v1, v2, a[7], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[7]); + MONT_MAC(t0, t1, t2, u[1], m[6]); + MONT_MAC(v0, v1, v2, u[2], m[5]); + MONT_MAC(w0, w1, w2, u[3], m[4]); + MONT_MAC(t0, t1, t2, u[4], m[3]); + MONT_MAC(v0, v1, v2, u[5], m[2]); + MONT_MAC(w0, w1, w2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MAC(t0, t1, t2, a[0], b[8]); + MONT_MUL(v0, v1, v2, a[1], b[7]); + MONT_MUL(w0, w1, w2, a[2], b[6]); + MONT_MAC(t0, t1, t2, a[3], b[5]); + MONT_MAC(v0, v1, v2, a[4], b[4]); + MONT_MAC(w0, w1, w2, a[5], b[3]); + MONT_MAC(t0, t1, t2, a[6], b[2]); + MONT_MAC(v0, v1, v2, a[7], b[1]); + MONT_MAC(w0, w1, w2, a[8], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[8]); + MONT_MAC(v0, v1, v2, u[1], m[7]); + MONT_MAC(w0, w1, w2, u[2], m[6]); + MONT_MAC(t0, t1, t2, u[3], m[5]); + MONT_MAC(v0, v1, v2, u[4], m[4]); + MONT_MAC(w0, w1, w2, u[5], m[3]); + MONT_MAC(t0, t1, t2, u[6], m[2]); + MONT_MAC(v0, v1, v2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MAC(t0, t1, t2, a[0], b[9]); + MONT_MUL(v0, v1, v2, a[1], b[8]); + MONT_MUL(w0, w1, w2, a[2], b[7]); + MONT_MAC(t0, t1, t2, a[3], b[6]); + MONT_MAC(v0, v1, v2, a[4], b[5]); + MONT_MAC(w0, w1, w2, a[5], b[4]); + MONT_MAC(t0, t1, t2, a[6], b[3]); + MONT_MAC(v0, v1, v2, a[7], b[2]); + MONT_MAC(w0, w1, w2, a[8], b[1]); + MONT_MAC(t0, t1, t2, a[9], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[9]); + MONT_MAC(w0, w1, w2, u[1], m[8]); + MONT_MAC(t0, t1, t2, u[2], m[7]); + MONT_MAC(v0, v1, v2, u[3], m[6]); + MONT_MAC(w0, w1, w2, u[4], m[5]); + MONT_MAC(t0, t1, t2, u[5], m[4]); + MONT_MAC(v0, v1, v2, u[6], m[3]); + MONT_MAC(w0, w1, w2, u[7], m[2]); + MONT_MAC(t0, t1, t2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MAC(t0, t1, t2, a[0], b[10]); + MONT_MUL(v0, v1, v2, a[1], b[9]); + MONT_MUL(w0, w1, w2, a[2], b[8]); + MONT_MAC(t0, t1, t2, a[3], b[7]); + MONT_MAC(v0, v1, v2, a[4], b[6]); + MONT_MAC(w0, w1, w2, a[5], b[5]); + MONT_MAC(t0, t1, t2, a[6], b[4]); + MONT_MAC(v0, v1, v2, a[7], b[3]); + MONT_MAC(w0, w1, w2, a[8], b[2]); + MONT_MAC(t0, t1, t2, a[9], b[1]); + MONT_MAC(v0, v1, v2, a[10], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[10]); + MONT_MAC(t0, t1, t2, u[1], m[9]); + MONT_MAC(v0, v1, v2, u[2], m[8]); + MONT_MAC(w0, w1, w2, u[3], m[7]); + MONT_MAC(t0, t1, t2, u[4], m[6]); + MONT_MAC(v0, v1, v2, u[5], m[5]); + MONT_MAC(w0, w1, w2, u[6], m[4]); + MONT_MAC(t0, t1, t2, u[7], m[3]); + MONT_MAC(v0, v1, v2, u[8], m[2]); + MONT_MAC(w0, w1, w2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MAC(t0, t1, t2, a[0], b[11]); + MONT_MUL(v0, v1, v2, a[1], b[10]); + MONT_MUL(w0, w1, w2, a[2], b[9]); + MONT_MAC(t0, t1, t2, a[3], b[8]); + MONT_MAC(v0, v1, v2, a[4], b[7]); + MONT_MAC(w0, w1, w2, a[5], b[6]); + MONT_MAC(t0, t1, t2, a[6], b[5]); + MONT_MAC(v0, v1, v2, a[7], b[4]); + MONT_MAC(w0, w1, w2, a[8], b[3]); + MONT_MAC(t0, t1, t2, a[9], b[2]); + MONT_MAC(v0, v1, v2, a[10], b[1]); + MONT_MAC(w0, w1, w2, a[11], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MAC(v0, v1, v2, u[1], m[10]); + MONT_MAC(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MAC(t0, t1, t2, a[0], b[12]); + MONT_MUL(v0, v1, v2, a[1], b[11]); + MONT_MUL(w0, w1, w2, a[2], b[10]); + MONT_MAC(t0, t1, t2, a[3], b[9]); + MONT_MAC(v0, v1, v2, a[4], b[8]); + MONT_MAC(w0, w1, w2, a[5], b[7]); + MONT_MAC(t0, t1, t2, a[6], b[6]); + MONT_MAC(v0, v1, v2, a[7], b[5]); + MONT_MAC(w0, w1, w2, a[8], b[4]); + MONT_MAC(t0, t1, t2, a[9], b[3]); + MONT_MAC(v0, v1, v2, a[10], b[2]); + MONT_MAC(w0, w1, w2, a[11], b[1]); + MONT_MAC(t0, t1, t2, a[12], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[12]); + MONT_MAC(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MAC(t0, t1, t2, a[0], b[13]); + MONT_MUL(v0, v1, v2, a[1], b[12]); + MONT_MUL(w0, w1, w2, a[2], b[11]); + MONT_MAC(t0, t1, t2, a[3], b[10]); + MONT_MAC(v0, v1, v2, a[4], b[9]); + MONT_MAC(w0, w1, w2, a[5], b[8]); + MONT_MAC(t0, t1, t2, a[6], b[7]); + MONT_MAC(v0, v1, v2, a[7], b[6]); + MONT_MAC(w0, w1, w2, a[8], b[5]); + MONT_MAC(t0, t1, t2, a[9], b[4]); + MONT_MAC(v0, v1, v2, a[10], b[3]); + MONT_MAC(w0, w1, w2, a[11], b[2]); + MONT_MAC(t0, t1, t2, a[12], b[1]); + MONT_MAC(v0, v1, v2, a[13], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[13]); + MONT_MAC(t0, t1, t2, u[1], m[12]); + MONT_MAC(v0, v1, v2, u[2], m[11]); + MONT_MAC(w0, w1, w2, u[3], m[10]); + MONT_MAC(t0, t1, t2, u[4], m[9]); + MONT_MAC(v0, v1, v2, u[5], m[8]); + MONT_MAC(w0, w1, w2, u[6], m[7]); + MONT_MAC(t0, t1, t2, u[7], m[6]); + MONT_MAC(v0, v1, v2, u[8], m[5]); + MONT_MAC(w0, w1, w2, u[9], m[4]); + MONT_MAC(t0, t1, t2, u[10], m[3]); + MONT_MAC(v0, v1, v2, u[11], m[2]); + MONT_MAC(w0, w1, w2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MAC(t0, t1, t2, a[0], b[14]); + MONT_MUL(v0, v1, v2, a[1], b[13]); + MONT_MUL(w0, w1, w2, a[2], b[12]); + MONT_MAC(t0, t1, t2, a[3], b[11]); + MONT_MAC(v0, v1, v2, a[4], b[10]); + MONT_MAC(w0, w1, w2, a[5], b[9]); + MONT_MAC(t0, t1, t2, a[6], b[8]); + MONT_MAC(v0, v1, v2, a[7], b[7]); + MONT_MAC(w0, w1, w2, a[8], b[6]); + MONT_MAC(t0, t1, t2, a[9], b[5]); + MONT_MAC(v0, v1, v2, a[10], b[4]); + MONT_MAC(w0, w1, w2, a[11], b[3]); + MONT_MAC(t0, t1, t2, a[12], b[2]); + MONT_MAC(v0, v1, v2, a[13], b[1]); + MONT_MAC(w0, w1, w2, a[14], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[14]); + MONT_MAC(v0, v1, v2, u[1], m[13]); + MONT_MAC(w0, w1, w2, u[2], m[12]); + MONT_MAC(t0, t1, t2, u[3], m[11]); + MONT_MAC(v0, v1, v2, u[4], m[10]); + MONT_MAC(w0, w1, w2, u[5], m[9]); + MONT_MAC(t0, t1, t2, u[6], m[8]); + MONT_MAC(v0, v1, v2, u[7], m[7]); + MONT_MAC(w0, w1, w2, u[8], m[6]); + MONT_MAC(t0, t1, t2, u[9], m[5]); + MONT_MAC(v0, v1, v2, u[10], m[4]); + MONT_MAC(w0, w1, w2, u[11], m[3]); + MONT_MAC(t0, t1, t2, u[12], m[2]); + MONT_MAC(v0, v1, v2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MAC(t0, t1, t2, a[0], b[15]); + MONT_MUL(v0, v1, v2, a[1], b[14]); + MONT_MUL(w0, w1, w2, a[2], b[13]); + MONT_MAC(t0, t1, t2, a[3], b[12]); + MONT_MAC(v0, v1, v2, a[4], b[11]); + MONT_MAC(w0, w1, w2, a[5], b[10]); + MONT_MAC(t0, t1, t2, a[6], b[9]); + MONT_MAC(v0, v1, v2, a[7], b[8]); + MONT_MAC(w0, w1, w2, a[8], b[7]); + MONT_MAC(t0, t1, t2, a[9], b[6]); + MONT_MAC(v0, v1, v2, a[10], b[5]); + MONT_MAC(w0, w1, w2, a[11], b[4]); + MONT_MAC(t0, t1, t2, a[12], b[3]); + MONT_MAC(v0, v1, v2, a[13], b[2]); + MONT_MAC(w0, w1, w2, a[14], b[1]); + MONT_MAC(t0, t1, t2, a[15], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[15]); + MONT_MAC(w0, w1, w2, u[1], m[14]); + MONT_MAC(t0, t1, t2, u[2], m[13]); + MONT_MAC(v0, v1, v2, u[3], m[12]); + MONT_MAC(w0, w1, w2, u[4], m[11]); + MONT_MAC(t0, t1, t2, u[5], m[10]); + MONT_MAC(v0, v1, v2, u[6], m[9]); + MONT_MAC(w0, w1, w2, u[7], m[8]); + MONT_MAC(t0, t1, t2, u[8], m[7]); + MONT_MAC(v0, v1, v2, u[9], m[6]); + MONT_MAC(w0, w1, w2, u[10], m[5]); + MONT_MAC(t0, t1, t2, u[11], m[4]); + MONT_MAC(v0, v1, v2, u[12], m[3]); + MONT_MAC(w0, w1, w2, u[13], m[2]); + MONT_MAC(t0, t1, t2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MAC(t0, t1, t2, a[1], b[15]); + MONT_MUL(v0, v1, v2, a[2], b[14]); + MONT_MUL(w0, w1, w2, a[3], b[13]); + MONT_MAC(t0, t1, t2, a[4], b[12]); + MONT_MAC(v0, v1, v2, a[5], b[11]); + MONT_MAC(w0, w1, w2, a[6], b[10]); + MONT_MAC(t0, t1, t2, a[7], b[9]); + MONT_MAC(v0, v1, v2, a[8], b[8]); + MONT_MAC(w0, w1, w2, a[9], b[7]); + MONT_MAC(t0, t1, t2, a[10], b[6]); + MONT_MAC(v0, v1, v2, a[11], b[5]); + MONT_MAC(w0, w1, w2, a[12], b[4]); + MONT_MAC(t0, t1, t2, a[13], b[3]); + MONT_MAC(v0, v1, v2, a[14], b[2]); + MONT_MAC(w0, w1, w2, a[15], b[1]); + MONT_MAC(t0, t1, t2, u[1], m[15]); + MONT_MAC(v0, v1, v2, u[2], m[14]); + MONT_MAC(w0, w1, w2, u[3], m[13]); + MONT_MAC(t0, t1, t2, u[4], m[12]); + MONT_MAC(v0, v1, v2, u[5], m[11]); + MONT_MAC(w0, w1, w2, u[6], m[10]); + MONT_MAC(t0, t1, t2, u[7], m[9]); + MONT_MAC(v0, v1, v2, u[8], m[8]); + MONT_MAC(w0, w1, w2, u[9], m[7]); + MONT_MAC(t0, t1, t2, u[10], m[6]); + MONT_MAC(v0, v1, v2, u[11], m[5]); + MONT_MAC(w0, w1, w2, u[12], m[4]); + MONT_MAC(t0, t1, t2, u[13], m[3]); + MONT_MAC(v0, v1, v2, u[14], m[2]); + MONT_MAC(w0, w1, w2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MAC(t0, t1, t2, a[2], b[15]); + MONT_MUL(v0, v1, v2, a[3], b[14]); + MONT_MUL(w0, w1, w2, a[4], b[13]); + MONT_MAC(t0, t1, t2, a[5], b[12]); + MONT_MAC(v0, v1, v2, a[6], b[11]); + MONT_MAC(w0, w1, w2, a[7], b[10]); + MONT_MAC(t0, t1, t2, a[8], b[9]); + MONT_MAC(v0, v1, v2, a[9], b[8]); + MONT_MAC(w0, w1, w2, a[10], b[7]); + MONT_MAC(t0, t1, t2, a[11], b[6]); + MONT_MAC(v0, v1, v2, a[12], b[5]); + MONT_MAC(w0, w1, w2, a[13], b[4]); + MONT_MAC(t0, t1, t2, a[14], b[3]); + MONT_MAC(v0, v1, v2, a[15], b[2]); + MONT_MAC(w0, w1, w2, u[2], m[15]); + MONT_MAC(t0, t1, t2, u[3], m[14]); + MONT_MAC(v0, v1, v2, u[4], m[13]); + MONT_MAC(w0, w1, w2, u[5], m[12]); + MONT_MAC(t0, t1, t2, u[6], m[11]); + MONT_MAC(v0, v1, v2, u[7], m[10]); + MONT_MAC(w0, w1, w2, u[8], m[9]); + MONT_MAC(t0, t1, t2, u[9], m[8]); + MONT_MAC(v0, v1, v2, u[10], m[7]); + MONT_MAC(w0, w1, w2, u[11], m[6]); + MONT_MAC(t0, t1, t2, u[12], m[5]); + MONT_MAC(v0, v1, v2, u[13], m[4]); + MONT_MAC(w0, w1, w2, u[14], m[3]); + MONT_MAC(t0, t1, t2, u[15], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MAC(t0, t1, t2, a[3], b[15]); + MONT_MUL(v0, v1, v2, a[4], b[14]); + MONT_MUL(w0, w1, w2, a[5], b[13]); + MONT_MAC(t0, t1, t2, a[6], b[12]); + MONT_MAC(v0, v1, v2, a[7], b[11]); + MONT_MAC(w0, w1, w2, a[8], b[10]); + MONT_MAC(t0, t1, t2, a[9], b[9]); + MONT_MAC(v0, v1, v2, a[10], b[8]); + MONT_MAC(w0, w1, w2, a[11], b[7]); + MONT_MAC(t0, t1, t2, a[12], b[6]); + MONT_MAC(v0, v1, v2, a[13], b[5]); + MONT_MAC(w0, w1, w2, a[14], b[4]); + MONT_MAC(t0, t1, t2, a[15], b[3]); + MONT_MAC(v0, v1, v2, u[3], m[15]); + MONT_MAC(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MAC(t0, t1, t2, a[4], b[15]); + MONT_MUL(v0, v1, v2, a[5], b[14]); + MONT_MUL(w0, w1, w2, a[6], b[13]); + MONT_MAC(t0, t1, t2, a[7], b[12]); + MONT_MAC(v0, v1, v2, a[8], b[11]); + MONT_MAC(w0, w1, w2, a[9], b[10]); + MONT_MAC(t0, t1, t2, a[10], b[9]); + MONT_MAC(v0, v1, v2, a[11], b[8]); + MONT_MAC(w0, w1, w2, a[12], b[7]); + MONT_MAC(t0, t1, t2, a[13], b[6]); + MONT_MAC(v0, v1, v2, a[14], b[5]); + MONT_MAC(w0, w1, w2, a[15], b[4]); + MONT_MAC(t0, t1, t2, u[4], m[15]); + MONT_MAC(v0, v1, v2, u[5], m[14]); + MONT_MAC(w0, w1, w2, u[6], m[13]); + MONT_MAC(t0, t1, t2, u[7], m[12]); + MONT_MAC(v0, v1, v2, u[8], m[11]); + MONT_MAC(w0, w1, w2, u[9], m[10]); + MONT_MAC(t0, t1, t2, u[10], m[9]); + MONT_MAC(v0, v1, v2, u[11], m[8]); + MONT_MAC(w0, w1, w2, u[12], m[7]); + MONT_MAC(t0, t1, t2, u[13], m[6]); + MONT_MAC(v0, v1, v2, u[14], m[5]); + MONT_MAC(w0, w1, w2, u[15], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MAC(t0, t1, t2, a[5], b[15]); + MONT_MUL(v0, v1, v2, a[6], b[14]); + MONT_MUL(w0, w1, w2, a[7], b[13]); + MONT_MAC(t0, t1, t2, a[8], b[12]); + MONT_MAC(v0, v1, v2, a[9], b[11]); + MONT_MAC(w0, w1, w2, a[10], b[10]); + MONT_MAC(t0, t1, t2, a[11], b[9]); + MONT_MAC(v0, v1, v2, a[12], b[8]); + MONT_MAC(w0, w1, w2, a[13], b[7]); + MONT_MAC(t0, t1, t2, a[14], b[6]); + MONT_MAC(v0, v1, v2, a[15], b[5]); + MONT_MAC(w0, w1, w2, u[5], m[15]); + MONT_MAC(t0, t1, t2, u[6], m[14]); + MONT_MAC(v0, v1, v2, u[7], m[13]); + MONT_MAC(w0, w1, w2, u[8], m[12]); + MONT_MAC(t0, t1, t2, u[9], m[11]); + MONT_MAC(v0, v1, v2, u[10], m[10]); + MONT_MAC(w0, w1, w2, u[11], m[9]); + MONT_MAC(t0, t1, t2, u[12], m[8]); + MONT_MAC(v0, v1, v2, u[13], m[7]); + MONT_MAC(w0, w1, w2, u[14], m[6]); + MONT_MAC(t0, t1, t2, u[15], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MAC(t0, t1, t2, a[6], b[15]); + MONT_MUL(v0, v1, v2, a[7], b[14]); + MONT_MUL(w0, w1, w2, a[8], b[13]); + MONT_MAC(t0, t1, t2, a[9], b[12]); + MONT_MAC(v0, v1, v2, a[10], b[11]); + MONT_MAC(w0, w1, w2, a[11], b[10]); + MONT_MAC(t0, t1, t2, a[12], b[9]); + MONT_MAC(v0, v1, v2, a[13], b[8]); + MONT_MAC(w0, w1, w2, a[14], b[7]); + MONT_MAC(t0, t1, t2, a[15], b[6]); + MONT_MAC(v0, v1, v2, u[6], m[15]); + MONT_MAC(w0, w1, w2, u[7], m[14]); + MONT_MAC(t0, t1, t2, u[8], m[13]); + MONT_MAC(v0, v1, v2, u[9], m[12]); + MONT_MAC(w0, w1, w2, u[10], m[11]); + MONT_MAC(t0, t1, t2, u[11], m[10]); + MONT_MAC(v0, v1, v2, u[12], m[9]); + MONT_MAC(w0, w1, w2, u[13], m[8]); + MONT_MAC(t0, t1, t2, u[14], m[7]); + MONT_MAC(v0, v1, v2, u[15], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MAC(t0, t1, t2, a[7], b[15]); + MONT_MUL(v0, v1, v2, a[8], b[14]); + MONT_MUL(w0, w1, w2, a[9], b[13]); + MONT_MAC(t0, t1, t2, a[10], b[12]); + MONT_MAC(v0, v1, v2, a[11], b[11]); + MONT_MAC(w0, w1, w2, a[12], b[10]); + MONT_MAC(t0, t1, t2, a[13], b[9]); + MONT_MAC(v0, v1, v2, a[14], b[8]); + MONT_MAC(w0, w1, w2, a[15], b[7]); + MONT_MAC(t0, t1, t2, u[7], m[15]); + MONT_MAC(v0, v1, v2, u[8], m[14]); + MONT_MAC(w0, w1, w2, u[9], m[13]); + MONT_MAC(t0, t1, t2, u[10], m[12]); + MONT_MAC(v0, v1, v2, u[11], m[11]); + MONT_MAC(w0, w1, w2, u[12], m[10]); + MONT_MAC(t0, t1, t2, u[13], m[9]); + MONT_MAC(v0, v1, v2, u[14], m[8]); + MONT_MAC(w0, w1, w2, u[15], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MAC(t0, t1, t2, a[8], b[15]); + MONT_MUL(v0, v1, v2, a[9], b[14]); + MONT_MUL(w0, w1, w2, a[10], b[13]); + MONT_MAC(t0, t1, t2, a[11], b[12]); + MONT_MAC(v0, v1, v2, a[12], b[11]); + MONT_MAC(w0, w1, w2, a[13], b[10]); + MONT_MAC(t0, t1, t2, a[14], b[9]); + MONT_MAC(v0, v1, v2, a[15], b[8]); + MONT_MAC(w0, w1, w2, u[8], m[15]); + MONT_MAC(t0, t1, t2, u[9], m[14]); + MONT_MAC(v0, v1, v2, u[10], m[13]); + MONT_MAC(w0, w1, w2, u[11], m[12]); + MONT_MAC(t0, t1, t2, u[12], m[11]); + MONT_MAC(v0, v1, v2, u[13], m[10]); + MONT_MAC(w0, w1, w2, u[14], m[9]); + MONT_MAC(t0, t1, t2, u[15], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MAC(t0, t1, t2, a[9], b[15]); + MONT_MUL(v0, v1, v2, a[10], b[14]); + MONT_MUL(w0, w1, w2, a[11], b[13]); + MONT_MAC(t0, t1, t2, a[12], b[12]); + MONT_MAC(v0, v1, v2, a[13], b[11]); + MONT_MAC(w0, w1, w2, a[14], b[10]); + MONT_MAC(t0, t1, t2, a[15], b[9]); + MONT_MAC(v0, v1, v2, u[9], m[15]); + MONT_MAC(w0, w1, w2, u[10], m[14]); + MONT_MAC(t0, t1, t2, u[11], m[13]); + MONT_MAC(v0, v1, v2, u[12], m[12]); + MONT_MAC(w0, w1, w2, u[13], m[11]); + MONT_MAC(t0, t1, t2, u[14], m[10]); + MONT_MAC(v0, v1, v2, u[15], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MAC(t0, t1, t2, a[10], b[15]); + MONT_MUL(v0, v1, v2, a[11], b[14]); + MONT_MUL(w0, w1, w2, a[12], b[13]); + MONT_MAC(t0, t1, t2, a[13], b[12]); + MONT_MAC(v0, v1, v2, a[14], b[11]); + MONT_MAC(w0, w1, w2, a[15], b[10]); + MONT_MAC(t0, t1, t2, u[10], m[15]); + MONT_MAC(v0, v1, v2, u[11], m[14]); + MONT_MAC(w0, w1, w2, u[12], m[13]); + MONT_MAC(t0, t1, t2, u[13], m[12]); + MONT_MAC(v0, v1, v2, u[14], m[11]); + MONT_MAC(w0, w1, w2, u[15], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MAC(t0, t1, t2, a[11], b[15]); + MONT_MUL(v0, v1, v2, a[12], b[14]); + MONT_MUL(w0, w1, w2, a[13], b[13]); + MONT_MAC(t0, t1, t2, a[14], b[12]); + MONT_MAC(v0, v1, v2, a[15], b[11]); + MONT_MAC(w0, w1, w2, u[11], m[15]); + MONT_MAC(t0, t1, t2, u[12], m[14]); + MONT_MAC(v0, v1, v2, u[13], m[13]); + MONT_MAC(w0, w1, w2, u[14], m[12]); + MONT_MAC(t0, t1, t2, u[15], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MAC(t0, t1, t2, a[12], b[15]); + MONT_MUL(v0, v1, v2, a[13], b[14]); + MONT_MUL(w0, w1, w2, a[14], b[13]); + MONT_MAC(t0, t1, t2, a[15], b[12]); + MONT_MAC(v0, v1, v2, u[12], m[15]); + MONT_MAC(w0, w1, w2, u[13], m[14]); + MONT_MAC(t0, t1, t2, u[14], m[13]); + MONT_MAC(v0, v1, v2, u[15], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MAC(t0, t1, t2, a[13], b[15]); + MONT_MUL(v0, v1, v2, a[14], b[14]); + MONT_MUL(w0, w1, w2, a[15], b[13]); + MONT_MAC(t0, t1, t2, u[13], m[15]); + MONT_MAC(v0, v1, v2, u[14], m[14]); + MONT_MAC(w0, w1, w2, u[15], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MAC(t0, t1, t2, a[14], b[15]); + MONT_MUL(v0, v1, v2, a[15], b[14]); + MONT_MUL(w0, w1, w2, u[14], m[15]); + MONT_MAC(t0, t1, t2, u[15], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MAC(t0, t1, t2, a[15], b[15]); + MONT_MUL(v0, v1, v2, u[15], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[15] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 16); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 16); +} + +/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ +static void crypton_mont_sqr16(limb_t *r, const limb_t *a, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 16; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + limb_t s0, s1, s2, c0, c1, c2; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], a[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MUL(s0, s1, s2, a[0], a[1]); + MONT_MAC(t0, t1, t2, u[0], m[1]); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MUL(s0, s1, s2, a[0], a[2]); + MONT_MAC(t0, t1, t2, a[1], a[1]); + MONT_MUL(v0, v1, v2, u[0], m[2]); + MONT_MUL(w0, w1, w2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MUL(s0, s1, s2, a[0], a[3]); + MONT_MUL(c0, c1, c2, a[1], a[2]); + MONT_MAC(t0, t1, t2, u[0], m[3]); + MONT_MUL(v0, v1, v2, u[1], m[2]); + MONT_MUL(w0, w1, w2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MUL(s0, s1, s2, a[0], a[4]); + MONT_MUL(c0, c1, c2, a[1], a[3]); + MONT_MAC(t0, t1, t2, a[2], a[2]); + MONT_MUL(v0, v1, v2, u[0], m[4]); + MONT_MUL(w0, w1, w2, u[1], m[3]); + MONT_MAC(t0, t1, t2, u[2], m[2]); + MONT_MAC(v0, v1, v2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MUL(s0, s1, s2, a[0], a[5]); + MONT_MUL(c0, c1, c2, a[1], a[4]); + MONT_MAC(s0, s1, s2, a[2], a[3]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MUL(v0, v1, v2, u[1], m[4]); + MONT_MUL(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MUL(s0, s1, s2, a[0], a[6]); + MONT_MUL(c0, c1, c2, a[1], a[5]); + MONT_MAC(s0, s1, s2, a[2], a[4]); + MONT_MAC(t0, t1, t2, a[3], a[3]); + MONT_MUL(v0, v1, v2, u[0], m[6]); + MONT_MUL(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MUL(s0, s1, s2, a[0], a[7]); + MONT_MUL(c0, c1, c2, a[1], a[6]); + MONT_MAC(s0, s1, s2, a[2], a[5]); + MONT_MAC(c0, c1, c2, a[3], a[4]); + MONT_MAC(t0, t1, t2, u[0], m[7]); + MONT_MUL(v0, v1, v2, u[1], m[6]); + MONT_MUL(w0, w1, w2, u[2], m[5]); + MONT_MAC(t0, t1, t2, u[3], m[4]); + MONT_MAC(v0, v1, v2, u[4], m[3]); + MONT_MAC(w0, w1, w2, u[5], m[2]); + MONT_MAC(t0, t1, t2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MUL(s0, s1, s2, a[0], a[8]); + MONT_MUL(c0, c1, c2, a[1], a[7]); + MONT_MAC(s0, s1, s2, a[2], a[6]); + MONT_MAC(c0, c1, c2, a[3], a[5]); + MONT_MAC(t0, t1, t2, a[4], a[4]); + MONT_MUL(v0, v1, v2, u[0], m[8]); + MONT_MUL(w0, w1, w2, u[1], m[7]); + MONT_MAC(t0, t1, t2, u[2], m[6]); + MONT_MAC(v0, v1, v2, u[3], m[5]); + MONT_MAC(w0, w1, w2, u[4], m[4]); + MONT_MAC(t0, t1, t2, u[5], m[3]); + MONT_MAC(v0, v1, v2, u[6], m[2]); + MONT_MAC(w0, w1, w2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MUL(s0, s1, s2, a[0], a[9]); + MONT_MUL(c0, c1, c2, a[1], a[8]); + MONT_MAC(s0, s1, s2, a[2], a[7]); + MONT_MAC(c0, c1, c2, a[3], a[6]); + MONT_MAC(s0, s1, s2, a[4], a[5]); + MONT_MAC(t0, t1, t2, u[0], m[9]); + MONT_MUL(v0, v1, v2, u[1], m[8]); + MONT_MUL(w0, w1, w2, u[2], m[7]); + MONT_MAC(t0, t1, t2, u[3], m[6]); + MONT_MAC(v0, v1, v2, u[4], m[5]); + MONT_MAC(w0, w1, w2, u[5], m[4]); + MONT_MAC(t0, t1, t2, u[6], m[3]); + MONT_MAC(v0, v1, v2, u[7], m[2]); + MONT_MAC(w0, w1, w2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MUL(s0, s1, s2, a[0], a[10]); + MONT_MUL(c0, c1, c2, a[1], a[9]); + MONT_MAC(s0, s1, s2, a[2], a[8]); + MONT_MAC(c0, c1, c2, a[3], a[7]); + MONT_MAC(s0, s1, s2, a[4], a[6]); + MONT_MAC(t0, t1, t2, a[5], a[5]); + MONT_MUL(v0, v1, v2, u[0], m[10]); + MONT_MUL(w0, w1, w2, u[1], m[9]); + MONT_MAC(t0, t1, t2, u[2], m[8]); + MONT_MAC(v0, v1, v2, u[3], m[7]); + MONT_MAC(w0, w1, w2, u[4], m[6]); + MONT_MAC(t0, t1, t2, u[5], m[5]); + MONT_MAC(v0, v1, v2, u[6], m[4]); + MONT_MAC(w0, w1, w2, u[7], m[3]); + MONT_MAC(t0, t1, t2, u[8], m[2]); + MONT_MAC(v0, v1, v2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MUL(s0, s1, s2, a[0], a[11]); + MONT_MUL(c0, c1, c2, a[1], a[10]); + MONT_MAC(s0, s1, s2, a[2], a[9]); + MONT_MAC(c0, c1, c2, a[3], a[8]); + MONT_MAC(s0, s1, s2, a[4], a[7]); + MONT_MAC(c0, c1, c2, a[5], a[6]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MUL(v0, v1, v2, u[1], m[10]); + MONT_MUL(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MUL(s0, s1, s2, a[0], a[12]); + MONT_MUL(c0, c1, c2, a[1], a[11]); + MONT_MAC(s0, s1, s2, a[2], a[10]); + MONT_MAC(c0, c1, c2, a[3], a[9]); + MONT_MAC(s0, s1, s2, a[4], a[8]); + MONT_MAC(c0, c1, c2, a[5], a[7]); + MONT_MAC(t0, t1, t2, a[6], a[6]); + MONT_MUL(v0, v1, v2, u[0], m[12]); + MONT_MUL(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MUL(s0, s1, s2, a[0], a[13]); + MONT_MUL(c0, c1, c2, a[1], a[12]); + MONT_MAC(s0, s1, s2, a[2], a[11]); + MONT_MAC(c0, c1, c2, a[3], a[10]); + MONT_MAC(s0, s1, s2, a[4], a[9]); + MONT_MAC(c0, c1, c2, a[5], a[8]); + MONT_MAC(s0, s1, s2, a[6], a[7]); + MONT_MAC(t0, t1, t2, u[0], m[13]); + MONT_MUL(v0, v1, v2, u[1], m[12]); + MONT_MUL(w0, w1, w2, u[2], m[11]); + MONT_MAC(t0, t1, t2, u[3], m[10]); + MONT_MAC(v0, v1, v2, u[4], m[9]); + MONT_MAC(w0, w1, w2, u[5], m[8]); + MONT_MAC(t0, t1, t2, u[6], m[7]); + MONT_MAC(v0, v1, v2, u[7], m[6]); + MONT_MAC(w0, w1, w2, u[8], m[5]); + MONT_MAC(t0, t1, t2, u[9], m[4]); + MONT_MAC(v0, v1, v2, u[10], m[3]); + MONT_MAC(w0, w1, w2, u[11], m[2]); + MONT_MAC(t0, t1, t2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MUL(s0, s1, s2, a[0], a[14]); + MONT_MUL(c0, c1, c2, a[1], a[13]); + MONT_MAC(s0, s1, s2, a[2], a[12]); + MONT_MAC(c0, c1, c2, a[3], a[11]); + MONT_MAC(s0, s1, s2, a[4], a[10]); + MONT_MAC(c0, c1, c2, a[5], a[9]); + MONT_MAC(s0, s1, s2, a[6], a[8]); + MONT_MAC(t0, t1, t2, a[7], a[7]); + MONT_MUL(v0, v1, v2, u[0], m[14]); + MONT_MUL(w0, w1, w2, u[1], m[13]); + MONT_MAC(t0, t1, t2, u[2], m[12]); + MONT_MAC(v0, v1, v2, u[3], m[11]); + MONT_MAC(w0, w1, w2, u[4], m[10]); + MONT_MAC(t0, t1, t2, u[5], m[9]); + MONT_MAC(v0, v1, v2, u[6], m[8]); + MONT_MAC(w0, w1, w2, u[7], m[7]); + MONT_MAC(t0, t1, t2, u[8], m[6]); + MONT_MAC(v0, v1, v2, u[9], m[5]); + MONT_MAC(w0, w1, w2, u[10], m[4]); + MONT_MAC(t0, t1, t2, u[11], m[3]); + MONT_MAC(v0, v1, v2, u[12], m[2]); + MONT_MAC(w0, w1, w2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MUL(s0, s1, s2, a[0], a[15]); + MONT_MUL(c0, c1, c2, a[1], a[14]); + MONT_MAC(s0, s1, s2, a[2], a[13]); + MONT_MAC(c0, c1, c2, a[3], a[12]); + MONT_MAC(s0, s1, s2, a[4], a[11]); + MONT_MAC(c0, c1, c2, a[5], a[10]); + MONT_MAC(s0, s1, s2, a[6], a[9]); + MONT_MAC(c0, c1, c2, a[7], a[8]); + MONT_MAC(t0, t1, t2, u[0], m[15]); + MONT_MUL(v0, v1, v2, u[1], m[14]); + MONT_MUL(w0, w1, w2, u[2], m[13]); + MONT_MAC(t0, t1, t2, u[3], m[12]); + MONT_MAC(v0, v1, v2, u[4], m[11]); + MONT_MAC(w0, w1, w2, u[5], m[10]); + MONT_MAC(t0, t1, t2, u[6], m[9]); + MONT_MAC(v0, v1, v2, u[7], m[8]); + MONT_MAC(w0, w1, w2, u[8], m[7]); + MONT_MAC(t0, t1, t2, u[9], m[6]); + MONT_MAC(v0, v1, v2, u[10], m[5]); + MONT_MAC(w0, w1, w2, u[11], m[4]); + MONT_MAC(t0, t1, t2, u[12], m[3]); + MONT_MAC(v0, v1, v2, u[13], m[2]); + MONT_MAC(w0, w1, w2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MUL(s0, s1, s2, a[1], a[15]); + MONT_MUL(c0, c1, c2, a[2], a[14]); + MONT_MAC(s0, s1, s2, a[3], a[13]); + MONT_MAC(c0, c1, c2, a[4], a[12]); + MONT_MAC(s0, s1, s2, a[5], a[11]); + MONT_MAC(c0, c1, c2, a[6], a[10]); + MONT_MAC(s0, s1, s2, a[7], a[9]); + MONT_MAC(t0, t1, t2, a[8], a[8]); + MONT_MUL(v0, v1, v2, u[1], m[15]); + MONT_MUL(w0, w1, w2, u[2], m[14]); + MONT_MAC(t0, t1, t2, u[3], m[13]); + MONT_MAC(v0, v1, v2, u[4], m[12]); + MONT_MAC(w0, w1, w2, u[5], m[11]); + MONT_MAC(t0, t1, t2, u[6], m[10]); + MONT_MAC(v0, v1, v2, u[7], m[9]); + MONT_MAC(w0, w1, w2, u[8], m[8]); + MONT_MAC(t0, t1, t2, u[9], m[7]); + MONT_MAC(v0, v1, v2, u[10], m[6]); + MONT_MAC(w0, w1, w2, u[11], m[5]); + MONT_MAC(t0, t1, t2, u[12], m[4]); + MONT_MAC(v0, v1, v2, u[13], m[3]); + MONT_MAC(w0, w1, w2, u[14], m[2]); + MONT_MAC(t0, t1, t2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MUL(s0, s1, s2, a[2], a[15]); + MONT_MUL(c0, c1, c2, a[3], a[14]); + MONT_MAC(s0, s1, s2, a[4], a[13]); + MONT_MAC(c0, c1, c2, a[5], a[12]); + MONT_MAC(s0, s1, s2, a[6], a[11]); + MONT_MAC(c0, c1, c2, a[7], a[10]); + MONT_MAC(s0, s1, s2, a[8], a[9]); + MONT_MAC(t0, t1, t2, u[2], m[15]); + MONT_MUL(v0, v1, v2, u[3], m[14]); + MONT_MUL(w0, w1, w2, u[4], m[13]); + MONT_MAC(t0, t1, t2, u[5], m[12]); + MONT_MAC(v0, v1, v2, u[6], m[11]); + MONT_MAC(w0, w1, w2, u[7], m[10]); + MONT_MAC(t0, t1, t2, u[8], m[9]); + MONT_MAC(v0, v1, v2, u[9], m[8]); + MONT_MAC(w0, w1, w2, u[10], m[7]); + MONT_MAC(t0, t1, t2, u[11], m[6]); + MONT_MAC(v0, v1, v2, u[12], m[5]); + MONT_MAC(w0, w1, w2, u[13], m[4]); + MONT_MAC(t0, t1, t2, u[14], m[3]); + MONT_MAC(v0, v1, v2, u[15], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MUL(s0, s1, s2, a[3], a[15]); + MONT_MUL(c0, c1, c2, a[4], a[14]); + MONT_MAC(s0, s1, s2, a[5], a[13]); + MONT_MAC(c0, c1, c2, a[6], a[12]); + MONT_MAC(s0, s1, s2, a[7], a[11]); + MONT_MAC(c0, c1, c2, a[8], a[10]); + MONT_MAC(t0, t1, t2, a[9], a[9]); + MONT_MUL(v0, v1, v2, u[3], m[15]); + MONT_MUL(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MUL(s0, s1, s2, a[4], a[15]); + MONT_MUL(c0, c1, c2, a[5], a[14]); + MONT_MAC(s0, s1, s2, a[6], a[13]); + MONT_MAC(c0, c1, c2, a[7], a[12]); + MONT_MAC(s0, s1, s2, a[8], a[11]); + MONT_MAC(c0, c1, c2, a[9], a[10]); + MONT_MAC(t0, t1, t2, u[4], m[15]); + MONT_MUL(v0, v1, v2, u[5], m[14]); + MONT_MUL(w0, w1, w2, u[6], m[13]); + MONT_MAC(t0, t1, t2, u[7], m[12]); + MONT_MAC(v0, v1, v2, u[8], m[11]); + MONT_MAC(w0, w1, w2, u[9], m[10]); + MONT_MAC(t0, t1, t2, u[10], m[9]); + MONT_MAC(v0, v1, v2, u[11], m[8]); + MONT_MAC(w0, w1, w2, u[12], m[7]); + MONT_MAC(t0, t1, t2, u[13], m[6]); + MONT_MAC(v0, v1, v2, u[14], m[5]); + MONT_MAC(w0, w1, w2, u[15], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MUL(s0, s1, s2, a[5], a[15]); + MONT_MUL(c0, c1, c2, a[6], a[14]); + MONT_MAC(s0, s1, s2, a[7], a[13]); + MONT_MAC(c0, c1, c2, a[8], a[12]); + MONT_MAC(s0, s1, s2, a[9], a[11]); + MONT_MAC(t0, t1, t2, a[10], a[10]); + MONT_MUL(v0, v1, v2, u[5], m[15]); + MONT_MUL(w0, w1, w2, u[6], m[14]); + MONT_MAC(t0, t1, t2, u[7], m[13]); + MONT_MAC(v0, v1, v2, u[8], m[12]); + MONT_MAC(w0, w1, w2, u[9], m[11]); + MONT_MAC(t0, t1, t2, u[10], m[10]); + MONT_MAC(v0, v1, v2, u[11], m[9]); + MONT_MAC(w0, w1, w2, u[12], m[8]); + MONT_MAC(t0, t1, t2, u[13], m[7]); + MONT_MAC(v0, v1, v2, u[14], m[6]); + MONT_MAC(w0, w1, w2, u[15], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MUL(s0, s1, s2, a[6], a[15]); + MONT_MUL(c0, c1, c2, a[7], a[14]); + MONT_MAC(s0, s1, s2, a[8], a[13]); + MONT_MAC(c0, c1, c2, a[9], a[12]); + MONT_MAC(s0, s1, s2, a[10], a[11]); + MONT_MAC(t0, t1, t2, u[6], m[15]); + MONT_MUL(v0, v1, v2, u[7], m[14]); + MONT_MUL(w0, w1, w2, u[8], m[13]); + MONT_MAC(t0, t1, t2, u[9], m[12]); + MONT_MAC(v0, v1, v2, u[10], m[11]); + MONT_MAC(w0, w1, w2, u[11], m[10]); + MONT_MAC(t0, t1, t2, u[12], m[9]); + MONT_MAC(v0, v1, v2, u[13], m[8]); + MONT_MAC(w0, w1, w2, u[14], m[7]); + MONT_MAC(t0, t1, t2, u[15], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MUL(s0, s1, s2, a[7], a[15]); + MONT_MUL(c0, c1, c2, a[8], a[14]); + MONT_MAC(s0, s1, s2, a[9], a[13]); + MONT_MAC(c0, c1, c2, a[10], a[12]); + MONT_MAC(t0, t1, t2, a[11], a[11]); + MONT_MUL(v0, v1, v2, u[7], m[15]); + MONT_MUL(w0, w1, w2, u[8], m[14]); + MONT_MAC(t0, t1, t2, u[9], m[13]); + MONT_MAC(v0, v1, v2, u[10], m[12]); + MONT_MAC(w0, w1, w2, u[11], m[11]); + MONT_MAC(t0, t1, t2, u[12], m[10]); + MONT_MAC(v0, v1, v2, u[13], m[9]); + MONT_MAC(w0, w1, w2, u[14], m[8]); + MONT_MAC(t0, t1, t2, u[15], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MUL(s0, s1, s2, a[8], a[15]); + MONT_MUL(c0, c1, c2, a[9], a[14]); + MONT_MAC(s0, s1, s2, a[10], a[13]); + MONT_MAC(c0, c1, c2, a[11], a[12]); + MONT_MAC(t0, t1, t2, u[8], m[15]); + MONT_MUL(v0, v1, v2, u[9], m[14]); + MONT_MUL(w0, w1, w2, u[10], m[13]); + MONT_MAC(t0, t1, t2, u[11], m[12]); + MONT_MAC(v0, v1, v2, u[12], m[11]); + MONT_MAC(w0, w1, w2, u[13], m[10]); + MONT_MAC(t0, t1, t2, u[14], m[9]); + MONT_MAC(v0, v1, v2, u[15], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MUL(s0, s1, s2, a[9], a[15]); + MONT_MUL(c0, c1, c2, a[10], a[14]); + MONT_MAC(s0, s1, s2, a[11], a[13]); + MONT_MAC(t0, t1, t2, a[12], a[12]); + MONT_MUL(v0, v1, v2, u[9], m[15]); + MONT_MUL(w0, w1, w2, u[10], m[14]); + MONT_MAC(t0, t1, t2, u[11], m[13]); + MONT_MAC(v0, v1, v2, u[12], m[12]); + MONT_MAC(w0, w1, w2, u[13], m[11]); + MONT_MAC(t0, t1, t2, u[14], m[10]); + MONT_MAC(v0, v1, v2, u[15], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MUL(s0, s1, s2, a[10], a[15]); + MONT_MUL(c0, c1, c2, a[11], a[14]); + MONT_MAC(s0, s1, s2, a[12], a[13]); + MONT_MAC(t0, t1, t2, u[10], m[15]); + MONT_MUL(v0, v1, v2, u[11], m[14]); + MONT_MUL(w0, w1, w2, u[12], m[13]); + MONT_MAC(t0, t1, t2, u[13], m[12]); + MONT_MAC(v0, v1, v2, u[14], m[11]); + MONT_MAC(w0, w1, w2, u[15], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MUL(s0, s1, s2, a[11], a[15]); + MONT_MUL(c0, c1, c2, a[12], a[14]); + MONT_MAC(t0, t1, t2, a[13], a[13]); + MONT_MUL(v0, v1, v2, u[11], m[15]); + MONT_MUL(w0, w1, w2, u[12], m[14]); + MONT_MAC(t0, t1, t2, u[13], m[13]); + MONT_MAC(v0, v1, v2, u[14], m[12]); + MONT_MAC(w0, w1, w2, u[15], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MUL(s0, s1, s2, a[12], a[15]); + MONT_MUL(c0, c1, c2, a[13], a[14]); + MONT_MAC(t0, t1, t2, u[12], m[15]); + MONT_MUL(v0, v1, v2, u[13], m[14]); + MONT_MUL(w0, w1, w2, u[14], m[13]); + MONT_MAC(t0, t1, t2, u[15], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MUL(s0, s1, s2, a[13], a[15]); + MONT_MAC(t0, t1, t2, a[14], a[14]); + MONT_MUL(v0, v1, v2, u[13], m[15]); + MONT_MUL(w0, w1, w2, u[14], m[14]); + MONT_MAC(t0, t1, t2, u[15], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MUL(s0, s1, s2, a[14], a[15]); + MONT_MAC(t0, t1, t2, u[14], m[15]); + MONT_MUL(v0, v1, v2, u[15], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MAC(t0, t1, t2, a[15], a[15]); + MONT_MUL(v0, v1, v2, u[15], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[15] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 16); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 16); +} + +/* r = a * b * R^-1 mod m, for a and b under m; t is 2n limbs of scratch */ +static void crypton_mont_mul24(limb_t *r, const limb_t *a, const limb_t *b, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 24; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], b[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MAC(t0, t1, t2, a[0], b[1]); + MONT_MUL(v0, v1, v2, a[1], b[0]); + MONT_MUL(w0, w1, w2, u[0], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MAC(t0, t1, t2, a[0], b[2]); + MONT_MUL(v0, v1, v2, a[1], b[1]); + MONT_MUL(w0, w1, w2, a[2], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[2]); + MONT_MAC(v0, v1, v2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MAC(t0, t1, t2, a[0], b[3]); + MONT_MUL(v0, v1, v2, a[1], b[2]); + MONT_MUL(w0, w1, w2, a[2], b[1]); + MONT_MAC(t0, t1, t2, a[3], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[3]); + MONT_MAC(w0, w1, w2, u[1], m[2]); + MONT_MAC(t0, t1, t2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MAC(t0, t1, t2, a[0], b[4]); + MONT_MUL(v0, v1, v2, a[1], b[3]); + MONT_MUL(w0, w1, w2, a[2], b[2]); + MONT_MAC(t0, t1, t2, a[3], b[1]); + MONT_MAC(v0, v1, v2, a[4], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[4]); + MONT_MAC(t0, t1, t2, u[1], m[3]); + MONT_MAC(v0, v1, v2, u[2], m[2]); + MONT_MAC(w0, w1, w2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MAC(t0, t1, t2, a[0], b[5]); + MONT_MUL(v0, v1, v2, a[1], b[4]); + MONT_MUL(w0, w1, w2, a[2], b[3]); + MONT_MAC(t0, t1, t2, a[3], b[2]); + MONT_MAC(v0, v1, v2, a[4], b[1]); + MONT_MAC(w0, w1, w2, a[5], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MAC(v0, v1, v2, u[1], m[4]); + MONT_MAC(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MAC(t0, t1, t2, a[0], b[6]); + MONT_MUL(v0, v1, v2, a[1], b[5]); + MONT_MUL(w0, w1, w2, a[2], b[4]); + MONT_MAC(t0, t1, t2, a[3], b[3]); + MONT_MAC(v0, v1, v2, a[4], b[2]); + MONT_MAC(w0, w1, w2, a[5], b[1]); + MONT_MAC(t0, t1, t2, a[6], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[6]); + MONT_MAC(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MAC(t0, t1, t2, a[0], b[7]); + MONT_MUL(v0, v1, v2, a[1], b[6]); + MONT_MUL(w0, w1, w2, a[2], b[5]); + MONT_MAC(t0, t1, t2, a[3], b[4]); + MONT_MAC(v0, v1, v2, a[4], b[3]); + MONT_MAC(w0, w1, w2, a[5], b[2]); + MONT_MAC(t0, t1, t2, a[6], b[1]); + MONT_MAC(v0, v1, v2, a[7], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[7]); + MONT_MAC(t0, t1, t2, u[1], m[6]); + MONT_MAC(v0, v1, v2, u[2], m[5]); + MONT_MAC(w0, w1, w2, u[3], m[4]); + MONT_MAC(t0, t1, t2, u[4], m[3]); + MONT_MAC(v0, v1, v2, u[5], m[2]); + MONT_MAC(w0, w1, w2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MAC(t0, t1, t2, a[0], b[8]); + MONT_MUL(v0, v1, v2, a[1], b[7]); + MONT_MUL(w0, w1, w2, a[2], b[6]); + MONT_MAC(t0, t1, t2, a[3], b[5]); + MONT_MAC(v0, v1, v2, a[4], b[4]); + MONT_MAC(w0, w1, w2, a[5], b[3]); + MONT_MAC(t0, t1, t2, a[6], b[2]); + MONT_MAC(v0, v1, v2, a[7], b[1]); + MONT_MAC(w0, w1, w2, a[8], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[8]); + MONT_MAC(v0, v1, v2, u[1], m[7]); + MONT_MAC(w0, w1, w2, u[2], m[6]); + MONT_MAC(t0, t1, t2, u[3], m[5]); + MONT_MAC(v0, v1, v2, u[4], m[4]); + MONT_MAC(w0, w1, w2, u[5], m[3]); + MONT_MAC(t0, t1, t2, u[6], m[2]); + MONT_MAC(v0, v1, v2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MAC(t0, t1, t2, a[0], b[9]); + MONT_MUL(v0, v1, v2, a[1], b[8]); + MONT_MUL(w0, w1, w2, a[2], b[7]); + MONT_MAC(t0, t1, t2, a[3], b[6]); + MONT_MAC(v0, v1, v2, a[4], b[5]); + MONT_MAC(w0, w1, w2, a[5], b[4]); + MONT_MAC(t0, t1, t2, a[6], b[3]); + MONT_MAC(v0, v1, v2, a[7], b[2]); + MONT_MAC(w0, w1, w2, a[8], b[1]); + MONT_MAC(t0, t1, t2, a[9], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[9]); + MONT_MAC(w0, w1, w2, u[1], m[8]); + MONT_MAC(t0, t1, t2, u[2], m[7]); + MONT_MAC(v0, v1, v2, u[3], m[6]); + MONT_MAC(w0, w1, w2, u[4], m[5]); + MONT_MAC(t0, t1, t2, u[5], m[4]); + MONT_MAC(v0, v1, v2, u[6], m[3]); + MONT_MAC(w0, w1, w2, u[7], m[2]); + MONT_MAC(t0, t1, t2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MAC(t0, t1, t2, a[0], b[10]); + MONT_MUL(v0, v1, v2, a[1], b[9]); + MONT_MUL(w0, w1, w2, a[2], b[8]); + MONT_MAC(t0, t1, t2, a[3], b[7]); + MONT_MAC(v0, v1, v2, a[4], b[6]); + MONT_MAC(w0, w1, w2, a[5], b[5]); + MONT_MAC(t0, t1, t2, a[6], b[4]); + MONT_MAC(v0, v1, v2, a[7], b[3]); + MONT_MAC(w0, w1, w2, a[8], b[2]); + MONT_MAC(t0, t1, t2, a[9], b[1]); + MONT_MAC(v0, v1, v2, a[10], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[10]); + MONT_MAC(t0, t1, t2, u[1], m[9]); + MONT_MAC(v0, v1, v2, u[2], m[8]); + MONT_MAC(w0, w1, w2, u[3], m[7]); + MONT_MAC(t0, t1, t2, u[4], m[6]); + MONT_MAC(v0, v1, v2, u[5], m[5]); + MONT_MAC(w0, w1, w2, u[6], m[4]); + MONT_MAC(t0, t1, t2, u[7], m[3]); + MONT_MAC(v0, v1, v2, u[8], m[2]); + MONT_MAC(w0, w1, w2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MAC(t0, t1, t2, a[0], b[11]); + MONT_MUL(v0, v1, v2, a[1], b[10]); + MONT_MUL(w0, w1, w2, a[2], b[9]); + MONT_MAC(t0, t1, t2, a[3], b[8]); + MONT_MAC(v0, v1, v2, a[4], b[7]); + MONT_MAC(w0, w1, w2, a[5], b[6]); + MONT_MAC(t0, t1, t2, a[6], b[5]); + MONT_MAC(v0, v1, v2, a[7], b[4]); + MONT_MAC(w0, w1, w2, a[8], b[3]); + MONT_MAC(t0, t1, t2, a[9], b[2]); + MONT_MAC(v0, v1, v2, a[10], b[1]); + MONT_MAC(w0, w1, w2, a[11], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MAC(v0, v1, v2, u[1], m[10]); + MONT_MAC(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MAC(t0, t1, t2, a[0], b[12]); + MONT_MUL(v0, v1, v2, a[1], b[11]); + MONT_MUL(w0, w1, w2, a[2], b[10]); + MONT_MAC(t0, t1, t2, a[3], b[9]); + MONT_MAC(v0, v1, v2, a[4], b[8]); + MONT_MAC(w0, w1, w2, a[5], b[7]); + MONT_MAC(t0, t1, t2, a[6], b[6]); + MONT_MAC(v0, v1, v2, a[7], b[5]); + MONT_MAC(w0, w1, w2, a[8], b[4]); + MONT_MAC(t0, t1, t2, a[9], b[3]); + MONT_MAC(v0, v1, v2, a[10], b[2]); + MONT_MAC(w0, w1, w2, a[11], b[1]); + MONT_MAC(t0, t1, t2, a[12], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[12]); + MONT_MAC(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MAC(t0, t1, t2, a[0], b[13]); + MONT_MUL(v0, v1, v2, a[1], b[12]); + MONT_MUL(w0, w1, w2, a[2], b[11]); + MONT_MAC(t0, t1, t2, a[3], b[10]); + MONT_MAC(v0, v1, v2, a[4], b[9]); + MONT_MAC(w0, w1, w2, a[5], b[8]); + MONT_MAC(t0, t1, t2, a[6], b[7]); + MONT_MAC(v0, v1, v2, a[7], b[6]); + MONT_MAC(w0, w1, w2, a[8], b[5]); + MONT_MAC(t0, t1, t2, a[9], b[4]); + MONT_MAC(v0, v1, v2, a[10], b[3]); + MONT_MAC(w0, w1, w2, a[11], b[2]); + MONT_MAC(t0, t1, t2, a[12], b[1]); + MONT_MAC(v0, v1, v2, a[13], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[13]); + MONT_MAC(t0, t1, t2, u[1], m[12]); + MONT_MAC(v0, v1, v2, u[2], m[11]); + MONT_MAC(w0, w1, w2, u[3], m[10]); + MONT_MAC(t0, t1, t2, u[4], m[9]); + MONT_MAC(v0, v1, v2, u[5], m[8]); + MONT_MAC(w0, w1, w2, u[6], m[7]); + MONT_MAC(t0, t1, t2, u[7], m[6]); + MONT_MAC(v0, v1, v2, u[8], m[5]); + MONT_MAC(w0, w1, w2, u[9], m[4]); + MONT_MAC(t0, t1, t2, u[10], m[3]); + MONT_MAC(v0, v1, v2, u[11], m[2]); + MONT_MAC(w0, w1, w2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MAC(t0, t1, t2, a[0], b[14]); + MONT_MUL(v0, v1, v2, a[1], b[13]); + MONT_MUL(w0, w1, w2, a[2], b[12]); + MONT_MAC(t0, t1, t2, a[3], b[11]); + MONT_MAC(v0, v1, v2, a[4], b[10]); + MONT_MAC(w0, w1, w2, a[5], b[9]); + MONT_MAC(t0, t1, t2, a[6], b[8]); + MONT_MAC(v0, v1, v2, a[7], b[7]); + MONT_MAC(w0, w1, w2, a[8], b[6]); + MONT_MAC(t0, t1, t2, a[9], b[5]); + MONT_MAC(v0, v1, v2, a[10], b[4]); + MONT_MAC(w0, w1, w2, a[11], b[3]); + MONT_MAC(t0, t1, t2, a[12], b[2]); + MONT_MAC(v0, v1, v2, a[13], b[1]); + MONT_MAC(w0, w1, w2, a[14], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[14]); + MONT_MAC(v0, v1, v2, u[1], m[13]); + MONT_MAC(w0, w1, w2, u[2], m[12]); + MONT_MAC(t0, t1, t2, u[3], m[11]); + MONT_MAC(v0, v1, v2, u[4], m[10]); + MONT_MAC(w0, w1, w2, u[5], m[9]); + MONT_MAC(t0, t1, t2, u[6], m[8]); + MONT_MAC(v0, v1, v2, u[7], m[7]); + MONT_MAC(w0, w1, w2, u[8], m[6]); + MONT_MAC(t0, t1, t2, u[9], m[5]); + MONT_MAC(v0, v1, v2, u[10], m[4]); + MONT_MAC(w0, w1, w2, u[11], m[3]); + MONT_MAC(t0, t1, t2, u[12], m[2]); + MONT_MAC(v0, v1, v2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MAC(t0, t1, t2, a[0], b[15]); + MONT_MUL(v0, v1, v2, a[1], b[14]); + MONT_MUL(w0, w1, w2, a[2], b[13]); + MONT_MAC(t0, t1, t2, a[3], b[12]); + MONT_MAC(v0, v1, v2, a[4], b[11]); + MONT_MAC(w0, w1, w2, a[5], b[10]); + MONT_MAC(t0, t1, t2, a[6], b[9]); + MONT_MAC(v0, v1, v2, a[7], b[8]); + MONT_MAC(w0, w1, w2, a[8], b[7]); + MONT_MAC(t0, t1, t2, a[9], b[6]); + MONT_MAC(v0, v1, v2, a[10], b[5]); + MONT_MAC(w0, w1, w2, a[11], b[4]); + MONT_MAC(t0, t1, t2, a[12], b[3]); + MONT_MAC(v0, v1, v2, a[13], b[2]); + MONT_MAC(w0, w1, w2, a[14], b[1]); + MONT_MAC(t0, t1, t2, a[15], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[15]); + MONT_MAC(w0, w1, w2, u[1], m[14]); + MONT_MAC(t0, t1, t2, u[2], m[13]); + MONT_MAC(v0, v1, v2, u[3], m[12]); + MONT_MAC(w0, w1, w2, u[4], m[11]); + MONT_MAC(t0, t1, t2, u[5], m[10]); + MONT_MAC(v0, v1, v2, u[6], m[9]); + MONT_MAC(w0, w1, w2, u[7], m[8]); + MONT_MAC(t0, t1, t2, u[8], m[7]); + MONT_MAC(v0, v1, v2, u[9], m[6]); + MONT_MAC(w0, w1, w2, u[10], m[5]); + MONT_MAC(t0, t1, t2, u[11], m[4]); + MONT_MAC(v0, v1, v2, u[12], m[3]); + MONT_MAC(w0, w1, w2, u[13], m[2]); + MONT_MAC(t0, t1, t2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MAC(t0, t1, t2, a[0], b[16]); + MONT_MUL(v0, v1, v2, a[1], b[15]); + MONT_MUL(w0, w1, w2, a[2], b[14]); + MONT_MAC(t0, t1, t2, a[3], b[13]); + MONT_MAC(v0, v1, v2, a[4], b[12]); + MONT_MAC(w0, w1, w2, a[5], b[11]); + MONT_MAC(t0, t1, t2, a[6], b[10]); + MONT_MAC(v0, v1, v2, a[7], b[9]); + MONT_MAC(w0, w1, w2, a[8], b[8]); + MONT_MAC(t0, t1, t2, a[9], b[7]); + MONT_MAC(v0, v1, v2, a[10], b[6]); + MONT_MAC(w0, w1, w2, a[11], b[5]); + MONT_MAC(t0, t1, t2, a[12], b[4]); + MONT_MAC(v0, v1, v2, a[13], b[3]); + MONT_MAC(w0, w1, w2, a[14], b[2]); + MONT_MAC(t0, t1, t2, a[15], b[1]); + MONT_MAC(v0, v1, v2, a[16], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[16]); + MONT_MAC(t0, t1, t2, u[1], m[15]); + MONT_MAC(v0, v1, v2, u[2], m[14]); + MONT_MAC(w0, w1, w2, u[3], m[13]); + MONT_MAC(t0, t1, t2, u[4], m[12]); + MONT_MAC(v0, v1, v2, u[5], m[11]); + MONT_MAC(w0, w1, w2, u[6], m[10]); + MONT_MAC(t0, t1, t2, u[7], m[9]); + MONT_MAC(v0, v1, v2, u[8], m[8]); + MONT_MAC(w0, w1, w2, u[9], m[7]); + MONT_MAC(t0, t1, t2, u[10], m[6]); + MONT_MAC(v0, v1, v2, u[11], m[5]); + MONT_MAC(w0, w1, w2, u[12], m[4]); + MONT_MAC(t0, t1, t2, u[13], m[3]); + MONT_MAC(v0, v1, v2, u[14], m[2]); + MONT_MAC(w0, w1, w2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[16] = t0 * n0; + MONT_MAC(t0, t1, t2, u[16], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MAC(t0, t1, t2, a[0], b[17]); + MONT_MUL(v0, v1, v2, a[1], b[16]); + MONT_MUL(w0, w1, w2, a[2], b[15]); + MONT_MAC(t0, t1, t2, a[3], b[14]); + MONT_MAC(v0, v1, v2, a[4], b[13]); + MONT_MAC(w0, w1, w2, a[5], b[12]); + MONT_MAC(t0, t1, t2, a[6], b[11]); + MONT_MAC(v0, v1, v2, a[7], b[10]); + MONT_MAC(w0, w1, w2, a[8], b[9]); + MONT_MAC(t0, t1, t2, a[9], b[8]); + MONT_MAC(v0, v1, v2, a[10], b[7]); + MONT_MAC(w0, w1, w2, a[11], b[6]); + MONT_MAC(t0, t1, t2, a[12], b[5]); + MONT_MAC(v0, v1, v2, a[13], b[4]); + MONT_MAC(w0, w1, w2, a[14], b[3]); + MONT_MAC(t0, t1, t2, a[15], b[2]); + MONT_MAC(v0, v1, v2, a[16], b[1]); + MONT_MAC(w0, w1, w2, a[17], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[17]); + MONT_MAC(v0, v1, v2, u[1], m[16]); + MONT_MAC(w0, w1, w2, u[2], m[15]); + MONT_MAC(t0, t1, t2, u[3], m[14]); + MONT_MAC(v0, v1, v2, u[4], m[13]); + MONT_MAC(w0, w1, w2, u[5], m[12]); + MONT_MAC(t0, t1, t2, u[6], m[11]); + MONT_MAC(v0, v1, v2, u[7], m[10]); + MONT_MAC(w0, w1, w2, u[8], m[9]); + MONT_MAC(t0, t1, t2, u[9], m[8]); + MONT_MAC(v0, v1, v2, u[10], m[7]); + MONT_MAC(w0, w1, w2, u[11], m[6]); + MONT_MAC(t0, t1, t2, u[12], m[5]); + MONT_MAC(v0, v1, v2, u[13], m[4]); + MONT_MAC(w0, w1, w2, u[14], m[3]); + MONT_MAC(t0, t1, t2, u[15], m[2]); + MONT_MAC(v0, v1, v2, u[16], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[17] = t0 * n0; + MONT_MAC(t0, t1, t2, u[17], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MAC(t0, t1, t2, a[0], b[18]); + MONT_MUL(v0, v1, v2, a[1], b[17]); + MONT_MUL(w0, w1, w2, a[2], b[16]); + MONT_MAC(t0, t1, t2, a[3], b[15]); + MONT_MAC(v0, v1, v2, a[4], b[14]); + MONT_MAC(w0, w1, w2, a[5], b[13]); + MONT_MAC(t0, t1, t2, a[6], b[12]); + MONT_MAC(v0, v1, v2, a[7], b[11]); + MONT_MAC(w0, w1, w2, a[8], b[10]); + MONT_MAC(t0, t1, t2, a[9], b[9]); + MONT_MAC(v0, v1, v2, a[10], b[8]); + MONT_MAC(w0, w1, w2, a[11], b[7]); + MONT_MAC(t0, t1, t2, a[12], b[6]); + MONT_MAC(v0, v1, v2, a[13], b[5]); + MONT_MAC(w0, w1, w2, a[14], b[4]); + MONT_MAC(t0, t1, t2, a[15], b[3]); + MONT_MAC(v0, v1, v2, a[16], b[2]); + MONT_MAC(w0, w1, w2, a[17], b[1]); + MONT_MAC(t0, t1, t2, a[18], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[18]); + MONT_MAC(w0, w1, w2, u[1], m[17]); + MONT_MAC(t0, t1, t2, u[2], m[16]); + MONT_MAC(v0, v1, v2, u[3], m[15]); + MONT_MAC(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_MAC(w0, w1, w2, u[16], m[2]); + MONT_MAC(t0, t1, t2, u[17], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[18] = t0 * n0; + MONT_MAC(t0, t1, t2, u[18], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MAC(t0, t1, t2, a[0], b[19]); + MONT_MUL(v0, v1, v2, a[1], b[18]); + MONT_MUL(w0, w1, w2, a[2], b[17]); + MONT_MAC(t0, t1, t2, a[3], b[16]); + MONT_MAC(v0, v1, v2, a[4], b[15]); + MONT_MAC(w0, w1, w2, a[5], b[14]); + MONT_MAC(t0, t1, t2, a[6], b[13]); + MONT_MAC(v0, v1, v2, a[7], b[12]); + MONT_MAC(w0, w1, w2, a[8], b[11]); + MONT_MAC(t0, t1, t2, a[9], b[10]); + MONT_MAC(v0, v1, v2, a[10], b[9]); + MONT_MAC(w0, w1, w2, a[11], b[8]); + MONT_MAC(t0, t1, t2, a[12], b[7]); + MONT_MAC(v0, v1, v2, a[13], b[6]); + MONT_MAC(w0, w1, w2, a[14], b[5]); + MONT_MAC(t0, t1, t2, a[15], b[4]); + MONT_MAC(v0, v1, v2, a[16], b[3]); + MONT_MAC(w0, w1, w2, a[17], b[2]); + MONT_MAC(t0, t1, t2, a[18], b[1]); + MONT_MAC(v0, v1, v2, a[19], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[19]); + MONT_MAC(t0, t1, t2, u[1], m[18]); + MONT_MAC(v0, v1, v2, u[2], m[17]); + MONT_MAC(w0, w1, w2, u[3], m[16]); + MONT_MAC(t0, t1, t2, u[4], m[15]); + MONT_MAC(v0, v1, v2, u[5], m[14]); + MONT_MAC(w0, w1, w2, u[6], m[13]); + MONT_MAC(t0, t1, t2, u[7], m[12]); + MONT_MAC(v0, v1, v2, u[8], m[11]); + MONT_MAC(w0, w1, w2, u[9], m[10]); + MONT_MAC(t0, t1, t2, u[10], m[9]); + MONT_MAC(v0, v1, v2, u[11], m[8]); + MONT_MAC(w0, w1, w2, u[12], m[7]); + MONT_MAC(t0, t1, t2, u[13], m[6]); + MONT_MAC(v0, v1, v2, u[14], m[5]); + MONT_MAC(w0, w1, w2, u[15], m[4]); + MONT_MAC(t0, t1, t2, u[16], m[3]); + MONT_MAC(v0, v1, v2, u[17], m[2]); + MONT_MAC(w0, w1, w2, u[18], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[19] = t0 * n0; + MONT_MAC(t0, t1, t2, u[19], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MAC(t0, t1, t2, a[0], b[20]); + MONT_MUL(v0, v1, v2, a[1], b[19]); + MONT_MUL(w0, w1, w2, a[2], b[18]); + MONT_MAC(t0, t1, t2, a[3], b[17]); + MONT_MAC(v0, v1, v2, a[4], b[16]); + MONT_MAC(w0, w1, w2, a[5], b[15]); + MONT_MAC(t0, t1, t2, a[6], b[14]); + MONT_MAC(v0, v1, v2, a[7], b[13]); + MONT_MAC(w0, w1, w2, a[8], b[12]); + MONT_MAC(t0, t1, t2, a[9], b[11]); + MONT_MAC(v0, v1, v2, a[10], b[10]); + MONT_MAC(w0, w1, w2, a[11], b[9]); + MONT_MAC(t0, t1, t2, a[12], b[8]); + MONT_MAC(v0, v1, v2, a[13], b[7]); + MONT_MAC(w0, w1, w2, a[14], b[6]); + MONT_MAC(t0, t1, t2, a[15], b[5]); + MONT_MAC(v0, v1, v2, a[16], b[4]); + MONT_MAC(w0, w1, w2, a[17], b[3]); + MONT_MAC(t0, t1, t2, a[18], b[2]); + MONT_MAC(v0, v1, v2, a[19], b[1]); + MONT_MAC(w0, w1, w2, a[20], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[20]); + MONT_MAC(v0, v1, v2, u[1], m[19]); + MONT_MAC(w0, w1, w2, u[2], m[18]); + MONT_MAC(t0, t1, t2, u[3], m[17]); + MONT_MAC(v0, v1, v2, u[4], m[16]); + MONT_MAC(w0, w1, w2, u[5], m[15]); + MONT_MAC(t0, t1, t2, u[6], m[14]); + MONT_MAC(v0, v1, v2, u[7], m[13]); + MONT_MAC(w0, w1, w2, u[8], m[12]); + MONT_MAC(t0, t1, t2, u[9], m[11]); + MONT_MAC(v0, v1, v2, u[10], m[10]); + MONT_MAC(w0, w1, w2, u[11], m[9]); + MONT_MAC(t0, t1, t2, u[12], m[8]); + MONT_MAC(v0, v1, v2, u[13], m[7]); + MONT_MAC(w0, w1, w2, u[14], m[6]); + MONT_MAC(t0, t1, t2, u[15], m[5]); + MONT_MAC(v0, v1, v2, u[16], m[4]); + MONT_MAC(w0, w1, w2, u[17], m[3]); + MONT_MAC(t0, t1, t2, u[18], m[2]); + MONT_MAC(v0, v1, v2, u[19], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[20] = t0 * n0; + MONT_MAC(t0, t1, t2, u[20], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MAC(t0, t1, t2, a[0], b[21]); + MONT_MUL(v0, v1, v2, a[1], b[20]); + MONT_MUL(w0, w1, w2, a[2], b[19]); + MONT_MAC(t0, t1, t2, a[3], b[18]); + MONT_MAC(v0, v1, v2, a[4], b[17]); + MONT_MAC(w0, w1, w2, a[5], b[16]); + MONT_MAC(t0, t1, t2, a[6], b[15]); + MONT_MAC(v0, v1, v2, a[7], b[14]); + MONT_MAC(w0, w1, w2, a[8], b[13]); + MONT_MAC(t0, t1, t2, a[9], b[12]); + MONT_MAC(v0, v1, v2, a[10], b[11]); + MONT_MAC(w0, w1, w2, a[11], b[10]); + MONT_MAC(t0, t1, t2, a[12], b[9]); + MONT_MAC(v0, v1, v2, a[13], b[8]); + MONT_MAC(w0, w1, w2, a[14], b[7]); + MONT_MAC(t0, t1, t2, a[15], b[6]); + MONT_MAC(v0, v1, v2, a[16], b[5]); + MONT_MAC(w0, w1, w2, a[17], b[4]); + MONT_MAC(t0, t1, t2, a[18], b[3]); + MONT_MAC(v0, v1, v2, a[19], b[2]); + MONT_MAC(w0, w1, w2, a[20], b[1]); + MONT_MAC(t0, t1, t2, a[21], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[21]); + MONT_MAC(w0, w1, w2, u[1], m[20]); + MONT_MAC(t0, t1, t2, u[2], m[19]); + MONT_MAC(v0, v1, v2, u[3], m[18]); + MONT_MAC(w0, w1, w2, u[4], m[17]); + MONT_MAC(t0, t1, t2, u[5], m[16]); + MONT_MAC(v0, v1, v2, u[6], m[15]); + MONT_MAC(w0, w1, w2, u[7], m[14]); + MONT_MAC(t0, t1, t2, u[8], m[13]); + MONT_MAC(v0, v1, v2, u[9], m[12]); + MONT_MAC(w0, w1, w2, u[10], m[11]); + MONT_MAC(t0, t1, t2, u[11], m[10]); + MONT_MAC(v0, v1, v2, u[12], m[9]); + MONT_MAC(w0, w1, w2, u[13], m[8]); + MONT_MAC(t0, t1, t2, u[14], m[7]); + MONT_MAC(v0, v1, v2, u[15], m[6]); + MONT_MAC(w0, w1, w2, u[16], m[5]); + MONT_MAC(t0, t1, t2, u[17], m[4]); + MONT_MAC(v0, v1, v2, u[18], m[3]); + MONT_MAC(w0, w1, w2, u[19], m[2]); + MONT_MAC(t0, t1, t2, u[20], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[21] = t0 * n0; + MONT_MAC(t0, t1, t2, u[21], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MAC(t0, t1, t2, a[0], b[22]); + MONT_MUL(v0, v1, v2, a[1], b[21]); + MONT_MUL(w0, w1, w2, a[2], b[20]); + MONT_MAC(t0, t1, t2, a[3], b[19]); + MONT_MAC(v0, v1, v2, a[4], b[18]); + MONT_MAC(w0, w1, w2, a[5], b[17]); + MONT_MAC(t0, t1, t2, a[6], b[16]); + MONT_MAC(v0, v1, v2, a[7], b[15]); + MONT_MAC(w0, w1, w2, a[8], b[14]); + MONT_MAC(t0, t1, t2, a[9], b[13]); + MONT_MAC(v0, v1, v2, a[10], b[12]); + MONT_MAC(w0, w1, w2, a[11], b[11]); + MONT_MAC(t0, t1, t2, a[12], b[10]); + MONT_MAC(v0, v1, v2, a[13], b[9]); + MONT_MAC(w0, w1, w2, a[14], b[8]); + MONT_MAC(t0, t1, t2, a[15], b[7]); + MONT_MAC(v0, v1, v2, a[16], b[6]); + MONT_MAC(w0, w1, w2, a[17], b[5]); + MONT_MAC(t0, t1, t2, a[18], b[4]); + MONT_MAC(v0, v1, v2, a[19], b[3]); + MONT_MAC(w0, w1, w2, a[20], b[2]); + MONT_MAC(t0, t1, t2, a[21], b[1]); + MONT_MAC(v0, v1, v2, a[22], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[22]); + MONT_MAC(t0, t1, t2, u[1], m[21]); + MONT_MAC(v0, v1, v2, u[2], m[20]); + MONT_MAC(w0, w1, w2, u[3], m[19]); + MONT_MAC(t0, t1, t2, u[4], m[18]); + MONT_MAC(v0, v1, v2, u[5], m[17]); + MONT_MAC(w0, w1, w2, u[6], m[16]); + MONT_MAC(t0, t1, t2, u[7], m[15]); + MONT_MAC(v0, v1, v2, u[8], m[14]); + MONT_MAC(w0, w1, w2, u[9], m[13]); + MONT_MAC(t0, t1, t2, u[10], m[12]); + MONT_MAC(v0, v1, v2, u[11], m[11]); + MONT_MAC(w0, w1, w2, u[12], m[10]); + MONT_MAC(t0, t1, t2, u[13], m[9]); + MONT_MAC(v0, v1, v2, u[14], m[8]); + MONT_MAC(w0, w1, w2, u[15], m[7]); + MONT_MAC(t0, t1, t2, u[16], m[6]); + MONT_MAC(v0, v1, v2, u[17], m[5]); + MONT_MAC(w0, w1, w2, u[18], m[4]); + MONT_MAC(t0, t1, t2, u[19], m[3]); + MONT_MAC(v0, v1, v2, u[20], m[2]); + MONT_MAC(w0, w1, w2, u[21], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[22] = t0 * n0; + MONT_MAC(t0, t1, t2, u[22], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MAC(t0, t1, t2, a[0], b[23]); + MONT_MUL(v0, v1, v2, a[1], b[22]); + MONT_MUL(w0, w1, w2, a[2], b[21]); + MONT_MAC(t0, t1, t2, a[3], b[20]); + MONT_MAC(v0, v1, v2, a[4], b[19]); + MONT_MAC(w0, w1, w2, a[5], b[18]); + MONT_MAC(t0, t1, t2, a[6], b[17]); + MONT_MAC(v0, v1, v2, a[7], b[16]); + MONT_MAC(w0, w1, w2, a[8], b[15]); + MONT_MAC(t0, t1, t2, a[9], b[14]); + MONT_MAC(v0, v1, v2, a[10], b[13]); + MONT_MAC(w0, w1, w2, a[11], b[12]); + MONT_MAC(t0, t1, t2, a[12], b[11]); + MONT_MAC(v0, v1, v2, a[13], b[10]); + MONT_MAC(w0, w1, w2, a[14], b[9]); + MONT_MAC(t0, t1, t2, a[15], b[8]); + MONT_MAC(v0, v1, v2, a[16], b[7]); + MONT_MAC(w0, w1, w2, a[17], b[6]); + MONT_MAC(t0, t1, t2, a[18], b[5]); + MONT_MAC(v0, v1, v2, a[19], b[4]); + MONT_MAC(w0, w1, w2, a[20], b[3]); + MONT_MAC(t0, t1, t2, a[21], b[2]); + MONT_MAC(v0, v1, v2, a[22], b[1]); + MONT_MAC(w0, w1, w2, a[23], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[23]); + MONT_MAC(v0, v1, v2, u[1], m[22]); + MONT_MAC(w0, w1, w2, u[2], m[21]); + MONT_MAC(t0, t1, t2, u[3], m[20]); + MONT_MAC(v0, v1, v2, u[4], m[19]); + MONT_MAC(w0, w1, w2, u[5], m[18]); + MONT_MAC(t0, t1, t2, u[6], m[17]); + MONT_MAC(v0, v1, v2, u[7], m[16]); + MONT_MAC(w0, w1, w2, u[8], m[15]); + MONT_MAC(t0, t1, t2, u[9], m[14]); + MONT_MAC(v0, v1, v2, u[10], m[13]); + MONT_MAC(w0, w1, w2, u[11], m[12]); + MONT_MAC(t0, t1, t2, u[12], m[11]); + MONT_MAC(v0, v1, v2, u[13], m[10]); + MONT_MAC(w0, w1, w2, u[14], m[9]); + MONT_MAC(t0, t1, t2, u[15], m[8]); + MONT_MAC(v0, v1, v2, u[16], m[7]); + MONT_MAC(w0, w1, w2, u[17], m[6]); + MONT_MAC(t0, t1, t2, u[18], m[5]); + MONT_MAC(v0, v1, v2, u[19], m[4]); + MONT_MAC(w0, w1, w2, u[20], m[3]); + MONT_MAC(t0, t1, t2, u[21], m[2]); + MONT_MAC(v0, v1, v2, u[22], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[23] = t0 * n0; + MONT_MAC(t0, t1, t2, u[23], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MAC(t0, t1, t2, a[1], b[23]); + MONT_MUL(v0, v1, v2, a[2], b[22]); + MONT_MUL(w0, w1, w2, a[3], b[21]); + MONT_MAC(t0, t1, t2, a[4], b[20]); + MONT_MAC(v0, v1, v2, a[5], b[19]); + MONT_MAC(w0, w1, w2, a[6], b[18]); + MONT_MAC(t0, t1, t2, a[7], b[17]); + MONT_MAC(v0, v1, v2, a[8], b[16]); + MONT_MAC(w0, w1, w2, a[9], b[15]); + MONT_MAC(t0, t1, t2, a[10], b[14]); + MONT_MAC(v0, v1, v2, a[11], b[13]); + MONT_MAC(w0, w1, w2, a[12], b[12]); + MONT_MAC(t0, t1, t2, a[13], b[11]); + MONT_MAC(v0, v1, v2, a[14], b[10]); + MONT_MAC(w0, w1, w2, a[15], b[9]); + MONT_MAC(t0, t1, t2, a[16], b[8]); + MONT_MAC(v0, v1, v2, a[17], b[7]); + MONT_MAC(w0, w1, w2, a[18], b[6]); + MONT_MAC(t0, t1, t2, a[19], b[5]); + MONT_MAC(v0, v1, v2, a[20], b[4]); + MONT_MAC(w0, w1, w2, a[21], b[3]); + MONT_MAC(t0, t1, t2, a[22], b[2]); + MONT_MAC(v0, v1, v2, a[23], b[1]); + MONT_MAC(w0, w1, w2, u[1], m[23]); + MONT_MAC(t0, t1, t2, u[2], m[22]); + MONT_MAC(v0, v1, v2, u[3], m[21]); + MONT_MAC(w0, w1, w2, u[4], m[20]); + MONT_MAC(t0, t1, t2, u[5], m[19]); + MONT_MAC(v0, v1, v2, u[6], m[18]); + MONT_MAC(w0, w1, w2, u[7], m[17]); + MONT_MAC(t0, t1, t2, u[8], m[16]); + MONT_MAC(v0, v1, v2, u[9], m[15]); + MONT_MAC(w0, w1, w2, u[10], m[14]); + MONT_MAC(t0, t1, t2, u[11], m[13]); + MONT_MAC(v0, v1, v2, u[12], m[12]); + MONT_MAC(w0, w1, w2, u[13], m[11]); + MONT_MAC(t0, t1, t2, u[14], m[10]); + MONT_MAC(v0, v1, v2, u[15], m[9]); + MONT_MAC(w0, w1, w2, u[16], m[8]); + MONT_MAC(t0, t1, t2, u[17], m[7]); + MONT_MAC(v0, v1, v2, u[18], m[6]); + MONT_MAC(w0, w1, w2, u[19], m[5]); + MONT_MAC(t0, t1, t2, u[20], m[4]); + MONT_MAC(v0, v1, v2, u[21], m[3]); + MONT_MAC(w0, w1, w2, u[22], m[2]); + MONT_MAC(t0, t1, t2, u[23], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MAC(t0, t1, t2, a[2], b[23]); + MONT_MUL(v0, v1, v2, a[3], b[22]); + MONT_MUL(w0, w1, w2, a[4], b[21]); + MONT_MAC(t0, t1, t2, a[5], b[20]); + MONT_MAC(v0, v1, v2, a[6], b[19]); + MONT_MAC(w0, w1, w2, a[7], b[18]); + MONT_MAC(t0, t1, t2, a[8], b[17]); + MONT_MAC(v0, v1, v2, a[9], b[16]); + MONT_MAC(w0, w1, w2, a[10], b[15]); + MONT_MAC(t0, t1, t2, a[11], b[14]); + MONT_MAC(v0, v1, v2, a[12], b[13]); + MONT_MAC(w0, w1, w2, a[13], b[12]); + MONT_MAC(t0, t1, t2, a[14], b[11]); + MONT_MAC(v0, v1, v2, a[15], b[10]); + MONT_MAC(w0, w1, w2, a[16], b[9]); + MONT_MAC(t0, t1, t2, a[17], b[8]); + MONT_MAC(v0, v1, v2, a[18], b[7]); + MONT_MAC(w0, w1, w2, a[19], b[6]); + MONT_MAC(t0, t1, t2, a[20], b[5]); + MONT_MAC(v0, v1, v2, a[21], b[4]); + MONT_MAC(w0, w1, w2, a[22], b[3]); + MONT_MAC(t0, t1, t2, a[23], b[2]); + MONT_MAC(v0, v1, v2, u[2], m[23]); + MONT_MAC(w0, w1, w2, u[3], m[22]); + MONT_MAC(t0, t1, t2, u[4], m[21]); + MONT_MAC(v0, v1, v2, u[5], m[20]); + MONT_MAC(w0, w1, w2, u[6], m[19]); + MONT_MAC(t0, t1, t2, u[7], m[18]); + MONT_MAC(v0, v1, v2, u[8], m[17]); + MONT_MAC(w0, w1, w2, u[9], m[16]); + MONT_MAC(t0, t1, t2, u[10], m[15]); + MONT_MAC(v0, v1, v2, u[11], m[14]); + MONT_MAC(w0, w1, w2, u[12], m[13]); + MONT_MAC(t0, t1, t2, u[13], m[12]); + MONT_MAC(v0, v1, v2, u[14], m[11]); + MONT_MAC(w0, w1, w2, u[15], m[10]); + MONT_MAC(t0, t1, t2, u[16], m[9]); + MONT_MAC(v0, v1, v2, u[17], m[8]); + MONT_MAC(w0, w1, w2, u[18], m[7]); + MONT_MAC(t0, t1, t2, u[19], m[6]); + MONT_MAC(v0, v1, v2, u[20], m[5]); + MONT_MAC(w0, w1, w2, u[21], m[4]); + MONT_MAC(t0, t1, t2, u[22], m[3]); + MONT_MAC(v0, v1, v2, u[23], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MAC(t0, t1, t2, a[3], b[23]); + MONT_MUL(v0, v1, v2, a[4], b[22]); + MONT_MUL(w0, w1, w2, a[5], b[21]); + MONT_MAC(t0, t1, t2, a[6], b[20]); + MONT_MAC(v0, v1, v2, a[7], b[19]); + MONT_MAC(w0, w1, w2, a[8], b[18]); + MONT_MAC(t0, t1, t2, a[9], b[17]); + MONT_MAC(v0, v1, v2, a[10], b[16]); + MONT_MAC(w0, w1, w2, a[11], b[15]); + MONT_MAC(t0, t1, t2, a[12], b[14]); + MONT_MAC(v0, v1, v2, a[13], b[13]); + MONT_MAC(w0, w1, w2, a[14], b[12]); + MONT_MAC(t0, t1, t2, a[15], b[11]); + MONT_MAC(v0, v1, v2, a[16], b[10]); + MONT_MAC(w0, w1, w2, a[17], b[9]); + MONT_MAC(t0, t1, t2, a[18], b[8]); + MONT_MAC(v0, v1, v2, a[19], b[7]); + MONT_MAC(w0, w1, w2, a[20], b[6]); + MONT_MAC(t0, t1, t2, a[21], b[5]); + MONT_MAC(v0, v1, v2, a[22], b[4]); + MONT_MAC(w0, w1, w2, a[23], b[3]); + MONT_MAC(t0, t1, t2, u[3], m[23]); + MONT_MAC(v0, v1, v2, u[4], m[22]); + MONT_MAC(w0, w1, w2, u[5], m[21]); + MONT_MAC(t0, t1, t2, u[6], m[20]); + MONT_MAC(v0, v1, v2, u[7], m[19]); + MONT_MAC(w0, w1, w2, u[8], m[18]); + MONT_MAC(t0, t1, t2, u[9], m[17]); + MONT_MAC(v0, v1, v2, u[10], m[16]); + MONT_MAC(w0, w1, w2, u[11], m[15]); + MONT_MAC(t0, t1, t2, u[12], m[14]); + MONT_MAC(v0, v1, v2, u[13], m[13]); + MONT_MAC(w0, w1, w2, u[14], m[12]); + MONT_MAC(t0, t1, t2, u[15], m[11]); + MONT_MAC(v0, v1, v2, u[16], m[10]); + MONT_MAC(w0, w1, w2, u[17], m[9]); + MONT_MAC(t0, t1, t2, u[18], m[8]); + MONT_MAC(v0, v1, v2, u[19], m[7]); + MONT_MAC(w0, w1, w2, u[20], m[6]); + MONT_MAC(t0, t1, t2, u[21], m[5]); + MONT_MAC(v0, v1, v2, u[22], m[4]); + MONT_MAC(w0, w1, w2, u[23], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MAC(t0, t1, t2, a[4], b[23]); + MONT_MUL(v0, v1, v2, a[5], b[22]); + MONT_MUL(w0, w1, w2, a[6], b[21]); + MONT_MAC(t0, t1, t2, a[7], b[20]); + MONT_MAC(v0, v1, v2, a[8], b[19]); + MONT_MAC(w0, w1, w2, a[9], b[18]); + MONT_MAC(t0, t1, t2, a[10], b[17]); + MONT_MAC(v0, v1, v2, a[11], b[16]); + MONT_MAC(w0, w1, w2, a[12], b[15]); + MONT_MAC(t0, t1, t2, a[13], b[14]); + MONT_MAC(v0, v1, v2, a[14], b[13]); + MONT_MAC(w0, w1, w2, a[15], b[12]); + MONT_MAC(t0, t1, t2, a[16], b[11]); + MONT_MAC(v0, v1, v2, a[17], b[10]); + MONT_MAC(w0, w1, w2, a[18], b[9]); + MONT_MAC(t0, t1, t2, a[19], b[8]); + MONT_MAC(v0, v1, v2, a[20], b[7]); + MONT_MAC(w0, w1, w2, a[21], b[6]); + MONT_MAC(t0, t1, t2, a[22], b[5]); + MONT_MAC(v0, v1, v2, a[23], b[4]); + MONT_MAC(w0, w1, w2, u[4], m[23]); + MONT_MAC(t0, t1, t2, u[5], m[22]); + MONT_MAC(v0, v1, v2, u[6], m[21]); + MONT_MAC(w0, w1, w2, u[7], m[20]); + MONT_MAC(t0, t1, t2, u[8], m[19]); + MONT_MAC(v0, v1, v2, u[9], m[18]); + MONT_MAC(w0, w1, w2, u[10], m[17]); + MONT_MAC(t0, t1, t2, u[11], m[16]); + MONT_MAC(v0, v1, v2, u[12], m[15]); + MONT_MAC(w0, w1, w2, u[13], m[14]); + MONT_MAC(t0, t1, t2, u[14], m[13]); + MONT_MAC(v0, v1, v2, u[15], m[12]); + MONT_MAC(w0, w1, w2, u[16], m[11]); + MONT_MAC(t0, t1, t2, u[17], m[10]); + MONT_MAC(v0, v1, v2, u[18], m[9]); + MONT_MAC(w0, w1, w2, u[19], m[8]); + MONT_MAC(t0, t1, t2, u[20], m[7]); + MONT_MAC(v0, v1, v2, u[21], m[6]); + MONT_MAC(w0, w1, w2, u[22], m[5]); + MONT_MAC(t0, t1, t2, u[23], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MAC(t0, t1, t2, a[5], b[23]); + MONT_MUL(v0, v1, v2, a[6], b[22]); + MONT_MUL(w0, w1, w2, a[7], b[21]); + MONT_MAC(t0, t1, t2, a[8], b[20]); + MONT_MAC(v0, v1, v2, a[9], b[19]); + MONT_MAC(w0, w1, w2, a[10], b[18]); + MONT_MAC(t0, t1, t2, a[11], b[17]); + MONT_MAC(v0, v1, v2, a[12], b[16]); + MONT_MAC(w0, w1, w2, a[13], b[15]); + MONT_MAC(t0, t1, t2, a[14], b[14]); + MONT_MAC(v0, v1, v2, a[15], b[13]); + MONT_MAC(w0, w1, w2, a[16], b[12]); + MONT_MAC(t0, t1, t2, a[17], b[11]); + MONT_MAC(v0, v1, v2, a[18], b[10]); + MONT_MAC(w0, w1, w2, a[19], b[9]); + MONT_MAC(t0, t1, t2, a[20], b[8]); + MONT_MAC(v0, v1, v2, a[21], b[7]); + MONT_MAC(w0, w1, w2, a[22], b[6]); + MONT_MAC(t0, t1, t2, a[23], b[5]); + MONT_MAC(v0, v1, v2, u[5], m[23]); + MONT_MAC(w0, w1, w2, u[6], m[22]); + MONT_MAC(t0, t1, t2, u[7], m[21]); + MONT_MAC(v0, v1, v2, u[8], m[20]); + MONT_MAC(w0, w1, w2, u[9], m[19]); + MONT_MAC(t0, t1, t2, u[10], m[18]); + MONT_MAC(v0, v1, v2, u[11], m[17]); + MONT_MAC(w0, w1, w2, u[12], m[16]); + MONT_MAC(t0, t1, t2, u[13], m[15]); + MONT_MAC(v0, v1, v2, u[14], m[14]); + MONT_MAC(w0, w1, w2, u[15], m[13]); + MONT_MAC(t0, t1, t2, u[16], m[12]); + MONT_MAC(v0, v1, v2, u[17], m[11]); + MONT_MAC(w0, w1, w2, u[18], m[10]); + MONT_MAC(t0, t1, t2, u[19], m[9]); + MONT_MAC(v0, v1, v2, u[20], m[8]); + MONT_MAC(w0, w1, w2, u[21], m[7]); + MONT_MAC(t0, t1, t2, u[22], m[6]); + MONT_MAC(v0, v1, v2, u[23], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MAC(t0, t1, t2, a[6], b[23]); + MONT_MUL(v0, v1, v2, a[7], b[22]); + MONT_MUL(w0, w1, w2, a[8], b[21]); + MONT_MAC(t0, t1, t2, a[9], b[20]); + MONT_MAC(v0, v1, v2, a[10], b[19]); + MONT_MAC(w0, w1, w2, a[11], b[18]); + MONT_MAC(t0, t1, t2, a[12], b[17]); + MONT_MAC(v0, v1, v2, a[13], b[16]); + MONT_MAC(w0, w1, w2, a[14], b[15]); + MONT_MAC(t0, t1, t2, a[15], b[14]); + MONT_MAC(v0, v1, v2, a[16], b[13]); + MONT_MAC(w0, w1, w2, a[17], b[12]); + MONT_MAC(t0, t1, t2, a[18], b[11]); + MONT_MAC(v0, v1, v2, a[19], b[10]); + MONT_MAC(w0, w1, w2, a[20], b[9]); + MONT_MAC(t0, t1, t2, a[21], b[8]); + MONT_MAC(v0, v1, v2, a[22], b[7]); + MONT_MAC(w0, w1, w2, a[23], b[6]); + MONT_MAC(t0, t1, t2, u[6], m[23]); + MONT_MAC(v0, v1, v2, u[7], m[22]); + MONT_MAC(w0, w1, w2, u[8], m[21]); + MONT_MAC(t0, t1, t2, u[9], m[20]); + MONT_MAC(v0, v1, v2, u[10], m[19]); + MONT_MAC(w0, w1, w2, u[11], m[18]); + MONT_MAC(t0, t1, t2, u[12], m[17]); + MONT_MAC(v0, v1, v2, u[13], m[16]); + MONT_MAC(w0, w1, w2, u[14], m[15]); + MONT_MAC(t0, t1, t2, u[15], m[14]); + MONT_MAC(v0, v1, v2, u[16], m[13]); + MONT_MAC(w0, w1, w2, u[17], m[12]); + MONT_MAC(t0, t1, t2, u[18], m[11]); + MONT_MAC(v0, v1, v2, u[19], m[10]); + MONT_MAC(w0, w1, w2, u[20], m[9]); + MONT_MAC(t0, t1, t2, u[21], m[8]); + MONT_MAC(v0, v1, v2, u[22], m[7]); + MONT_MAC(w0, w1, w2, u[23], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MAC(t0, t1, t2, a[7], b[23]); + MONT_MUL(v0, v1, v2, a[8], b[22]); + MONT_MUL(w0, w1, w2, a[9], b[21]); + MONT_MAC(t0, t1, t2, a[10], b[20]); + MONT_MAC(v0, v1, v2, a[11], b[19]); + MONT_MAC(w0, w1, w2, a[12], b[18]); + MONT_MAC(t0, t1, t2, a[13], b[17]); + MONT_MAC(v0, v1, v2, a[14], b[16]); + MONT_MAC(w0, w1, w2, a[15], b[15]); + MONT_MAC(t0, t1, t2, a[16], b[14]); + MONT_MAC(v0, v1, v2, a[17], b[13]); + MONT_MAC(w0, w1, w2, a[18], b[12]); + MONT_MAC(t0, t1, t2, a[19], b[11]); + MONT_MAC(v0, v1, v2, a[20], b[10]); + MONT_MAC(w0, w1, w2, a[21], b[9]); + MONT_MAC(t0, t1, t2, a[22], b[8]); + MONT_MAC(v0, v1, v2, a[23], b[7]); + MONT_MAC(w0, w1, w2, u[7], m[23]); + MONT_MAC(t0, t1, t2, u[8], m[22]); + MONT_MAC(v0, v1, v2, u[9], m[21]); + MONT_MAC(w0, w1, w2, u[10], m[20]); + MONT_MAC(t0, t1, t2, u[11], m[19]); + MONT_MAC(v0, v1, v2, u[12], m[18]); + MONT_MAC(w0, w1, w2, u[13], m[17]); + MONT_MAC(t0, t1, t2, u[14], m[16]); + MONT_MAC(v0, v1, v2, u[15], m[15]); + MONT_MAC(w0, w1, w2, u[16], m[14]); + MONT_MAC(t0, t1, t2, u[17], m[13]); + MONT_MAC(v0, v1, v2, u[18], m[12]); + MONT_MAC(w0, w1, w2, u[19], m[11]); + MONT_MAC(t0, t1, t2, u[20], m[10]); + MONT_MAC(v0, v1, v2, u[21], m[9]); + MONT_MAC(w0, w1, w2, u[22], m[8]); + MONT_MAC(t0, t1, t2, u[23], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 31 */ + MONT_MAC(t0, t1, t2, a[8], b[23]); + MONT_MUL(v0, v1, v2, a[9], b[22]); + MONT_MUL(w0, w1, w2, a[10], b[21]); + MONT_MAC(t0, t1, t2, a[11], b[20]); + MONT_MAC(v0, v1, v2, a[12], b[19]); + MONT_MAC(w0, w1, w2, a[13], b[18]); + MONT_MAC(t0, t1, t2, a[14], b[17]); + MONT_MAC(v0, v1, v2, a[15], b[16]); + MONT_MAC(w0, w1, w2, a[16], b[15]); + MONT_MAC(t0, t1, t2, a[17], b[14]); + MONT_MAC(v0, v1, v2, a[18], b[13]); + MONT_MAC(w0, w1, w2, a[19], b[12]); + MONT_MAC(t0, t1, t2, a[20], b[11]); + MONT_MAC(v0, v1, v2, a[21], b[10]); + MONT_MAC(w0, w1, w2, a[22], b[9]); + MONT_MAC(t0, t1, t2, a[23], b[8]); + MONT_MAC(v0, v1, v2, u[8], m[23]); + MONT_MAC(w0, w1, w2, u[9], m[22]); + MONT_MAC(t0, t1, t2, u[10], m[21]); + MONT_MAC(v0, v1, v2, u[11], m[20]); + MONT_MAC(w0, w1, w2, u[12], m[19]); + MONT_MAC(t0, t1, t2, u[13], m[18]); + MONT_MAC(v0, v1, v2, u[14], m[17]); + MONT_MAC(w0, w1, w2, u[15], m[16]); + MONT_MAC(t0, t1, t2, u[16], m[15]); + MONT_MAC(v0, v1, v2, u[17], m[14]); + MONT_MAC(w0, w1, w2, u[18], m[13]); + MONT_MAC(t0, t1, t2, u[19], m[12]); + MONT_MAC(v0, v1, v2, u[20], m[11]); + MONT_MAC(w0, w1, w2, u[21], m[10]); + MONT_MAC(t0, t1, t2, u[22], m[9]); + MONT_MAC(v0, v1, v2, u[23], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 32 */ + MONT_MAC(t0, t1, t2, a[9], b[23]); + MONT_MUL(v0, v1, v2, a[10], b[22]); + MONT_MUL(w0, w1, w2, a[11], b[21]); + MONT_MAC(t0, t1, t2, a[12], b[20]); + MONT_MAC(v0, v1, v2, a[13], b[19]); + MONT_MAC(w0, w1, w2, a[14], b[18]); + MONT_MAC(t0, t1, t2, a[15], b[17]); + MONT_MAC(v0, v1, v2, a[16], b[16]); + MONT_MAC(w0, w1, w2, a[17], b[15]); + MONT_MAC(t0, t1, t2, a[18], b[14]); + MONT_MAC(v0, v1, v2, a[19], b[13]); + MONT_MAC(w0, w1, w2, a[20], b[12]); + MONT_MAC(t0, t1, t2, a[21], b[11]); + MONT_MAC(v0, v1, v2, a[22], b[10]); + MONT_MAC(w0, w1, w2, a[23], b[9]); + MONT_MAC(t0, t1, t2, u[9], m[23]); + MONT_MAC(v0, v1, v2, u[10], m[22]); + MONT_MAC(w0, w1, w2, u[11], m[21]); + MONT_MAC(t0, t1, t2, u[12], m[20]); + MONT_MAC(v0, v1, v2, u[13], m[19]); + MONT_MAC(w0, w1, w2, u[14], m[18]); + MONT_MAC(t0, t1, t2, u[15], m[17]); + MONT_MAC(v0, v1, v2, u[16], m[16]); + MONT_MAC(w0, w1, w2, u[17], m[15]); + MONT_MAC(t0, t1, t2, u[18], m[14]); + MONT_MAC(v0, v1, v2, u[19], m[13]); + MONT_MAC(w0, w1, w2, u[20], m[12]); + MONT_MAC(t0, t1, t2, u[21], m[11]); + MONT_MAC(v0, v1, v2, u[22], m[10]); + MONT_MAC(w0, w1, w2, u[23], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 33 */ + MONT_MAC(t0, t1, t2, a[10], b[23]); + MONT_MUL(v0, v1, v2, a[11], b[22]); + MONT_MUL(w0, w1, w2, a[12], b[21]); + MONT_MAC(t0, t1, t2, a[13], b[20]); + MONT_MAC(v0, v1, v2, a[14], b[19]); + MONT_MAC(w0, w1, w2, a[15], b[18]); + MONT_MAC(t0, t1, t2, a[16], b[17]); + MONT_MAC(v0, v1, v2, a[17], b[16]); + MONT_MAC(w0, w1, w2, a[18], b[15]); + MONT_MAC(t0, t1, t2, a[19], b[14]); + MONT_MAC(v0, v1, v2, a[20], b[13]); + MONT_MAC(w0, w1, w2, a[21], b[12]); + MONT_MAC(t0, t1, t2, a[22], b[11]); + MONT_MAC(v0, v1, v2, a[23], b[10]); + MONT_MAC(w0, w1, w2, u[10], m[23]); + MONT_MAC(t0, t1, t2, u[11], m[22]); + MONT_MAC(v0, v1, v2, u[12], m[21]); + MONT_MAC(w0, w1, w2, u[13], m[20]); + MONT_MAC(t0, t1, t2, u[14], m[19]); + MONT_MAC(v0, v1, v2, u[15], m[18]); + MONT_MAC(w0, w1, w2, u[16], m[17]); + MONT_MAC(t0, t1, t2, u[17], m[16]); + MONT_MAC(v0, v1, v2, u[18], m[15]); + MONT_MAC(w0, w1, w2, u[19], m[14]); + MONT_MAC(t0, t1, t2, u[20], m[13]); + MONT_MAC(v0, v1, v2, u[21], m[12]); + MONT_MAC(w0, w1, w2, u[22], m[11]); + MONT_MAC(t0, t1, t2, u[23], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 34 */ + MONT_MAC(t0, t1, t2, a[11], b[23]); + MONT_MUL(v0, v1, v2, a[12], b[22]); + MONT_MUL(w0, w1, w2, a[13], b[21]); + MONT_MAC(t0, t1, t2, a[14], b[20]); + MONT_MAC(v0, v1, v2, a[15], b[19]); + MONT_MAC(w0, w1, w2, a[16], b[18]); + MONT_MAC(t0, t1, t2, a[17], b[17]); + MONT_MAC(v0, v1, v2, a[18], b[16]); + MONT_MAC(w0, w1, w2, a[19], b[15]); + MONT_MAC(t0, t1, t2, a[20], b[14]); + MONT_MAC(v0, v1, v2, a[21], b[13]); + MONT_MAC(w0, w1, w2, a[22], b[12]); + MONT_MAC(t0, t1, t2, a[23], b[11]); + MONT_MAC(v0, v1, v2, u[11], m[23]); + MONT_MAC(w0, w1, w2, u[12], m[22]); + MONT_MAC(t0, t1, t2, u[13], m[21]); + MONT_MAC(v0, v1, v2, u[14], m[20]); + MONT_MAC(w0, w1, w2, u[15], m[19]); + MONT_MAC(t0, t1, t2, u[16], m[18]); + MONT_MAC(v0, v1, v2, u[17], m[17]); + MONT_MAC(w0, w1, w2, u[18], m[16]); + MONT_MAC(t0, t1, t2, u[19], m[15]); + MONT_MAC(v0, v1, v2, u[20], m[14]); + MONT_MAC(w0, w1, w2, u[21], m[13]); + MONT_MAC(t0, t1, t2, u[22], m[12]); + MONT_MAC(v0, v1, v2, u[23], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 35 */ + MONT_MAC(t0, t1, t2, a[12], b[23]); + MONT_MUL(v0, v1, v2, a[13], b[22]); + MONT_MUL(w0, w1, w2, a[14], b[21]); + MONT_MAC(t0, t1, t2, a[15], b[20]); + MONT_MAC(v0, v1, v2, a[16], b[19]); + MONT_MAC(w0, w1, w2, a[17], b[18]); + MONT_MAC(t0, t1, t2, a[18], b[17]); + MONT_MAC(v0, v1, v2, a[19], b[16]); + MONT_MAC(w0, w1, w2, a[20], b[15]); + MONT_MAC(t0, t1, t2, a[21], b[14]); + MONT_MAC(v0, v1, v2, a[22], b[13]); + MONT_MAC(w0, w1, w2, a[23], b[12]); + MONT_MAC(t0, t1, t2, u[12], m[23]); + MONT_MAC(v0, v1, v2, u[13], m[22]); + MONT_MAC(w0, w1, w2, u[14], m[21]); + MONT_MAC(t0, t1, t2, u[15], m[20]); + MONT_MAC(v0, v1, v2, u[16], m[19]); + MONT_MAC(w0, w1, w2, u[17], m[18]); + MONT_MAC(t0, t1, t2, u[18], m[17]); + MONT_MAC(v0, v1, v2, u[19], m[16]); + MONT_MAC(w0, w1, w2, u[20], m[15]); + MONT_MAC(t0, t1, t2, u[21], m[14]); + MONT_MAC(v0, v1, v2, u[22], m[13]); + MONT_MAC(w0, w1, w2, u[23], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 36 */ + MONT_MAC(t0, t1, t2, a[13], b[23]); + MONT_MUL(v0, v1, v2, a[14], b[22]); + MONT_MUL(w0, w1, w2, a[15], b[21]); + MONT_MAC(t0, t1, t2, a[16], b[20]); + MONT_MAC(v0, v1, v2, a[17], b[19]); + MONT_MAC(w0, w1, w2, a[18], b[18]); + MONT_MAC(t0, t1, t2, a[19], b[17]); + MONT_MAC(v0, v1, v2, a[20], b[16]); + MONT_MAC(w0, w1, w2, a[21], b[15]); + MONT_MAC(t0, t1, t2, a[22], b[14]); + MONT_MAC(v0, v1, v2, a[23], b[13]); + MONT_MAC(w0, w1, w2, u[13], m[23]); + MONT_MAC(t0, t1, t2, u[14], m[22]); + MONT_MAC(v0, v1, v2, u[15], m[21]); + MONT_MAC(w0, w1, w2, u[16], m[20]); + MONT_MAC(t0, t1, t2, u[17], m[19]); + MONT_MAC(v0, v1, v2, u[18], m[18]); + MONT_MAC(w0, w1, w2, u[19], m[17]); + MONT_MAC(t0, t1, t2, u[20], m[16]); + MONT_MAC(v0, v1, v2, u[21], m[15]); + MONT_MAC(w0, w1, w2, u[22], m[14]); + MONT_MAC(t0, t1, t2, u[23], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 37 */ + MONT_MAC(t0, t1, t2, a[14], b[23]); + MONT_MUL(v0, v1, v2, a[15], b[22]); + MONT_MUL(w0, w1, w2, a[16], b[21]); + MONT_MAC(t0, t1, t2, a[17], b[20]); + MONT_MAC(v0, v1, v2, a[18], b[19]); + MONT_MAC(w0, w1, w2, a[19], b[18]); + MONT_MAC(t0, t1, t2, a[20], b[17]); + MONT_MAC(v0, v1, v2, a[21], b[16]); + MONT_MAC(w0, w1, w2, a[22], b[15]); + MONT_MAC(t0, t1, t2, a[23], b[14]); + MONT_MAC(v0, v1, v2, u[14], m[23]); + MONT_MAC(w0, w1, w2, u[15], m[22]); + MONT_MAC(t0, t1, t2, u[16], m[21]); + MONT_MAC(v0, v1, v2, u[17], m[20]); + MONT_MAC(w0, w1, w2, u[18], m[19]); + MONT_MAC(t0, t1, t2, u[19], m[18]); + MONT_MAC(v0, v1, v2, u[20], m[17]); + MONT_MAC(w0, w1, w2, u[21], m[16]); + MONT_MAC(t0, t1, t2, u[22], m[15]); + MONT_MAC(v0, v1, v2, u[23], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 38 */ + MONT_MAC(t0, t1, t2, a[15], b[23]); + MONT_MUL(v0, v1, v2, a[16], b[22]); + MONT_MUL(w0, w1, w2, a[17], b[21]); + MONT_MAC(t0, t1, t2, a[18], b[20]); + MONT_MAC(v0, v1, v2, a[19], b[19]); + MONT_MAC(w0, w1, w2, a[20], b[18]); + MONT_MAC(t0, t1, t2, a[21], b[17]); + MONT_MAC(v0, v1, v2, a[22], b[16]); + MONT_MAC(w0, w1, w2, a[23], b[15]); + MONT_MAC(t0, t1, t2, u[15], m[23]); + MONT_MAC(v0, v1, v2, u[16], m[22]); + MONT_MAC(w0, w1, w2, u[17], m[21]); + MONT_MAC(t0, t1, t2, u[18], m[20]); + MONT_MAC(v0, v1, v2, u[19], m[19]); + MONT_MAC(w0, w1, w2, u[20], m[18]); + MONT_MAC(t0, t1, t2, u[21], m[17]); + MONT_MAC(v0, v1, v2, u[22], m[16]); + MONT_MAC(w0, w1, w2, u[23], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 39 */ + MONT_MAC(t0, t1, t2, a[16], b[23]); + MONT_MUL(v0, v1, v2, a[17], b[22]); + MONT_MUL(w0, w1, w2, a[18], b[21]); + MONT_MAC(t0, t1, t2, a[19], b[20]); + MONT_MAC(v0, v1, v2, a[20], b[19]); + MONT_MAC(w0, w1, w2, a[21], b[18]); + MONT_MAC(t0, t1, t2, a[22], b[17]); + MONT_MAC(v0, v1, v2, a[23], b[16]); + MONT_MAC(w0, w1, w2, u[16], m[23]); + MONT_MAC(t0, t1, t2, u[17], m[22]); + MONT_MAC(v0, v1, v2, u[18], m[21]); + MONT_MAC(w0, w1, w2, u[19], m[20]); + MONT_MAC(t0, t1, t2, u[20], m[19]); + MONT_MAC(v0, v1, v2, u[21], m[18]); + MONT_MAC(w0, w1, w2, u[22], m[17]); + MONT_MAC(t0, t1, t2, u[23], m[16]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[15] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 40 */ + MONT_MAC(t0, t1, t2, a[17], b[23]); + MONT_MUL(v0, v1, v2, a[18], b[22]); + MONT_MUL(w0, w1, w2, a[19], b[21]); + MONT_MAC(t0, t1, t2, a[20], b[20]); + MONT_MAC(v0, v1, v2, a[21], b[19]); + MONT_MAC(w0, w1, w2, a[22], b[18]); + MONT_MAC(t0, t1, t2, a[23], b[17]); + MONT_MAC(v0, v1, v2, u[17], m[23]); + MONT_MAC(w0, w1, w2, u[18], m[22]); + MONT_MAC(t0, t1, t2, u[19], m[21]); + MONT_MAC(v0, v1, v2, u[20], m[20]); + MONT_MAC(w0, w1, w2, u[21], m[19]); + MONT_MAC(t0, t1, t2, u[22], m[18]); + MONT_MAC(v0, v1, v2, u[23], m[17]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[16] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 41 */ + MONT_MAC(t0, t1, t2, a[18], b[23]); + MONT_MUL(v0, v1, v2, a[19], b[22]); + MONT_MUL(w0, w1, w2, a[20], b[21]); + MONT_MAC(t0, t1, t2, a[21], b[20]); + MONT_MAC(v0, v1, v2, a[22], b[19]); + MONT_MAC(w0, w1, w2, a[23], b[18]); + MONT_MAC(t0, t1, t2, u[18], m[23]); + MONT_MAC(v0, v1, v2, u[19], m[22]); + MONT_MAC(w0, w1, w2, u[20], m[21]); + MONT_MAC(t0, t1, t2, u[21], m[20]); + MONT_MAC(v0, v1, v2, u[22], m[19]); + MONT_MAC(w0, w1, w2, u[23], m[18]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[17] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 42 */ + MONT_MAC(t0, t1, t2, a[19], b[23]); + MONT_MUL(v0, v1, v2, a[20], b[22]); + MONT_MUL(w0, w1, w2, a[21], b[21]); + MONT_MAC(t0, t1, t2, a[22], b[20]); + MONT_MAC(v0, v1, v2, a[23], b[19]); + MONT_MAC(w0, w1, w2, u[19], m[23]); + MONT_MAC(t0, t1, t2, u[20], m[22]); + MONT_MAC(v0, v1, v2, u[21], m[21]); + MONT_MAC(w0, w1, w2, u[22], m[20]); + MONT_MAC(t0, t1, t2, u[23], m[19]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[18] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 43 */ + MONT_MAC(t0, t1, t2, a[20], b[23]); + MONT_MUL(v0, v1, v2, a[21], b[22]); + MONT_MUL(w0, w1, w2, a[22], b[21]); + MONT_MAC(t0, t1, t2, a[23], b[20]); + MONT_MAC(v0, v1, v2, u[20], m[23]); + MONT_MAC(w0, w1, w2, u[21], m[22]); + MONT_MAC(t0, t1, t2, u[22], m[21]); + MONT_MAC(v0, v1, v2, u[23], m[20]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[19] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 44 */ + MONT_MAC(t0, t1, t2, a[21], b[23]); + MONT_MUL(v0, v1, v2, a[22], b[22]); + MONT_MUL(w0, w1, w2, a[23], b[21]); + MONT_MAC(t0, t1, t2, u[21], m[23]); + MONT_MAC(v0, v1, v2, u[22], m[22]); + MONT_MAC(w0, w1, w2, u[23], m[21]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[20] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 45 */ + MONT_MAC(t0, t1, t2, a[22], b[23]); + MONT_MUL(v0, v1, v2, a[23], b[22]); + MONT_MUL(w0, w1, w2, u[22], m[23]); + MONT_MAC(t0, t1, t2, u[23], m[22]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[21] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 46 */ + MONT_MAC(t0, t1, t2, a[23], b[23]); + MONT_MUL(v0, v1, v2, u[23], m[23]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[22] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[23] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 24); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 24); +} + +/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ +static void crypton_mont_sqr24(limb_t *r, const limb_t *a, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 24; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + limb_t s0, s1, s2, c0, c1, c2; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], a[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MUL(s0, s1, s2, a[0], a[1]); + MONT_MAC(t0, t1, t2, u[0], m[1]); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MUL(s0, s1, s2, a[0], a[2]); + MONT_MAC(t0, t1, t2, a[1], a[1]); + MONT_MUL(v0, v1, v2, u[0], m[2]); + MONT_MUL(w0, w1, w2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MUL(s0, s1, s2, a[0], a[3]); + MONT_MUL(c0, c1, c2, a[1], a[2]); + MONT_MAC(t0, t1, t2, u[0], m[3]); + MONT_MUL(v0, v1, v2, u[1], m[2]); + MONT_MUL(w0, w1, w2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MUL(s0, s1, s2, a[0], a[4]); + MONT_MUL(c0, c1, c2, a[1], a[3]); + MONT_MAC(t0, t1, t2, a[2], a[2]); + MONT_MUL(v0, v1, v2, u[0], m[4]); + MONT_MUL(w0, w1, w2, u[1], m[3]); + MONT_MAC(t0, t1, t2, u[2], m[2]); + MONT_MAC(v0, v1, v2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MUL(s0, s1, s2, a[0], a[5]); + MONT_MUL(c0, c1, c2, a[1], a[4]); + MONT_MAC(s0, s1, s2, a[2], a[3]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MUL(v0, v1, v2, u[1], m[4]); + MONT_MUL(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MUL(s0, s1, s2, a[0], a[6]); + MONT_MUL(c0, c1, c2, a[1], a[5]); + MONT_MAC(s0, s1, s2, a[2], a[4]); + MONT_MAC(t0, t1, t2, a[3], a[3]); + MONT_MUL(v0, v1, v2, u[0], m[6]); + MONT_MUL(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MUL(s0, s1, s2, a[0], a[7]); + MONT_MUL(c0, c1, c2, a[1], a[6]); + MONT_MAC(s0, s1, s2, a[2], a[5]); + MONT_MAC(c0, c1, c2, a[3], a[4]); + MONT_MAC(t0, t1, t2, u[0], m[7]); + MONT_MUL(v0, v1, v2, u[1], m[6]); + MONT_MUL(w0, w1, w2, u[2], m[5]); + MONT_MAC(t0, t1, t2, u[3], m[4]); + MONT_MAC(v0, v1, v2, u[4], m[3]); + MONT_MAC(w0, w1, w2, u[5], m[2]); + MONT_MAC(t0, t1, t2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MUL(s0, s1, s2, a[0], a[8]); + MONT_MUL(c0, c1, c2, a[1], a[7]); + MONT_MAC(s0, s1, s2, a[2], a[6]); + MONT_MAC(c0, c1, c2, a[3], a[5]); + MONT_MAC(t0, t1, t2, a[4], a[4]); + MONT_MUL(v0, v1, v2, u[0], m[8]); + MONT_MUL(w0, w1, w2, u[1], m[7]); + MONT_MAC(t0, t1, t2, u[2], m[6]); + MONT_MAC(v0, v1, v2, u[3], m[5]); + MONT_MAC(w0, w1, w2, u[4], m[4]); + MONT_MAC(t0, t1, t2, u[5], m[3]); + MONT_MAC(v0, v1, v2, u[6], m[2]); + MONT_MAC(w0, w1, w2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MUL(s0, s1, s2, a[0], a[9]); + MONT_MUL(c0, c1, c2, a[1], a[8]); + MONT_MAC(s0, s1, s2, a[2], a[7]); + MONT_MAC(c0, c1, c2, a[3], a[6]); + MONT_MAC(s0, s1, s2, a[4], a[5]); + MONT_MAC(t0, t1, t2, u[0], m[9]); + MONT_MUL(v0, v1, v2, u[1], m[8]); + MONT_MUL(w0, w1, w2, u[2], m[7]); + MONT_MAC(t0, t1, t2, u[3], m[6]); + MONT_MAC(v0, v1, v2, u[4], m[5]); + MONT_MAC(w0, w1, w2, u[5], m[4]); + MONT_MAC(t0, t1, t2, u[6], m[3]); + MONT_MAC(v0, v1, v2, u[7], m[2]); + MONT_MAC(w0, w1, w2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MUL(s0, s1, s2, a[0], a[10]); + MONT_MUL(c0, c1, c2, a[1], a[9]); + MONT_MAC(s0, s1, s2, a[2], a[8]); + MONT_MAC(c0, c1, c2, a[3], a[7]); + MONT_MAC(s0, s1, s2, a[4], a[6]); + MONT_MAC(t0, t1, t2, a[5], a[5]); + MONT_MUL(v0, v1, v2, u[0], m[10]); + MONT_MUL(w0, w1, w2, u[1], m[9]); + MONT_MAC(t0, t1, t2, u[2], m[8]); + MONT_MAC(v0, v1, v2, u[3], m[7]); + MONT_MAC(w0, w1, w2, u[4], m[6]); + MONT_MAC(t0, t1, t2, u[5], m[5]); + MONT_MAC(v0, v1, v2, u[6], m[4]); + MONT_MAC(w0, w1, w2, u[7], m[3]); + MONT_MAC(t0, t1, t2, u[8], m[2]); + MONT_MAC(v0, v1, v2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MUL(s0, s1, s2, a[0], a[11]); + MONT_MUL(c0, c1, c2, a[1], a[10]); + MONT_MAC(s0, s1, s2, a[2], a[9]); + MONT_MAC(c0, c1, c2, a[3], a[8]); + MONT_MAC(s0, s1, s2, a[4], a[7]); + MONT_MAC(c0, c1, c2, a[5], a[6]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MUL(v0, v1, v2, u[1], m[10]); + MONT_MUL(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MUL(s0, s1, s2, a[0], a[12]); + MONT_MUL(c0, c1, c2, a[1], a[11]); + MONT_MAC(s0, s1, s2, a[2], a[10]); + MONT_MAC(c0, c1, c2, a[3], a[9]); + MONT_MAC(s0, s1, s2, a[4], a[8]); + MONT_MAC(c0, c1, c2, a[5], a[7]); + MONT_MAC(t0, t1, t2, a[6], a[6]); + MONT_MUL(v0, v1, v2, u[0], m[12]); + MONT_MUL(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MUL(s0, s1, s2, a[0], a[13]); + MONT_MUL(c0, c1, c2, a[1], a[12]); + MONT_MAC(s0, s1, s2, a[2], a[11]); + MONT_MAC(c0, c1, c2, a[3], a[10]); + MONT_MAC(s0, s1, s2, a[4], a[9]); + MONT_MAC(c0, c1, c2, a[5], a[8]); + MONT_MAC(s0, s1, s2, a[6], a[7]); + MONT_MAC(t0, t1, t2, u[0], m[13]); + MONT_MUL(v0, v1, v2, u[1], m[12]); + MONT_MUL(w0, w1, w2, u[2], m[11]); + MONT_MAC(t0, t1, t2, u[3], m[10]); + MONT_MAC(v0, v1, v2, u[4], m[9]); + MONT_MAC(w0, w1, w2, u[5], m[8]); + MONT_MAC(t0, t1, t2, u[6], m[7]); + MONT_MAC(v0, v1, v2, u[7], m[6]); + MONT_MAC(w0, w1, w2, u[8], m[5]); + MONT_MAC(t0, t1, t2, u[9], m[4]); + MONT_MAC(v0, v1, v2, u[10], m[3]); + MONT_MAC(w0, w1, w2, u[11], m[2]); + MONT_MAC(t0, t1, t2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MUL(s0, s1, s2, a[0], a[14]); + MONT_MUL(c0, c1, c2, a[1], a[13]); + MONT_MAC(s0, s1, s2, a[2], a[12]); + MONT_MAC(c0, c1, c2, a[3], a[11]); + MONT_MAC(s0, s1, s2, a[4], a[10]); + MONT_MAC(c0, c1, c2, a[5], a[9]); + MONT_MAC(s0, s1, s2, a[6], a[8]); + MONT_MAC(t0, t1, t2, a[7], a[7]); + MONT_MUL(v0, v1, v2, u[0], m[14]); + MONT_MUL(w0, w1, w2, u[1], m[13]); + MONT_MAC(t0, t1, t2, u[2], m[12]); + MONT_MAC(v0, v1, v2, u[3], m[11]); + MONT_MAC(w0, w1, w2, u[4], m[10]); + MONT_MAC(t0, t1, t2, u[5], m[9]); + MONT_MAC(v0, v1, v2, u[6], m[8]); + MONT_MAC(w0, w1, w2, u[7], m[7]); + MONT_MAC(t0, t1, t2, u[8], m[6]); + MONT_MAC(v0, v1, v2, u[9], m[5]); + MONT_MAC(w0, w1, w2, u[10], m[4]); + MONT_MAC(t0, t1, t2, u[11], m[3]); + MONT_MAC(v0, v1, v2, u[12], m[2]); + MONT_MAC(w0, w1, w2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MUL(s0, s1, s2, a[0], a[15]); + MONT_MUL(c0, c1, c2, a[1], a[14]); + MONT_MAC(s0, s1, s2, a[2], a[13]); + MONT_MAC(c0, c1, c2, a[3], a[12]); + MONT_MAC(s0, s1, s2, a[4], a[11]); + MONT_MAC(c0, c1, c2, a[5], a[10]); + MONT_MAC(s0, s1, s2, a[6], a[9]); + MONT_MAC(c0, c1, c2, a[7], a[8]); + MONT_MAC(t0, t1, t2, u[0], m[15]); + MONT_MUL(v0, v1, v2, u[1], m[14]); + MONT_MUL(w0, w1, w2, u[2], m[13]); + MONT_MAC(t0, t1, t2, u[3], m[12]); + MONT_MAC(v0, v1, v2, u[4], m[11]); + MONT_MAC(w0, w1, w2, u[5], m[10]); + MONT_MAC(t0, t1, t2, u[6], m[9]); + MONT_MAC(v0, v1, v2, u[7], m[8]); + MONT_MAC(w0, w1, w2, u[8], m[7]); + MONT_MAC(t0, t1, t2, u[9], m[6]); + MONT_MAC(v0, v1, v2, u[10], m[5]); + MONT_MAC(w0, w1, w2, u[11], m[4]); + MONT_MAC(t0, t1, t2, u[12], m[3]); + MONT_MAC(v0, v1, v2, u[13], m[2]); + MONT_MAC(w0, w1, w2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MUL(s0, s1, s2, a[0], a[16]); + MONT_MUL(c0, c1, c2, a[1], a[15]); + MONT_MAC(s0, s1, s2, a[2], a[14]); + MONT_MAC(c0, c1, c2, a[3], a[13]); + MONT_MAC(s0, s1, s2, a[4], a[12]); + MONT_MAC(c0, c1, c2, a[5], a[11]); + MONT_MAC(s0, s1, s2, a[6], a[10]); + MONT_MAC(c0, c1, c2, a[7], a[9]); + MONT_MAC(t0, t1, t2, a[8], a[8]); + MONT_MUL(v0, v1, v2, u[0], m[16]); + MONT_MUL(w0, w1, w2, u[1], m[15]); + MONT_MAC(t0, t1, t2, u[2], m[14]); + MONT_MAC(v0, v1, v2, u[3], m[13]); + MONT_MAC(w0, w1, w2, u[4], m[12]); + MONT_MAC(t0, t1, t2, u[5], m[11]); + MONT_MAC(v0, v1, v2, u[6], m[10]); + MONT_MAC(w0, w1, w2, u[7], m[9]); + MONT_MAC(t0, t1, t2, u[8], m[8]); + MONT_MAC(v0, v1, v2, u[9], m[7]); + MONT_MAC(w0, w1, w2, u[10], m[6]); + MONT_MAC(t0, t1, t2, u[11], m[5]); + MONT_MAC(v0, v1, v2, u[12], m[4]); + MONT_MAC(w0, w1, w2, u[13], m[3]); + MONT_MAC(t0, t1, t2, u[14], m[2]); + MONT_MAC(v0, v1, v2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[16] = t0 * n0; + MONT_MAC(t0, t1, t2, u[16], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MUL(s0, s1, s2, a[0], a[17]); + MONT_MUL(c0, c1, c2, a[1], a[16]); + MONT_MAC(s0, s1, s2, a[2], a[15]); + MONT_MAC(c0, c1, c2, a[3], a[14]); + MONT_MAC(s0, s1, s2, a[4], a[13]); + MONT_MAC(c0, c1, c2, a[5], a[12]); + MONT_MAC(s0, s1, s2, a[6], a[11]); + MONT_MAC(c0, c1, c2, a[7], a[10]); + MONT_MAC(s0, s1, s2, a[8], a[9]); + MONT_MAC(t0, t1, t2, u[0], m[17]); + MONT_MUL(v0, v1, v2, u[1], m[16]); + MONT_MUL(w0, w1, w2, u[2], m[15]); + MONT_MAC(t0, t1, t2, u[3], m[14]); + MONT_MAC(v0, v1, v2, u[4], m[13]); + MONT_MAC(w0, w1, w2, u[5], m[12]); + MONT_MAC(t0, t1, t2, u[6], m[11]); + MONT_MAC(v0, v1, v2, u[7], m[10]); + MONT_MAC(w0, w1, w2, u[8], m[9]); + MONT_MAC(t0, t1, t2, u[9], m[8]); + MONT_MAC(v0, v1, v2, u[10], m[7]); + MONT_MAC(w0, w1, w2, u[11], m[6]); + MONT_MAC(t0, t1, t2, u[12], m[5]); + MONT_MAC(v0, v1, v2, u[13], m[4]); + MONT_MAC(w0, w1, w2, u[14], m[3]); + MONT_MAC(t0, t1, t2, u[15], m[2]); + MONT_MAC(v0, v1, v2, u[16], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[17] = t0 * n0; + MONT_MAC(t0, t1, t2, u[17], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MUL(s0, s1, s2, a[0], a[18]); + MONT_MUL(c0, c1, c2, a[1], a[17]); + MONT_MAC(s0, s1, s2, a[2], a[16]); + MONT_MAC(c0, c1, c2, a[3], a[15]); + MONT_MAC(s0, s1, s2, a[4], a[14]); + MONT_MAC(c0, c1, c2, a[5], a[13]); + MONT_MAC(s0, s1, s2, a[6], a[12]); + MONT_MAC(c0, c1, c2, a[7], a[11]); + MONT_MAC(s0, s1, s2, a[8], a[10]); + MONT_MAC(t0, t1, t2, a[9], a[9]); + MONT_MUL(v0, v1, v2, u[0], m[18]); + MONT_MUL(w0, w1, w2, u[1], m[17]); + MONT_MAC(t0, t1, t2, u[2], m[16]); + MONT_MAC(v0, v1, v2, u[3], m[15]); + MONT_MAC(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_MAC(w0, w1, w2, u[16], m[2]); + MONT_MAC(t0, t1, t2, u[17], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[18] = t0 * n0; + MONT_MAC(t0, t1, t2, u[18], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MUL(s0, s1, s2, a[0], a[19]); + MONT_MUL(c0, c1, c2, a[1], a[18]); + MONT_MAC(s0, s1, s2, a[2], a[17]); + MONT_MAC(c0, c1, c2, a[3], a[16]); + MONT_MAC(s0, s1, s2, a[4], a[15]); + MONT_MAC(c0, c1, c2, a[5], a[14]); + MONT_MAC(s0, s1, s2, a[6], a[13]); + MONT_MAC(c0, c1, c2, a[7], a[12]); + MONT_MAC(s0, s1, s2, a[8], a[11]); + MONT_MAC(c0, c1, c2, a[9], a[10]); + MONT_MAC(t0, t1, t2, u[0], m[19]); + MONT_MUL(v0, v1, v2, u[1], m[18]); + MONT_MUL(w0, w1, w2, u[2], m[17]); + MONT_MAC(t0, t1, t2, u[3], m[16]); + MONT_MAC(v0, v1, v2, u[4], m[15]); + MONT_MAC(w0, w1, w2, u[5], m[14]); + MONT_MAC(t0, t1, t2, u[6], m[13]); + MONT_MAC(v0, v1, v2, u[7], m[12]); + MONT_MAC(w0, w1, w2, u[8], m[11]); + MONT_MAC(t0, t1, t2, u[9], m[10]); + MONT_MAC(v0, v1, v2, u[10], m[9]); + MONT_MAC(w0, w1, w2, u[11], m[8]); + MONT_MAC(t0, t1, t2, u[12], m[7]); + MONT_MAC(v0, v1, v2, u[13], m[6]); + MONT_MAC(w0, w1, w2, u[14], m[5]); + MONT_MAC(t0, t1, t2, u[15], m[4]); + MONT_MAC(v0, v1, v2, u[16], m[3]); + MONT_MAC(w0, w1, w2, u[17], m[2]); + MONT_MAC(t0, t1, t2, u[18], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[19] = t0 * n0; + MONT_MAC(t0, t1, t2, u[19], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MUL(s0, s1, s2, a[0], a[20]); + MONT_MUL(c0, c1, c2, a[1], a[19]); + MONT_MAC(s0, s1, s2, a[2], a[18]); + MONT_MAC(c0, c1, c2, a[3], a[17]); + MONT_MAC(s0, s1, s2, a[4], a[16]); + MONT_MAC(c0, c1, c2, a[5], a[15]); + MONT_MAC(s0, s1, s2, a[6], a[14]); + MONT_MAC(c0, c1, c2, a[7], a[13]); + MONT_MAC(s0, s1, s2, a[8], a[12]); + MONT_MAC(c0, c1, c2, a[9], a[11]); + MONT_MAC(t0, t1, t2, a[10], a[10]); + MONT_MUL(v0, v1, v2, u[0], m[20]); + MONT_MUL(w0, w1, w2, u[1], m[19]); + MONT_MAC(t0, t1, t2, u[2], m[18]); + MONT_MAC(v0, v1, v2, u[3], m[17]); + MONT_MAC(w0, w1, w2, u[4], m[16]); + MONT_MAC(t0, t1, t2, u[5], m[15]); + MONT_MAC(v0, v1, v2, u[6], m[14]); + MONT_MAC(w0, w1, w2, u[7], m[13]); + MONT_MAC(t0, t1, t2, u[8], m[12]); + MONT_MAC(v0, v1, v2, u[9], m[11]); + MONT_MAC(w0, w1, w2, u[10], m[10]); + MONT_MAC(t0, t1, t2, u[11], m[9]); + MONT_MAC(v0, v1, v2, u[12], m[8]); + MONT_MAC(w0, w1, w2, u[13], m[7]); + MONT_MAC(t0, t1, t2, u[14], m[6]); + MONT_MAC(v0, v1, v2, u[15], m[5]); + MONT_MAC(w0, w1, w2, u[16], m[4]); + MONT_MAC(t0, t1, t2, u[17], m[3]); + MONT_MAC(v0, v1, v2, u[18], m[2]); + MONT_MAC(w0, w1, w2, u[19], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[20] = t0 * n0; + MONT_MAC(t0, t1, t2, u[20], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MUL(s0, s1, s2, a[0], a[21]); + MONT_MUL(c0, c1, c2, a[1], a[20]); + MONT_MAC(s0, s1, s2, a[2], a[19]); + MONT_MAC(c0, c1, c2, a[3], a[18]); + MONT_MAC(s0, s1, s2, a[4], a[17]); + MONT_MAC(c0, c1, c2, a[5], a[16]); + MONT_MAC(s0, s1, s2, a[6], a[15]); + MONT_MAC(c0, c1, c2, a[7], a[14]); + MONT_MAC(s0, s1, s2, a[8], a[13]); + MONT_MAC(c0, c1, c2, a[9], a[12]); + MONT_MAC(s0, s1, s2, a[10], a[11]); + MONT_MAC(t0, t1, t2, u[0], m[21]); + MONT_MUL(v0, v1, v2, u[1], m[20]); + MONT_MUL(w0, w1, w2, u[2], m[19]); + MONT_MAC(t0, t1, t2, u[3], m[18]); + MONT_MAC(v0, v1, v2, u[4], m[17]); + MONT_MAC(w0, w1, w2, u[5], m[16]); + MONT_MAC(t0, t1, t2, u[6], m[15]); + MONT_MAC(v0, v1, v2, u[7], m[14]); + MONT_MAC(w0, w1, w2, u[8], m[13]); + MONT_MAC(t0, t1, t2, u[9], m[12]); + MONT_MAC(v0, v1, v2, u[10], m[11]); + MONT_MAC(w0, w1, w2, u[11], m[10]); + MONT_MAC(t0, t1, t2, u[12], m[9]); + MONT_MAC(v0, v1, v2, u[13], m[8]); + MONT_MAC(w0, w1, w2, u[14], m[7]); + MONT_MAC(t0, t1, t2, u[15], m[6]); + MONT_MAC(v0, v1, v2, u[16], m[5]); + MONT_MAC(w0, w1, w2, u[17], m[4]); + MONT_MAC(t0, t1, t2, u[18], m[3]); + MONT_MAC(v0, v1, v2, u[19], m[2]); + MONT_MAC(w0, w1, w2, u[20], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[21] = t0 * n0; + MONT_MAC(t0, t1, t2, u[21], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MUL(s0, s1, s2, a[0], a[22]); + MONT_MUL(c0, c1, c2, a[1], a[21]); + MONT_MAC(s0, s1, s2, a[2], a[20]); + MONT_MAC(c0, c1, c2, a[3], a[19]); + MONT_MAC(s0, s1, s2, a[4], a[18]); + MONT_MAC(c0, c1, c2, a[5], a[17]); + MONT_MAC(s0, s1, s2, a[6], a[16]); + MONT_MAC(c0, c1, c2, a[7], a[15]); + MONT_MAC(s0, s1, s2, a[8], a[14]); + MONT_MAC(c0, c1, c2, a[9], a[13]); + MONT_MAC(s0, s1, s2, a[10], a[12]); + MONT_MAC(t0, t1, t2, a[11], a[11]); + MONT_MUL(v0, v1, v2, u[0], m[22]); + MONT_MUL(w0, w1, w2, u[1], m[21]); + MONT_MAC(t0, t1, t2, u[2], m[20]); + MONT_MAC(v0, v1, v2, u[3], m[19]); + MONT_MAC(w0, w1, w2, u[4], m[18]); + MONT_MAC(t0, t1, t2, u[5], m[17]); + MONT_MAC(v0, v1, v2, u[6], m[16]); + MONT_MAC(w0, w1, w2, u[7], m[15]); + MONT_MAC(t0, t1, t2, u[8], m[14]); + MONT_MAC(v0, v1, v2, u[9], m[13]); + MONT_MAC(w0, w1, w2, u[10], m[12]); + MONT_MAC(t0, t1, t2, u[11], m[11]); + MONT_MAC(v0, v1, v2, u[12], m[10]); + MONT_MAC(w0, w1, w2, u[13], m[9]); + MONT_MAC(t0, t1, t2, u[14], m[8]); + MONT_MAC(v0, v1, v2, u[15], m[7]); + MONT_MAC(w0, w1, w2, u[16], m[6]); + MONT_MAC(t0, t1, t2, u[17], m[5]); + MONT_MAC(v0, v1, v2, u[18], m[4]); + MONT_MAC(w0, w1, w2, u[19], m[3]); + MONT_MAC(t0, t1, t2, u[20], m[2]); + MONT_MAC(v0, v1, v2, u[21], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[22] = t0 * n0; + MONT_MAC(t0, t1, t2, u[22], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MUL(s0, s1, s2, a[0], a[23]); + MONT_MUL(c0, c1, c2, a[1], a[22]); + MONT_MAC(s0, s1, s2, a[2], a[21]); + MONT_MAC(c0, c1, c2, a[3], a[20]); + MONT_MAC(s0, s1, s2, a[4], a[19]); + MONT_MAC(c0, c1, c2, a[5], a[18]); + MONT_MAC(s0, s1, s2, a[6], a[17]); + MONT_MAC(c0, c1, c2, a[7], a[16]); + MONT_MAC(s0, s1, s2, a[8], a[15]); + MONT_MAC(c0, c1, c2, a[9], a[14]); + MONT_MAC(s0, s1, s2, a[10], a[13]); + MONT_MAC(c0, c1, c2, a[11], a[12]); + MONT_MAC(t0, t1, t2, u[0], m[23]); + MONT_MUL(v0, v1, v2, u[1], m[22]); + MONT_MUL(w0, w1, w2, u[2], m[21]); + MONT_MAC(t0, t1, t2, u[3], m[20]); + MONT_MAC(v0, v1, v2, u[4], m[19]); + MONT_MAC(w0, w1, w2, u[5], m[18]); + MONT_MAC(t0, t1, t2, u[6], m[17]); + MONT_MAC(v0, v1, v2, u[7], m[16]); + MONT_MAC(w0, w1, w2, u[8], m[15]); + MONT_MAC(t0, t1, t2, u[9], m[14]); + MONT_MAC(v0, v1, v2, u[10], m[13]); + MONT_MAC(w0, w1, w2, u[11], m[12]); + MONT_MAC(t0, t1, t2, u[12], m[11]); + MONT_MAC(v0, v1, v2, u[13], m[10]); + MONT_MAC(w0, w1, w2, u[14], m[9]); + MONT_MAC(t0, t1, t2, u[15], m[8]); + MONT_MAC(v0, v1, v2, u[16], m[7]); + MONT_MAC(w0, w1, w2, u[17], m[6]); + MONT_MAC(t0, t1, t2, u[18], m[5]); + MONT_MAC(v0, v1, v2, u[19], m[4]); + MONT_MAC(w0, w1, w2, u[20], m[3]); + MONT_MAC(t0, t1, t2, u[21], m[2]); + MONT_MAC(v0, v1, v2, u[22], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[23] = t0 * n0; + MONT_MAC(t0, t1, t2, u[23], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MUL(s0, s1, s2, a[1], a[23]); + MONT_MUL(c0, c1, c2, a[2], a[22]); + MONT_MAC(s0, s1, s2, a[3], a[21]); + MONT_MAC(c0, c1, c2, a[4], a[20]); + MONT_MAC(s0, s1, s2, a[5], a[19]); + MONT_MAC(c0, c1, c2, a[6], a[18]); + MONT_MAC(s0, s1, s2, a[7], a[17]); + MONT_MAC(c0, c1, c2, a[8], a[16]); + MONT_MAC(s0, s1, s2, a[9], a[15]); + MONT_MAC(c0, c1, c2, a[10], a[14]); + MONT_MAC(s0, s1, s2, a[11], a[13]); + MONT_MAC(t0, t1, t2, a[12], a[12]); + MONT_MUL(v0, v1, v2, u[1], m[23]); + MONT_MUL(w0, w1, w2, u[2], m[22]); + MONT_MAC(t0, t1, t2, u[3], m[21]); + MONT_MAC(v0, v1, v2, u[4], m[20]); + MONT_MAC(w0, w1, w2, u[5], m[19]); + MONT_MAC(t0, t1, t2, u[6], m[18]); + MONT_MAC(v0, v1, v2, u[7], m[17]); + MONT_MAC(w0, w1, w2, u[8], m[16]); + MONT_MAC(t0, t1, t2, u[9], m[15]); + MONT_MAC(v0, v1, v2, u[10], m[14]); + MONT_MAC(w0, w1, w2, u[11], m[13]); + MONT_MAC(t0, t1, t2, u[12], m[12]); + MONT_MAC(v0, v1, v2, u[13], m[11]); + MONT_MAC(w0, w1, w2, u[14], m[10]); + MONT_MAC(t0, t1, t2, u[15], m[9]); + MONT_MAC(v0, v1, v2, u[16], m[8]); + MONT_MAC(w0, w1, w2, u[17], m[7]); + MONT_MAC(t0, t1, t2, u[18], m[6]); + MONT_MAC(v0, v1, v2, u[19], m[5]); + MONT_MAC(w0, w1, w2, u[20], m[4]); + MONT_MAC(t0, t1, t2, u[21], m[3]); + MONT_MAC(v0, v1, v2, u[22], m[2]); + MONT_MAC(w0, w1, w2, u[23], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MUL(s0, s1, s2, a[2], a[23]); + MONT_MUL(c0, c1, c2, a[3], a[22]); + MONT_MAC(s0, s1, s2, a[4], a[21]); + MONT_MAC(c0, c1, c2, a[5], a[20]); + MONT_MAC(s0, s1, s2, a[6], a[19]); + MONT_MAC(c0, c1, c2, a[7], a[18]); + MONT_MAC(s0, s1, s2, a[8], a[17]); + MONT_MAC(c0, c1, c2, a[9], a[16]); + MONT_MAC(s0, s1, s2, a[10], a[15]); + MONT_MAC(c0, c1, c2, a[11], a[14]); + MONT_MAC(s0, s1, s2, a[12], a[13]); + MONT_MAC(t0, t1, t2, u[2], m[23]); + MONT_MUL(v0, v1, v2, u[3], m[22]); + MONT_MUL(w0, w1, w2, u[4], m[21]); + MONT_MAC(t0, t1, t2, u[5], m[20]); + MONT_MAC(v0, v1, v2, u[6], m[19]); + MONT_MAC(w0, w1, w2, u[7], m[18]); + MONT_MAC(t0, t1, t2, u[8], m[17]); + MONT_MAC(v0, v1, v2, u[9], m[16]); + MONT_MAC(w0, w1, w2, u[10], m[15]); + MONT_MAC(t0, t1, t2, u[11], m[14]); + MONT_MAC(v0, v1, v2, u[12], m[13]); + MONT_MAC(w0, w1, w2, u[13], m[12]); + MONT_MAC(t0, t1, t2, u[14], m[11]); + MONT_MAC(v0, v1, v2, u[15], m[10]); + MONT_MAC(w0, w1, w2, u[16], m[9]); + MONT_MAC(t0, t1, t2, u[17], m[8]); + MONT_MAC(v0, v1, v2, u[18], m[7]); + MONT_MAC(w0, w1, w2, u[19], m[6]); + MONT_MAC(t0, t1, t2, u[20], m[5]); + MONT_MAC(v0, v1, v2, u[21], m[4]); + MONT_MAC(w0, w1, w2, u[22], m[3]); + MONT_MAC(t0, t1, t2, u[23], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MUL(s0, s1, s2, a[3], a[23]); + MONT_MUL(c0, c1, c2, a[4], a[22]); + MONT_MAC(s0, s1, s2, a[5], a[21]); + MONT_MAC(c0, c1, c2, a[6], a[20]); + MONT_MAC(s0, s1, s2, a[7], a[19]); + MONT_MAC(c0, c1, c2, a[8], a[18]); + MONT_MAC(s0, s1, s2, a[9], a[17]); + MONT_MAC(c0, c1, c2, a[10], a[16]); + MONT_MAC(s0, s1, s2, a[11], a[15]); + MONT_MAC(c0, c1, c2, a[12], a[14]); + MONT_MAC(t0, t1, t2, a[13], a[13]); + MONT_MUL(v0, v1, v2, u[3], m[23]); + MONT_MUL(w0, w1, w2, u[4], m[22]); + MONT_MAC(t0, t1, t2, u[5], m[21]); + MONT_MAC(v0, v1, v2, u[6], m[20]); + MONT_MAC(w0, w1, w2, u[7], m[19]); + MONT_MAC(t0, t1, t2, u[8], m[18]); + MONT_MAC(v0, v1, v2, u[9], m[17]); + MONT_MAC(w0, w1, w2, u[10], m[16]); + MONT_MAC(t0, t1, t2, u[11], m[15]); + MONT_MAC(v0, v1, v2, u[12], m[14]); + MONT_MAC(w0, w1, w2, u[13], m[13]); + MONT_MAC(t0, t1, t2, u[14], m[12]); + MONT_MAC(v0, v1, v2, u[15], m[11]); + MONT_MAC(w0, w1, w2, u[16], m[10]); + MONT_MAC(t0, t1, t2, u[17], m[9]); + MONT_MAC(v0, v1, v2, u[18], m[8]); + MONT_MAC(w0, w1, w2, u[19], m[7]); + MONT_MAC(t0, t1, t2, u[20], m[6]); + MONT_MAC(v0, v1, v2, u[21], m[5]); + MONT_MAC(w0, w1, w2, u[22], m[4]); + MONT_MAC(t0, t1, t2, u[23], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MUL(s0, s1, s2, a[4], a[23]); + MONT_MUL(c0, c1, c2, a[5], a[22]); + MONT_MAC(s0, s1, s2, a[6], a[21]); + MONT_MAC(c0, c1, c2, a[7], a[20]); + MONT_MAC(s0, s1, s2, a[8], a[19]); + MONT_MAC(c0, c1, c2, a[9], a[18]); + MONT_MAC(s0, s1, s2, a[10], a[17]); + MONT_MAC(c0, c1, c2, a[11], a[16]); + MONT_MAC(s0, s1, s2, a[12], a[15]); + MONT_MAC(c0, c1, c2, a[13], a[14]); + MONT_MAC(t0, t1, t2, u[4], m[23]); + MONT_MUL(v0, v1, v2, u[5], m[22]); + MONT_MUL(w0, w1, w2, u[6], m[21]); + MONT_MAC(t0, t1, t2, u[7], m[20]); + MONT_MAC(v0, v1, v2, u[8], m[19]); + MONT_MAC(w0, w1, w2, u[9], m[18]); + MONT_MAC(t0, t1, t2, u[10], m[17]); + MONT_MAC(v0, v1, v2, u[11], m[16]); + MONT_MAC(w0, w1, w2, u[12], m[15]); + MONT_MAC(t0, t1, t2, u[13], m[14]); + MONT_MAC(v0, v1, v2, u[14], m[13]); + MONT_MAC(w0, w1, w2, u[15], m[12]); + MONT_MAC(t0, t1, t2, u[16], m[11]); + MONT_MAC(v0, v1, v2, u[17], m[10]); + MONT_MAC(w0, w1, w2, u[18], m[9]); + MONT_MAC(t0, t1, t2, u[19], m[8]); + MONT_MAC(v0, v1, v2, u[20], m[7]); + MONT_MAC(w0, w1, w2, u[21], m[6]); + MONT_MAC(t0, t1, t2, u[22], m[5]); + MONT_MAC(v0, v1, v2, u[23], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MUL(s0, s1, s2, a[5], a[23]); + MONT_MUL(c0, c1, c2, a[6], a[22]); + MONT_MAC(s0, s1, s2, a[7], a[21]); + MONT_MAC(c0, c1, c2, a[8], a[20]); + MONT_MAC(s0, s1, s2, a[9], a[19]); + MONT_MAC(c0, c1, c2, a[10], a[18]); + MONT_MAC(s0, s1, s2, a[11], a[17]); + MONT_MAC(c0, c1, c2, a[12], a[16]); + MONT_MAC(s0, s1, s2, a[13], a[15]); + MONT_MAC(t0, t1, t2, a[14], a[14]); + MONT_MUL(v0, v1, v2, u[5], m[23]); + MONT_MUL(w0, w1, w2, u[6], m[22]); + MONT_MAC(t0, t1, t2, u[7], m[21]); + MONT_MAC(v0, v1, v2, u[8], m[20]); + MONT_MAC(w0, w1, w2, u[9], m[19]); + MONT_MAC(t0, t1, t2, u[10], m[18]); + MONT_MAC(v0, v1, v2, u[11], m[17]); + MONT_MAC(w0, w1, w2, u[12], m[16]); + MONT_MAC(t0, t1, t2, u[13], m[15]); + MONT_MAC(v0, v1, v2, u[14], m[14]); + MONT_MAC(w0, w1, w2, u[15], m[13]); + MONT_MAC(t0, t1, t2, u[16], m[12]); + MONT_MAC(v0, v1, v2, u[17], m[11]); + MONT_MAC(w0, w1, w2, u[18], m[10]); + MONT_MAC(t0, t1, t2, u[19], m[9]); + MONT_MAC(v0, v1, v2, u[20], m[8]); + MONT_MAC(w0, w1, w2, u[21], m[7]); + MONT_MAC(t0, t1, t2, u[22], m[6]); + MONT_MAC(v0, v1, v2, u[23], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MUL(s0, s1, s2, a[6], a[23]); + MONT_MUL(c0, c1, c2, a[7], a[22]); + MONT_MAC(s0, s1, s2, a[8], a[21]); + MONT_MAC(c0, c1, c2, a[9], a[20]); + MONT_MAC(s0, s1, s2, a[10], a[19]); + MONT_MAC(c0, c1, c2, a[11], a[18]); + MONT_MAC(s0, s1, s2, a[12], a[17]); + MONT_MAC(c0, c1, c2, a[13], a[16]); + MONT_MAC(s0, s1, s2, a[14], a[15]); + MONT_MAC(t0, t1, t2, u[6], m[23]); + MONT_MUL(v0, v1, v2, u[7], m[22]); + MONT_MUL(w0, w1, w2, u[8], m[21]); + MONT_MAC(t0, t1, t2, u[9], m[20]); + MONT_MAC(v0, v1, v2, u[10], m[19]); + MONT_MAC(w0, w1, w2, u[11], m[18]); + MONT_MAC(t0, t1, t2, u[12], m[17]); + MONT_MAC(v0, v1, v2, u[13], m[16]); + MONT_MAC(w0, w1, w2, u[14], m[15]); + MONT_MAC(t0, t1, t2, u[15], m[14]); + MONT_MAC(v0, v1, v2, u[16], m[13]); + MONT_MAC(w0, w1, w2, u[17], m[12]); + MONT_MAC(t0, t1, t2, u[18], m[11]); + MONT_MAC(v0, v1, v2, u[19], m[10]); + MONT_MAC(w0, w1, w2, u[20], m[9]); + MONT_MAC(t0, t1, t2, u[21], m[8]); + MONT_MAC(v0, v1, v2, u[22], m[7]); + MONT_MAC(w0, w1, w2, u[23], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MUL(s0, s1, s2, a[7], a[23]); + MONT_MUL(c0, c1, c2, a[8], a[22]); + MONT_MAC(s0, s1, s2, a[9], a[21]); + MONT_MAC(c0, c1, c2, a[10], a[20]); + MONT_MAC(s0, s1, s2, a[11], a[19]); + MONT_MAC(c0, c1, c2, a[12], a[18]); + MONT_MAC(s0, s1, s2, a[13], a[17]); + MONT_MAC(c0, c1, c2, a[14], a[16]); + MONT_MAC(t0, t1, t2, a[15], a[15]); + MONT_MUL(v0, v1, v2, u[7], m[23]); + MONT_MUL(w0, w1, w2, u[8], m[22]); + MONT_MAC(t0, t1, t2, u[9], m[21]); + MONT_MAC(v0, v1, v2, u[10], m[20]); + MONT_MAC(w0, w1, w2, u[11], m[19]); + MONT_MAC(t0, t1, t2, u[12], m[18]); + MONT_MAC(v0, v1, v2, u[13], m[17]); + MONT_MAC(w0, w1, w2, u[14], m[16]); + MONT_MAC(t0, t1, t2, u[15], m[15]); + MONT_MAC(v0, v1, v2, u[16], m[14]); + MONT_MAC(w0, w1, w2, u[17], m[13]); + MONT_MAC(t0, t1, t2, u[18], m[12]); + MONT_MAC(v0, v1, v2, u[19], m[11]); + MONT_MAC(w0, w1, w2, u[20], m[10]); + MONT_MAC(t0, t1, t2, u[21], m[9]); + MONT_MAC(v0, v1, v2, u[22], m[8]); + MONT_MAC(w0, w1, w2, u[23], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 31 */ + MONT_MUL(s0, s1, s2, a[8], a[23]); + MONT_MUL(c0, c1, c2, a[9], a[22]); + MONT_MAC(s0, s1, s2, a[10], a[21]); + MONT_MAC(c0, c1, c2, a[11], a[20]); + MONT_MAC(s0, s1, s2, a[12], a[19]); + MONT_MAC(c0, c1, c2, a[13], a[18]); + MONT_MAC(s0, s1, s2, a[14], a[17]); + MONT_MAC(c0, c1, c2, a[15], a[16]); + MONT_MAC(t0, t1, t2, u[8], m[23]); + MONT_MUL(v0, v1, v2, u[9], m[22]); + MONT_MUL(w0, w1, w2, u[10], m[21]); + MONT_MAC(t0, t1, t2, u[11], m[20]); + MONT_MAC(v0, v1, v2, u[12], m[19]); + MONT_MAC(w0, w1, w2, u[13], m[18]); + MONT_MAC(t0, t1, t2, u[14], m[17]); + MONT_MAC(v0, v1, v2, u[15], m[16]); + MONT_MAC(w0, w1, w2, u[16], m[15]); + MONT_MAC(t0, t1, t2, u[17], m[14]); + MONT_MAC(v0, v1, v2, u[18], m[13]); + MONT_MAC(w0, w1, w2, u[19], m[12]); + MONT_MAC(t0, t1, t2, u[20], m[11]); + MONT_MAC(v0, v1, v2, u[21], m[10]); + MONT_MAC(w0, w1, w2, u[22], m[9]); + MONT_MAC(t0, t1, t2, u[23], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 32 */ + MONT_MUL(s0, s1, s2, a[9], a[23]); + MONT_MUL(c0, c1, c2, a[10], a[22]); + MONT_MAC(s0, s1, s2, a[11], a[21]); + MONT_MAC(c0, c1, c2, a[12], a[20]); + MONT_MAC(s0, s1, s2, a[13], a[19]); + MONT_MAC(c0, c1, c2, a[14], a[18]); + MONT_MAC(s0, s1, s2, a[15], a[17]); + MONT_MAC(t0, t1, t2, a[16], a[16]); + MONT_MUL(v0, v1, v2, u[9], m[23]); + MONT_MUL(w0, w1, w2, u[10], m[22]); + MONT_MAC(t0, t1, t2, u[11], m[21]); + MONT_MAC(v0, v1, v2, u[12], m[20]); + MONT_MAC(w0, w1, w2, u[13], m[19]); + MONT_MAC(t0, t1, t2, u[14], m[18]); + MONT_MAC(v0, v1, v2, u[15], m[17]); + MONT_MAC(w0, w1, w2, u[16], m[16]); + MONT_MAC(t0, t1, t2, u[17], m[15]); + MONT_MAC(v0, v1, v2, u[18], m[14]); + MONT_MAC(w0, w1, w2, u[19], m[13]); + MONT_MAC(t0, t1, t2, u[20], m[12]); + MONT_MAC(v0, v1, v2, u[21], m[11]); + MONT_MAC(w0, w1, w2, u[22], m[10]); + MONT_MAC(t0, t1, t2, u[23], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 33 */ + MONT_MUL(s0, s1, s2, a[10], a[23]); + MONT_MUL(c0, c1, c2, a[11], a[22]); + MONT_MAC(s0, s1, s2, a[12], a[21]); + MONT_MAC(c0, c1, c2, a[13], a[20]); + MONT_MAC(s0, s1, s2, a[14], a[19]); + MONT_MAC(c0, c1, c2, a[15], a[18]); + MONT_MAC(s0, s1, s2, a[16], a[17]); + MONT_MAC(t0, t1, t2, u[10], m[23]); + MONT_MUL(v0, v1, v2, u[11], m[22]); + MONT_MUL(w0, w1, w2, u[12], m[21]); + MONT_MAC(t0, t1, t2, u[13], m[20]); + MONT_MAC(v0, v1, v2, u[14], m[19]); + MONT_MAC(w0, w1, w2, u[15], m[18]); + MONT_MAC(t0, t1, t2, u[16], m[17]); + MONT_MAC(v0, v1, v2, u[17], m[16]); + MONT_MAC(w0, w1, w2, u[18], m[15]); + MONT_MAC(t0, t1, t2, u[19], m[14]); + MONT_MAC(v0, v1, v2, u[20], m[13]); + MONT_MAC(w0, w1, w2, u[21], m[12]); + MONT_MAC(t0, t1, t2, u[22], m[11]); + MONT_MAC(v0, v1, v2, u[23], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 34 */ + MONT_MUL(s0, s1, s2, a[11], a[23]); + MONT_MUL(c0, c1, c2, a[12], a[22]); + MONT_MAC(s0, s1, s2, a[13], a[21]); + MONT_MAC(c0, c1, c2, a[14], a[20]); + MONT_MAC(s0, s1, s2, a[15], a[19]); + MONT_MAC(c0, c1, c2, a[16], a[18]); + MONT_MAC(t0, t1, t2, a[17], a[17]); + MONT_MUL(v0, v1, v2, u[11], m[23]); + MONT_MUL(w0, w1, w2, u[12], m[22]); + MONT_MAC(t0, t1, t2, u[13], m[21]); + MONT_MAC(v0, v1, v2, u[14], m[20]); + MONT_MAC(w0, w1, w2, u[15], m[19]); + MONT_MAC(t0, t1, t2, u[16], m[18]); + MONT_MAC(v0, v1, v2, u[17], m[17]); + MONT_MAC(w0, w1, w2, u[18], m[16]); + MONT_MAC(t0, t1, t2, u[19], m[15]); + MONT_MAC(v0, v1, v2, u[20], m[14]); + MONT_MAC(w0, w1, w2, u[21], m[13]); + MONT_MAC(t0, t1, t2, u[22], m[12]); + MONT_MAC(v0, v1, v2, u[23], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 35 */ + MONT_MUL(s0, s1, s2, a[12], a[23]); + MONT_MUL(c0, c1, c2, a[13], a[22]); + MONT_MAC(s0, s1, s2, a[14], a[21]); + MONT_MAC(c0, c1, c2, a[15], a[20]); + MONT_MAC(s0, s1, s2, a[16], a[19]); + MONT_MAC(c0, c1, c2, a[17], a[18]); + MONT_MAC(t0, t1, t2, u[12], m[23]); + MONT_MUL(v0, v1, v2, u[13], m[22]); + MONT_MUL(w0, w1, w2, u[14], m[21]); + MONT_MAC(t0, t1, t2, u[15], m[20]); + MONT_MAC(v0, v1, v2, u[16], m[19]); + MONT_MAC(w0, w1, w2, u[17], m[18]); + MONT_MAC(t0, t1, t2, u[18], m[17]); + MONT_MAC(v0, v1, v2, u[19], m[16]); + MONT_MAC(w0, w1, w2, u[20], m[15]); + MONT_MAC(t0, t1, t2, u[21], m[14]); + MONT_MAC(v0, v1, v2, u[22], m[13]); + MONT_MAC(w0, w1, w2, u[23], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 36 */ + MONT_MUL(s0, s1, s2, a[13], a[23]); + MONT_MUL(c0, c1, c2, a[14], a[22]); + MONT_MAC(s0, s1, s2, a[15], a[21]); + MONT_MAC(c0, c1, c2, a[16], a[20]); + MONT_MAC(s0, s1, s2, a[17], a[19]); + MONT_MAC(t0, t1, t2, a[18], a[18]); + MONT_MUL(v0, v1, v2, u[13], m[23]); + MONT_MUL(w0, w1, w2, u[14], m[22]); + MONT_MAC(t0, t1, t2, u[15], m[21]); + MONT_MAC(v0, v1, v2, u[16], m[20]); + MONT_MAC(w0, w1, w2, u[17], m[19]); + MONT_MAC(t0, t1, t2, u[18], m[18]); + MONT_MAC(v0, v1, v2, u[19], m[17]); + MONT_MAC(w0, w1, w2, u[20], m[16]); + MONT_MAC(t0, t1, t2, u[21], m[15]); + MONT_MAC(v0, v1, v2, u[22], m[14]); + MONT_MAC(w0, w1, w2, u[23], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 37 */ + MONT_MUL(s0, s1, s2, a[14], a[23]); + MONT_MUL(c0, c1, c2, a[15], a[22]); + MONT_MAC(s0, s1, s2, a[16], a[21]); + MONT_MAC(c0, c1, c2, a[17], a[20]); + MONT_MAC(s0, s1, s2, a[18], a[19]); + MONT_MAC(t0, t1, t2, u[14], m[23]); + MONT_MUL(v0, v1, v2, u[15], m[22]); + MONT_MUL(w0, w1, w2, u[16], m[21]); + MONT_MAC(t0, t1, t2, u[17], m[20]); + MONT_MAC(v0, v1, v2, u[18], m[19]); + MONT_MAC(w0, w1, w2, u[19], m[18]); + MONT_MAC(t0, t1, t2, u[20], m[17]); + MONT_MAC(v0, v1, v2, u[21], m[16]); + MONT_MAC(w0, w1, w2, u[22], m[15]); + MONT_MAC(t0, t1, t2, u[23], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 38 */ + MONT_MUL(s0, s1, s2, a[15], a[23]); + MONT_MUL(c0, c1, c2, a[16], a[22]); + MONT_MAC(s0, s1, s2, a[17], a[21]); + MONT_MAC(c0, c1, c2, a[18], a[20]); + MONT_MAC(t0, t1, t2, a[19], a[19]); + MONT_MUL(v0, v1, v2, u[15], m[23]); + MONT_MUL(w0, w1, w2, u[16], m[22]); + MONT_MAC(t0, t1, t2, u[17], m[21]); + MONT_MAC(v0, v1, v2, u[18], m[20]); + MONT_MAC(w0, w1, w2, u[19], m[19]); + MONT_MAC(t0, t1, t2, u[20], m[18]); + MONT_MAC(v0, v1, v2, u[21], m[17]); + MONT_MAC(w0, w1, w2, u[22], m[16]); + MONT_MAC(t0, t1, t2, u[23], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 39 */ + MONT_MUL(s0, s1, s2, a[16], a[23]); + MONT_MUL(c0, c1, c2, a[17], a[22]); + MONT_MAC(s0, s1, s2, a[18], a[21]); + MONT_MAC(c0, c1, c2, a[19], a[20]); + MONT_MAC(t0, t1, t2, u[16], m[23]); + MONT_MUL(v0, v1, v2, u[17], m[22]); + MONT_MUL(w0, w1, w2, u[18], m[21]); + MONT_MAC(t0, t1, t2, u[19], m[20]); + MONT_MAC(v0, v1, v2, u[20], m[19]); + MONT_MAC(w0, w1, w2, u[21], m[18]); + MONT_MAC(t0, t1, t2, u[22], m[17]); + MONT_MAC(v0, v1, v2, u[23], m[16]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[15] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 40 */ + MONT_MUL(s0, s1, s2, a[17], a[23]); + MONT_MUL(c0, c1, c2, a[18], a[22]); + MONT_MAC(s0, s1, s2, a[19], a[21]); + MONT_MAC(t0, t1, t2, a[20], a[20]); + MONT_MUL(v0, v1, v2, u[17], m[23]); + MONT_MUL(w0, w1, w2, u[18], m[22]); + MONT_MAC(t0, t1, t2, u[19], m[21]); + MONT_MAC(v0, v1, v2, u[20], m[20]); + MONT_MAC(w0, w1, w2, u[21], m[19]); + MONT_MAC(t0, t1, t2, u[22], m[18]); + MONT_MAC(v0, v1, v2, u[23], m[17]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[16] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 41 */ + MONT_MUL(s0, s1, s2, a[18], a[23]); + MONT_MUL(c0, c1, c2, a[19], a[22]); + MONT_MAC(s0, s1, s2, a[20], a[21]); + MONT_MAC(t0, t1, t2, u[18], m[23]); + MONT_MUL(v0, v1, v2, u[19], m[22]); + MONT_MUL(w0, w1, w2, u[20], m[21]); + MONT_MAC(t0, t1, t2, u[21], m[20]); + MONT_MAC(v0, v1, v2, u[22], m[19]); + MONT_MAC(w0, w1, w2, u[23], m[18]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[17] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 42 */ + MONT_MUL(s0, s1, s2, a[19], a[23]); + MONT_MUL(c0, c1, c2, a[20], a[22]); + MONT_MAC(t0, t1, t2, a[21], a[21]); + MONT_MUL(v0, v1, v2, u[19], m[23]); + MONT_MUL(w0, w1, w2, u[20], m[22]); + MONT_MAC(t0, t1, t2, u[21], m[21]); + MONT_MAC(v0, v1, v2, u[22], m[20]); + MONT_MAC(w0, w1, w2, u[23], m[19]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[18] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 43 */ + MONT_MUL(s0, s1, s2, a[20], a[23]); + MONT_MUL(c0, c1, c2, a[21], a[22]); + MONT_MAC(t0, t1, t2, u[20], m[23]); + MONT_MUL(v0, v1, v2, u[21], m[22]); + MONT_MUL(w0, w1, w2, u[22], m[21]); + MONT_MAC(t0, t1, t2, u[23], m[20]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[19] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 44 */ + MONT_MUL(s0, s1, s2, a[21], a[23]); + MONT_MAC(t0, t1, t2, a[22], a[22]); + MONT_MUL(v0, v1, v2, u[21], m[23]); + MONT_MUL(w0, w1, w2, u[22], m[22]); + MONT_MAC(t0, t1, t2, u[23], m[21]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[20] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 45 */ + MONT_MUL(s0, s1, s2, a[22], a[23]); + MONT_MAC(t0, t1, t2, u[22], m[23]); + MONT_MUL(v0, v1, v2, u[23], m[22]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[21] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 46 */ + MONT_MAC(t0, t1, t2, a[23], a[23]); + MONT_MUL(v0, v1, v2, u[23], m[23]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[22] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[23] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 24); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 24); +} + +/* r = a * b * R^-1 mod m, for a and b under m; t is 2n limbs of scratch */ +static void crypton_mont_mul32(limb_t *r, const limb_t *a, const limb_t *b, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 32; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], b[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MAC(t0, t1, t2, a[0], b[1]); + MONT_MUL(v0, v1, v2, a[1], b[0]); + MONT_MUL(w0, w1, w2, u[0], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MAC(t0, t1, t2, a[0], b[2]); + MONT_MUL(v0, v1, v2, a[1], b[1]); + MONT_MUL(w0, w1, w2, a[2], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[2]); + MONT_MAC(v0, v1, v2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MAC(t0, t1, t2, a[0], b[3]); + MONT_MUL(v0, v1, v2, a[1], b[2]); + MONT_MUL(w0, w1, w2, a[2], b[1]); + MONT_MAC(t0, t1, t2, a[3], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[3]); + MONT_MAC(w0, w1, w2, u[1], m[2]); + MONT_MAC(t0, t1, t2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MAC(t0, t1, t2, a[0], b[4]); + MONT_MUL(v0, v1, v2, a[1], b[3]); + MONT_MUL(w0, w1, w2, a[2], b[2]); + MONT_MAC(t0, t1, t2, a[3], b[1]); + MONT_MAC(v0, v1, v2, a[4], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[4]); + MONT_MAC(t0, t1, t2, u[1], m[3]); + MONT_MAC(v0, v1, v2, u[2], m[2]); + MONT_MAC(w0, w1, w2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MAC(t0, t1, t2, a[0], b[5]); + MONT_MUL(v0, v1, v2, a[1], b[4]); + MONT_MUL(w0, w1, w2, a[2], b[3]); + MONT_MAC(t0, t1, t2, a[3], b[2]); + MONT_MAC(v0, v1, v2, a[4], b[1]); + MONT_MAC(w0, w1, w2, a[5], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MAC(v0, v1, v2, u[1], m[4]); + MONT_MAC(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MAC(t0, t1, t2, a[0], b[6]); + MONT_MUL(v0, v1, v2, a[1], b[5]); + MONT_MUL(w0, w1, w2, a[2], b[4]); + MONT_MAC(t0, t1, t2, a[3], b[3]); + MONT_MAC(v0, v1, v2, a[4], b[2]); + MONT_MAC(w0, w1, w2, a[5], b[1]); + MONT_MAC(t0, t1, t2, a[6], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[6]); + MONT_MAC(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MAC(t0, t1, t2, a[0], b[7]); + MONT_MUL(v0, v1, v2, a[1], b[6]); + MONT_MUL(w0, w1, w2, a[2], b[5]); + MONT_MAC(t0, t1, t2, a[3], b[4]); + MONT_MAC(v0, v1, v2, a[4], b[3]); + MONT_MAC(w0, w1, w2, a[5], b[2]); + MONT_MAC(t0, t1, t2, a[6], b[1]); + MONT_MAC(v0, v1, v2, a[7], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[7]); + MONT_MAC(t0, t1, t2, u[1], m[6]); + MONT_MAC(v0, v1, v2, u[2], m[5]); + MONT_MAC(w0, w1, w2, u[3], m[4]); + MONT_MAC(t0, t1, t2, u[4], m[3]); + MONT_MAC(v0, v1, v2, u[5], m[2]); + MONT_MAC(w0, w1, w2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MAC(t0, t1, t2, a[0], b[8]); + MONT_MUL(v0, v1, v2, a[1], b[7]); + MONT_MUL(w0, w1, w2, a[2], b[6]); + MONT_MAC(t0, t1, t2, a[3], b[5]); + MONT_MAC(v0, v1, v2, a[4], b[4]); + MONT_MAC(w0, w1, w2, a[5], b[3]); + MONT_MAC(t0, t1, t2, a[6], b[2]); + MONT_MAC(v0, v1, v2, a[7], b[1]); + MONT_MAC(w0, w1, w2, a[8], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[8]); + MONT_MAC(v0, v1, v2, u[1], m[7]); + MONT_MAC(w0, w1, w2, u[2], m[6]); + MONT_MAC(t0, t1, t2, u[3], m[5]); + MONT_MAC(v0, v1, v2, u[4], m[4]); + MONT_MAC(w0, w1, w2, u[5], m[3]); + MONT_MAC(t0, t1, t2, u[6], m[2]); + MONT_MAC(v0, v1, v2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MAC(t0, t1, t2, a[0], b[9]); + MONT_MUL(v0, v1, v2, a[1], b[8]); + MONT_MUL(w0, w1, w2, a[2], b[7]); + MONT_MAC(t0, t1, t2, a[3], b[6]); + MONT_MAC(v0, v1, v2, a[4], b[5]); + MONT_MAC(w0, w1, w2, a[5], b[4]); + MONT_MAC(t0, t1, t2, a[6], b[3]); + MONT_MAC(v0, v1, v2, a[7], b[2]); + MONT_MAC(w0, w1, w2, a[8], b[1]); + MONT_MAC(t0, t1, t2, a[9], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[9]); + MONT_MAC(w0, w1, w2, u[1], m[8]); + MONT_MAC(t0, t1, t2, u[2], m[7]); + MONT_MAC(v0, v1, v2, u[3], m[6]); + MONT_MAC(w0, w1, w2, u[4], m[5]); + MONT_MAC(t0, t1, t2, u[5], m[4]); + MONT_MAC(v0, v1, v2, u[6], m[3]); + MONT_MAC(w0, w1, w2, u[7], m[2]); + MONT_MAC(t0, t1, t2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MAC(t0, t1, t2, a[0], b[10]); + MONT_MUL(v0, v1, v2, a[1], b[9]); + MONT_MUL(w0, w1, w2, a[2], b[8]); + MONT_MAC(t0, t1, t2, a[3], b[7]); + MONT_MAC(v0, v1, v2, a[4], b[6]); + MONT_MAC(w0, w1, w2, a[5], b[5]); + MONT_MAC(t0, t1, t2, a[6], b[4]); + MONT_MAC(v0, v1, v2, a[7], b[3]); + MONT_MAC(w0, w1, w2, a[8], b[2]); + MONT_MAC(t0, t1, t2, a[9], b[1]); + MONT_MAC(v0, v1, v2, a[10], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[10]); + MONT_MAC(t0, t1, t2, u[1], m[9]); + MONT_MAC(v0, v1, v2, u[2], m[8]); + MONT_MAC(w0, w1, w2, u[3], m[7]); + MONT_MAC(t0, t1, t2, u[4], m[6]); + MONT_MAC(v0, v1, v2, u[5], m[5]); + MONT_MAC(w0, w1, w2, u[6], m[4]); + MONT_MAC(t0, t1, t2, u[7], m[3]); + MONT_MAC(v0, v1, v2, u[8], m[2]); + MONT_MAC(w0, w1, w2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MAC(t0, t1, t2, a[0], b[11]); + MONT_MUL(v0, v1, v2, a[1], b[10]); + MONT_MUL(w0, w1, w2, a[2], b[9]); + MONT_MAC(t0, t1, t2, a[3], b[8]); + MONT_MAC(v0, v1, v2, a[4], b[7]); + MONT_MAC(w0, w1, w2, a[5], b[6]); + MONT_MAC(t0, t1, t2, a[6], b[5]); + MONT_MAC(v0, v1, v2, a[7], b[4]); + MONT_MAC(w0, w1, w2, a[8], b[3]); + MONT_MAC(t0, t1, t2, a[9], b[2]); + MONT_MAC(v0, v1, v2, a[10], b[1]); + MONT_MAC(w0, w1, w2, a[11], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MAC(v0, v1, v2, u[1], m[10]); + MONT_MAC(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MAC(t0, t1, t2, a[0], b[12]); + MONT_MUL(v0, v1, v2, a[1], b[11]); + MONT_MUL(w0, w1, w2, a[2], b[10]); + MONT_MAC(t0, t1, t2, a[3], b[9]); + MONT_MAC(v0, v1, v2, a[4], b[8]); + MONT_MAC(w0, w1, w2, a[5], b[7]); + MONT_MAC(t0, t1, t2, a[6], b[6]); + MONT_MAC(v0, v1, v2, a[7], b[5]); + MONT_MAC(w0, w1, w2, a[8], b[4]); + MONT_MAC(t0, t1, t2, a[9], b[3]); + MONT_MAC(v0, v1, v2, a[10], b[2]); + MONT_MAC(w0, w1, w2, a[11], b[1]); + MONT_MAC(t0, t1, t2, a[12], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[12]); + MONT_MAC(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MAC(t0, t1, t2, a[0], b[13]); + MONT_MUL(v0, v1, v2, a[1], b[12]); + MONT_MUL(w0, w1, w2, a[2], b[11]); + MONT_MAC(t0, t1, t2, a[3], b[10]); + MONT_MAC(v0, v1, v2, a[4], b[9]); + MONT_MAC(w0, w1, w2, a[5], b[8]); + MONT_MAC(t0, t1, t2, a[6], b[7]); + MONT_MAC(v0, v1, v2, a[7], b[6]); + MONT_MAC(w0, w1, w2, a[8], b[5]); + MONT_MAC(t0, t1, t2, a[9], b[4]); + MONT_MAC(v0, v1, v2, a[10], b[3]); + MONT_MAC(w0, w1, w2, a[11], b[2]); + MONT_MAC(t0, t1, t2, a[12], b[1]); + MONT_MAC(v0, v1, v2, a[13], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[13]); + MONT_MAC(t0, t1, t2, u[1], m[12]); + MONT_MAC(v0, v1, v2, u[2], m[11]); + MONT_MAC(w0, w1, w2, u[3], m[10]); + MONT_MAC(t0, t1, t2, u[4], m[9]); + MONT_MAC(v0, v1, v2, u[5], m[8]); + MONT_MAC(w0, w1, w2, u[6], m[7]); + MONT_MAC(t0, t1, t2, u[7], m[6]); + MONT_MAC(v0, v1, v2, u[8], m[5]); + MONT_MAC(w0, w1, w2, u[9], m[4]); + MONT_MAC(t0, t1, t2, u[10], m[3]); + MONT_MAC(v0, v1, v2, u[11], m[2]); + MONT_MAC(w0, w1, w2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MAC(t0, t1, t2, a[0], b[14]); + MONT_MUL(v0, v1, v2, a[1], b[13]); + MONT_MUL(w0, w1, w2, a[2], b[12]); + MONT_MAC(t0, t1, t2, a[3], b[11]); + MONT_MAC(v0, v1, v2, a[4], b[10]); + MONT_MAC(w0, w1, w2, a[5], b[9]); + MONT_MAC(t0, t1, t2, a[6], b[8]); + MONT_MAC(v0, v1, v2, a[7], b[7]); + MONT_MAC(w0, w1, w2, a[8], b[6]); + MONT_MAC(t0, t1, t2, a[9], b[5]); + MONT_MAC(v0, v1, v2, a[10], b[4]); + MONT_MAC(w0, w1, w2, a[11], b[3]); + MONT_MAC(t0, t1, t2, a[12], b[2]); + MONT_MAC(v0, v1, v2, a[13], b[1]); + MONT_MAC(w0, w1, w2, a[14], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[14]); + MONT_MAC(v0, v1, v2, u[1], m[13]); + MONT_MAC(w0, w1, w2, u[2], m[12]); + MONT_MAC(t0, t1, t2, u[3], m[11]); + MONT_MAC(v0, v1, v2, u[4], m[10]); + MONT_MAC(w0, w1, w2, u[5], m[9]); + MONT_MAC(t0, t1, t2, u[6], m[8]); + MONT_MAC(v0, v1, v2, u[7], m[7]); + MONT_MAC(w0, w1, w2, u[8], m[6]); + MONT_MAC(t0, t1, t2, u[9], m[5]); + MONT_MAC(v0, v1, v2, u[10], m[4]); + MONT_MAC(w0, w1, w2, u[11], m[3]); + MONT_MAC(t0, t1, t2, u[12], m[2]); + MONT_MAC(v0, v1, v2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MAC(t0, t1, t2, a[0], b[15]); + MONT_MUL(v0, v1, v2, a[1], b[14]); + MONT_MUL(w0, w1, w2, a[2], b[13]); + MONT_MAC(t0, t1, t2, a[3], b[12]); + MONT_MAC(v0, v1, v2, a[4], b[11]); + MONT_MAC(w0, w1, w2, a[5], b[10]); + MONT_MAC(t0, t1, t2, a[6], b[9]); + MONT_MAC(v0, v1, v2, a[7], b[8]); + MONT_MAC(w0, w1, w2, a[8], b[7]); + MONT_MAC(t0, t1, t2, a[9], b[6]); + MONT_MAC(v0, v1, v2, a[10], b[5]); + MONT_MAC(w0, w1, w2, a[11], b[4]); + MONT_MAC(t0, t1, t2, a[12], b[3]); + MONT_MAC(v0, v1, v2, a[13], b[2]); + MONT_MAC(w0, w1, w2, a[14], b[1]); + MONT_MAC(t0, t1, t2, a[15], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[15]); + MONT_MAC(w0, w1, w2, u[1], m[14]); + MONT_MAC(t0, t1, t2, u[2], m[13]); + MONT_MAC(v0, v1, v2, u[3], m[12]); + MONT_MAC(w0, w1, w2, u[4], m[11]); + MONT_MAC(t0, t1, t2, u[5], m[10]); + MONT_MAC(v0, v1, v2, u[6], m[9]); + MONT_MAC(w0, w1, w2, u[7], m[8]); + MONT_MAC(t0, t1, t2, u[8], m[7]); + MONT_MAC(v0, v1, v2, u[9], m[6]); + MONT_MAC(w0, w1, w2, u[10], m[5]); + MONT_MAC(t0, t1, t2, u[11], m[4]); + MONT_MAC(v0, v1, v2, u[12], m[3]); + MONT_MAC(w0, w1, w2, u[13], m[2]); + MONT_MAC(t0, t1, t2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MAC(t0, t1, t2, a[0], b[16]); + MONT_MUL(v0, v1, v2, a[1], b[15]); + MONT_MUL(w0, w1, w2, a[2], b[14]); + MONT_MAC(t0, t1, t2, a[3], b[13]); + MONT_MAC(v0, v1, v2, a[4], b[12]); + MONT_MAC(w0, w1, w2, a[5], b[11]); + MONT_MAC(t0, t1, t2, a[6], b[10]); + MONT_MAC(v0, v1, v2, a[7], b[9]); + MONT_MAC(w0, w1, w2, a[8], b[8]); + MONT_MAC(t0, t1, t2, a[9], b[7]); + MONT_MAC(v0, v1, v2, a[10], b[6]); + MONT_MAC(w0, w1, w2, a[11], b[5]); + MONT_MAC(t0, t1, t2, a[12], b[4]); + MONT_MAC(v0, v1, v2, a[13], b[3]); + MONT_MAC(w0, w1, w2, a[14], b[2]); + MONT_MAC(t0, t1, t2, a[15], b[1]); + MONT_MAC(v0, v1, v2, a[16], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[16]); + MONT_MAC(t0, t1, t2, u[1], m[15]); + MONT_MAC(v0, v1, v2, u[2], m[14]); + MONT_MAC(w0, w1, w2, u[3], m[13]); + MONT_MAC(t0, t1, t2, u[4], m[12]); + MONT_MAC(v0, v1, v2, u[5], m[11]); + MONT_MAC(w0, w1, w2, u[6], m[10]); + MONT_MAC(t0, t1, t2, u[7], m[9]); + MONT_MAC(v0, v1, v2, u[8], m[8]); + MONT_MAC(w0, w1, w2, u[9], m[7]); + MONT_MAC(t0, t1, t2, u[10], m[6]); + MONT_MAC(v0, v1, v2, u[11], m[5]); + MONT_MAC(w0, w1, w2, u[12], m[4]); + MONT_MAC(t0, t1, t2, u[13], m[3]); + MONT_MAC(v0, v1, v2, u[14], m[2]); + MONT_MAC(w0, w1, w2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[16] = t0 * n0; + MONT_MAC(t0, t1, t2, u[16], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MAC(t0, t1, t2, a[0], b[17]); + MONT_MUL(v0, v1, v2, a[1], b[16]); + MONT_MUL(w0, w1, w2, a[2], b[15]); + MONT_MAC(t0, t1, t2, a[3], b[14]); + MONT_MAC(v0, v1, v2, a[4], b[13]); + MONT_MAC(w0, w1, w2, a[5], b[12]); + MONT_MAC(t0, t1, t2, a[6], b[11]); + MONT_MAC(v0, v1, v2, a[7], b[10]); + MONT_MAC(w0, w1, w2, a[8], b[9]); + MONT_MAC(t0, t1, t2, a[9], b[8]); + MONT_MAC(v0, v1, v2, a[10], b[7]); + MONT_MAC(w0, w1, w2, a[11], b[6]); + MONT_MAC(t0, t1, t2, a[12], b[5]); + MONT_MAC(v0, v1, v2, a[13], b[4]); + MONT_MAC(w0, w1, w2, a[14], b[3]); + MONT_MAC(t0, t1, t2, a[15], b[2]); + MONT_MAC(v0, v1, v2, a[16], b[1]); + MONT_MAC(w0, w1, w2, a[17], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[17]); + MONT_MAC(v0, v1, v2, u[1], m[16]); + MONT_MAC(w0, w1, w2, u[2], m[15]); + MONT_MAC(t0, t1, t2, u[3], m[14]); + MONT_MAC(v0, v1, v2, u[4], m[13]); + MONT_MAC(w0, w1, w2, u[5], m[12]); + MONT_MAC(t0, t1, t2, u[6], m[11]); + MONT_MAC(v0, v1, v2, u[7], m[10]); + MONT_MAC(w0, w1, w2, u[8], m[9]); + MONT_MAC(t0, t1, t2, u[9], m[8]); + MONT_MAC(v0, v1, v2, u[10], m[7]); + MONT_MAC(w0, w1, w2, u[11], m[6]); + MONT_MAC(t0, t1, t2, u[12], m[5]); + MONT_MAC(v0, v1, v2, u[13], m[4]); + MONT_MAC(w0, w1, w2, u[14], m[3]); + MONT_MAC(t0, t1, t2, u[15], m[2]); + MONT_MAC(v0, v1, v2, u[16], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[17] = t0 * n0; + MONT_MAC(t0, t1, t2, u[17], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MAC(t0, t1, t2, a[0], b[18]); + MONT_MUL(v0, v1, v2, a[1], b[17]); + MONT_MUL(w0, w1, w2, a[2], b[16]); + MONT_MAC(t0, t1, t2, a[3], b[15]); + MONT_MAC(v0, v1, v2, a[4], b[14]); + MONT_MAC(w0, w1, w2, a[5], b[13]); + MONT_MAC(t0, t1, t2, a[6], b[12]); + MONT_MAC(v0, v1, v2, a[7], b[11]); + MONT_MAC(w0, w1, w2, a[8], b[10]); + MONT_MAC(t0, t1, t2, a[9], b[9]); + MONT_MAC(v0, v1, v2, a[10], b[8]); + MONT_MAC(w0, w1, w2, a[11], b[7]); + MONT_MAC(t0, t1, t2, a[12], b[6]); + MONT_MAC(v0, v1, v2, a[13], b[5]); + MONT_MAC(w0, w1, w2, a[14], b[4]); + MONT_MAC(t0, t1, t2, a[15], b[3]); + MONT_MAC(v0, v1, v2, a[16], b[2]); + MONT_MAC(w0, w1, w2, a[17], b[1]); + MONT_MAC(t0, t1, t2, a[18], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[18]); + MONT_MAC(w0, w1, w2, u[1], m[17]); + MONT_MAC(t0, t1, t2, u[2], m[16]); + MONT_MAC(v0, v1, v2, u[3], m[15]); + MONT_MAC(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_MAC(w0, w1, w2, u[16], m[2]); + MONT_MAC(t0, t1, t2, u[17], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[18] = t0 * n0; + MONT_MAC(t0, t1, t2, u[18], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MAC(t0, t1, t2, a[0], b[19]); + MONT_MUL(v0, v1, v2, a[1], b[18]); + MONT_MUL(w0, w1, w2, a[2], b[17]); + MONT_MAC(t0, t1, t2, a[3], b[16]); + MONT_MAC(v0, v1, v2, a[4], b[15]); + MONT_MAC(w0, w1, w2, a[5], b[14]); + MONT_MAC(t0, t1, t2, a[6], b[13]); + MONT_MAC(v0, v1, v2, a[7], b[12]); + MONT_MAC(w0, w1, w2, a[8], b[11]); + MONT_MAC(t0, t1, t2, a[9], b[10]); + MONT_MAC(v0, v1, v2, a[10], b[9]); + MONT_MAC(w0, w1, w2, a[11], b[8]); + MONT_MAC(t0, t1, t2, a[12], b[7]); + MONT_MAC(v0, v1, v2, a[13], b[6]); + MONT_MAC(w0, w1, w2, a[14], b[5]); + MONT_MAC(t0, t1, t2, a[15], b[4]); + MONT_MAC(v0, v1, v2, a[16], b[3]); + MONT_MAC(w0, w1, w2, a[17], b[2]); + MONT_MAC(t0, t1, t2, a[18], b[1]); + MONT_MAC(v0, v1, v2, a[19], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[19]); + MONT_MAC(t0, t1, t2, u[1], m[18]); + MONT_MAC(v0, v1, v2, u[2], m[17]); + MONT_MAC(w0, w1, w2, u[3], m[16]); + MONT_MAC(t0, t1, t2, u[4], m[15]); + MONT_MAC(v0, v1, v2, u[5], m[14]); + MONT_MAC(w0, w1, w2, u[6], m[13]); + MONT_MAC(t0, t1, t2, u[7], m[12]); + MONT_MAC(v0, v1, v2, u[8], m[11]); + MONT_MAC(w0, w1, w2, u[9], m[10]); + MONT_MAC(t0, t1, t2, u[10], m[9]); + MONT_MAC(v0, v1, v2, u[11], m[8]); + MONT_MAC(w0, w1, w2, u[12], m[7]); + MONT_MAC(t0, t1, t2, u[13], m[6]); + MONT_MAC(v0, v1, v2, u[14], m[5]); + MONT_MAC(w0, w1, w2, u[15], m[4]); + MONT_MAC(t0, t1, t2, u[16], m[3]); + MONT_MAC(v0, v1, v2, u[17], m[2]); + MONT_MAC(w0, w1, w2, u[18], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[19] = t0 * n0; + MONT_MAC(t0, t1, t2, u[19], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MAC(t0, t1, t2, a[0], b[20]); + MONT_MUL(v0, v1, v2, a[1], b[19]); + MONT_MUL(w0, w1, w2, a[2], b[18]); + MONT_MAC(t0, t1, t2, a[3], b[17]); + MONT_MAC(v0, v1, v2, a[4], b[16]); + MONT_MAC(w0, w1, w2, a[5], b[15]); + MONT_MAC(t0, t1, t2, a[6], b[14]); + MONT_MAC(v0, v1, v2, a[7], b[13]); + MONT_MAC(w0, w1, w2, a[8], b[12]); + MONT_MAC(t0, t1, t2, a[9], b[11]); + MONT_MAC(v0, v1, v2, a[10], b[10]); + MONT_MAC(w0, w1, w2, a[11], b[9]); + MONT_MAC(t0, t1, t2, a[12], b[8]); + MONT_MAC(v0, v1, v2, a[13], b[7]); + MONT_MAC(w0, w1, w2, a[14], b[6]); + MONT_MAC(t0, t1, t2, a[15], b[5]); + MONT_MAC(v0, v1, v2, a[16], b[4]); + MONT_MAC(w0, w1, w2, a[17], b[3]); + MONT_MAC(t0, t1, t2, a[18], b[2]); + MONT_MAC(v0, v1, v2, a[19], b[1]); + MONT_MAC(w0, w1, w2, a[20], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[20]); + MONT_MAC(v0, v1, v2, u[1], m[19]); + MONT_MAC(w0, w1, w2, u[2], m[18]); + MONT_MAC(t0, t1, t2, u[3], m[17]); + MONT_MAC(v0, v1, v2, u[4], m[16]); + MONT_MAC(w0, w1, w2, u[5], m[15]); + MONT_MAC(t0, t1, t2, u[6], m[14]); + MONT_MAC(v0, v1, v2, u[7], m[13]); + MONT_MAC(w0, w1, w2, u[8], m[12]); + MONT_MAC(t0, t1, t2, u[9], m[11]); + MONT_MAC(v0, v1, v2, u[10], m[10]); + MONT_MAC(w0, w1, w2, u[11], m[9]); + MONT_MAC(t0, t1, t2, u[12], m[8]); + MONT_MAC(v0, v1, v2, u[13], m[7]); + MONT_MAC(w0, w1, w2, u[14], m[6]); + MONT_MAC(t0, t1, t2, u[15], m[5]); + MONT_MAC(v0, v1, v2, u[16], m[4]); + MONT_MAC(w0, w1, w2, u[17], m[3]); + MONT_MAC(t0, t1, t2, u[18], m[2]); + MONT_MAC(v0, v1, v2, u[19], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[20] = t0 * n0; + MONT_MAC(t0, t1, t2, u[20], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MAC(t0, t1, t2, a[0], b[21]); + MONT_MUL(v0, v1, v2, a[1], b[20]); + MONT_MUL(w0, w1, w2, a[2], b[19]); + MONT_MAC(t0, t1, t2, a[3], b[18]); + MONT_MAC(v0, v1, v2, a[4], b[17]); + MONT_MAC(w0, w1, w2, a[5], b[16]); + MONT_MAC(t0, t1, t2, a[6], b[15]); + MONT_MAC(v0, v1, v2, a[7], b[14]); + MONT_MAC(w0, w1, w2, a[8], b[13]); + MONT_MAC(t0, t1, t2, a[9], b[12]); + MONT_MAC(v0, v1, v2, a[10], b[11]); + MONT_MAC(w0, w1, w2, a[11], b[10]); + MONT_MAC(t0, t1, t2, a[12], b[9]); + MONT_MAC(v0, v1, v2, a[13], b[8]); + MONT_MAC(w0, w1, w2, a[14], b[7]); + MONT_MAC(t0, t1, t2, a[15], b[6]); + MONT_MAC(v0, v1, v2, a[16], b[5]); + MONT_MAC(w0, w1, w2, a[17], b[4]); + MONT_MAC(t0, t1, t2, a[18], b[3]); + MONT_MAC(v0, v1, v2, a[19], b[2]); + MONT_MAC(w0, w1, w2, a[20], b[1]); + MONT_MAC(t0, t1, t2, a[21], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[21]); + MONT_MAC(w0, w1, w2, u[1], m[20]); + MONT_MAC(t0, t1, t2, u[2], m[19]); + MONT_MAC(v0, v1, v2, u[3], m[18]); + MONT_MAC(w0, w1, w2, u[4], m[17]); + MONT_MAC(t0, t1, t2, u[5], m[16]); + MONT_MAC(v0, v1, v2, u[6], m[15]); + MONT_MAC(w0, w1, w2, u[7], m[14]); + MONT_MAC(t0, t1, t2, u[8], m[13]); + MONT_MAC(v0, v1, v2, u[9], m[12]); + MONT_MAC(w0, w1, w2, u[10], m[11]); + MONT_MAC(t0, t1, t2, u[11], m[10]); + MONT_MAC(v0, v1, v2, u[12], m[9]); + MONT_MAC(w0, w1, w2, u[13], m[8]); + MONT_MAC(t0, t1, t2, u[14], m[7]); + MONT_MAC(v0, v1, v2, u[15], m[6]); + MONT_MAC(w0, w1, w2, u[16], m[5]); + MONT_MAC(t0, t1, t2, u[17], m[4]); + MONT_MAC(v0, v1, v2, u[18], m[3]); + MONT_MAC(w0, w1, w2, u[19], m[2]); + MONT_MAC(t0, t1, t2, u[20], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[21] = t0 * n0; + MONT_MAC(t0, t1, t2, u[21], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MAC(t0, t1, t2, a[0], b[22]); + MONT_MUL(v0, v1, v2, a[1], b[21]); + MONT_MUL(w0, w1, w2, a[2], b[20]); + MONT_MAC(t0, t1, t2, a[3], b[19]); + MONT_MAC(v0, v1, v2, a[4], b[18]); + MONT_MAC(w0, w1, w2, a[5], b[17]); + MONT_MAC(t0, t1, t2, a[6], b[16]); + MONT_MAC(v0, v1, v2, a[7], b[15]); + MONT_MAC(w0, w1, w2, a[8], b[14]); + MONT_MAC(t0, t1, t2, a[9], b[13]); + MONT_MAC(v0, v1, v2, a[10], b[12]); + MONT_MAC(w0, w1, w2, a[11], b[11]); + MONT_MAC(t0, t1, t2, a[12], b[10]); + MONT_MAC(v0, v1, v2, a[13], b[9]); + MONT_MAC(w0, w1, w2, a[14], b[8]); + MONT_MAC(t0, t1, t2, a[15], b[7]); + MONT_MAC(v0, v1, v2, a[16], b[6]); + MONT_MAC(w0, w1, w2, a[17], b[5]); + MONT_MAC(t0, t1, t2, a[18], b[4]); + MONT_MAC(v0, v1, v2, a[19], b[3]); + MONT_MAC(w0, w1, w2, a[20], b[2]); + MONT_MAC(t0, t1, t2, a[21], b[1]); + MONT_MAC(v0, v1, v2, a[22], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[22]); + MONT_MAC(t0, t1, t2, u[1], m[21]); + MONT_MAC(v0, v1, v2, u[2], m[20]); + MONT_MAC(w0, w1, w2, u[3], m[19]); + MONT_MAC(t0, t1, t2, u[4], m[18]); + MONT_MAC(v0, v1, v2, u[5], m[17]); + MONT_MAC(w0, w1, w2, u[6], m[16]); + MONT_MAC(t0, t1, t2, u[7], m[15]); + MONT_MAC(v0, v1, v2, u[8], m[14]); + MONT_MAC(w0, w1, w2, u[9], m[13]); + MONT_MAC(t0, t1, t2, u[10], m[12]); + MONT_MAC(v0, v1, v2, u[11], m[11]); + MONT_MAC(w0, w1, w2, u[12], m[10]); + MONT_MAC(t0, t1, t2, u[13], m[9]); + MONT_MAC(v0, v1, v2, u[14], m[8]); + MONT_MAC(w0, w1, w2, u[15], m[7]); + MONT_MAC(t0, t1, t2, u[16], m[6]); + MONT_MAC(v0, v1, v2, u[17], m[5]); + MONT_MAC(w0, w1, w2, u[18], m[4]); + MONT_MAC(t0, t1, t2, u[19], m[3]); + MONT_MAC(v0, v1, v2, u[20], m[2]); + MONT_MAC(w0, w1, w2, u[21], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[22] = t0 * n0; + MONT_MAC(t0, t1, t2, u[22], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MAC(t0, t1, t2, a[0], b[23]); + MONT_MUL(v0, v1, v2, a[1], b[22]); + MONT_MUL(w0, w1, w2, a[2], b[21]); + MONT_MAC(t0, t1, t2, a[3], b[20]); + MONT_MAC(v0, v1, v2, a[4], b[19]); + MONT_MAC(w0, w1, w2, a[5], b[18]); + MONT_MAC(t0, t1, t2, a[6], b[17]); + MONT_MAC(v0, v1, v2, a[7], b[16]); + MONT_MAC(w0, w1, w2, a[8], b[15]); + MONT_MAC(t0, t1, t2, a[9], b[14]); + MONT_MAC(v0, v1, v2, a[10], b[13]); + MONT_MAC(w0, w1, w2, a[11], b[12]); + MONT_MAC(t0, t1, t2, a[12], b[11]); + MONT_MAC(v0, v1, v2, a[13], b[10]); + MONT_MAC(w0, w1, w2, a[14], b[9]); + MONT_MAC(t0, t1, t2, a[15], b[8]); + MONT_MAC(v0, v1, v2, a[16], b[7]); + MONT_MAC(w0, w1, w2, a[17], b[6]); + MONT_MAC(t0, t1, t2, a[18], b[5]); + MONT_MAC(v0, v1, v2, a[19], b[4]); + MONT_MAC(w0, w1, w2, a[20], b[3]); + MONT_MAC(t0, t1, t2, a[21], b[2]); + MONT_MAC(v0, v1, v2, a[22], b[1]); + MONT_MAC(w0, w1, w2, a[23], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[23]); + MONT_MAC(v0, v1, v2, u[1], m[22]); + MONT_MAC(w0, w1, w2, u[2], m[21]); + MONT_MAC(t0, t1, t2, u[3], m[20]); + MONT_MAC(v0, v1, v2, u[4], m[19]); + MONT_MAC(w0, w1, w2, u[5], m[18]); + MONT_MAC(t0, t1, t2, u[6], m[17]); + MONT_MAC(v0, v1, v2, u[7], m[16]); + MONT_MAC(w0, w1, w2, u[8], m[15]); + MONT_MAC(t0, t1, t2, u[9], m[14]); + MONT_MAC(v0, v1, v2, u[10], m[13]); + MONT_MAC(w0, w1, w2, u[11], m[12]); + MONT_MAC(t0, t1, t2, u[12], m[11]); + MONT_MAC(v0, v1, v2, u[13], m[10]); + MONT_MAC(w0, w1, w2, u[14], m[9]); + MONT_MAC(t0, t1, t2, u[15], m[8]); + MONT_MAC(v0, v1, v2, u[16], m[7]); + MONT_MAC(w0, w1, w2, u[17], m[6]); + MONT_MAC(t0, t1, t2, u[18], m[5]); + MONT_MAC(v0, v1, v2, u[19], m[4]); + MONT_MAC(w0, w1, w2, u[20], m[3]); + MONT_MAC(t0, t1, t2, u[21], m[2]); + MONT_MAC(v0, v1, v2, u[22], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[23] = t0 * n0; + MONT_MAC(t0, t1, t2, u[23], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MAC(t0, t1, t2, a[0], b[24]); + MONT_MUL(v0, v1, v2, a[1], b[23]); + MONT_MUL(w0, w1, w2, a[2], b[22]); + MONT_MAC(t0, t1, t2, a[3], b[21]); + MONT_MAC(v0, v1, v2, a[4], b[20]); + MONT_MAC(w0, w1, w2, a[5], b[19]); + MONT_MAC(t0, t1, t2, a[6], b[18]); + MONT_MAC(v0, v1, v2, a[7], b[17]); + MONT_MAC(w0, w1, w2, a[8], b[16]); + MONT_MAC(t0, t1, t2, a[9], b[15]); + MONT_MAC(v0, v1, v2, a[10], b[14]); + MONT_MAC(w0, w1, w2, a[11], b[13]); + MONT_MAC(t0, t1, t2, a[12], b[12]); + MONT_MAC(v0, v1, v2, a[13], b[11]); + MONT_MAC(w0, w1, w2, a[14], b[10]); + MONT_MAC(t0, t1, t2, a[15], b[9]); + MONT_MAC(v0, v1, v2, a[16], b[8]); + MONT_MAC(w0, w1, w2, a[17], b[7]); + MONT_MAC(t0, t1, t2, a[18], b[6]); + MONT_MAC(v0, v1, v2, a[19], b[5]); + MONT_MAC(w0, w1, w2, a[20], b[4]); + MONT_MAC(t0, t1, t2, a[21], b[3]); + MONT_MAC(v0, v1, v2, a[22], b[2]); + MONT_MAC(w0, w1, w2, a[23], b[1]); + MONT_MAC(t0, t1, t2, a[24], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[24]); + MONT_MAC(w0, w1, w2, u[1], m[23]); + MONT_MAC(t0, t1, t2, u[2], m[22]); + MONT_MAC(v0, v1, v2, u[3], m[21]); + MONT_MAC(w0, w1, w2, u[4], m[20]); + MONT_MAC(t0, t1, t2, u[5], m[19]); + MONT_MAC(v0, v1, v2, u[6], m[18]); + MONT_MAC(w0, w1, w2, u[7], m[17]); + MONT_MAC(t0, t1, t2, u[8], m[16]); + MONT_MAC(v0, v1, v2, u[9], m[15]); + MONT_MAC(w0, w1, w2, u[10], m[14]); + MONT_MAC(t0, t1, t2, u[11], m[13]); + MONT_MAC(v0, v1, v2, u[12], m[12]); + MONT_MAC(w0, w1, w2, u[13], m[11]); + MONT_MAC(t0, t1, t2, u[14], m[10]); + MONT_MAC(v0, v1, v2, u[15], m[9]); + MONT_MAC(w0, w1, w2, u[16], m[8]); + MONT_MAC(t0, t1, t2, u[17], m[7]); + MONT_MAC(v0, v1, v2, u[18], m[6]); + MONT_MAC(w0, w1, w2, u[19], m[5]); + MONT_MAC(t0, t1, t2, u[20], m[4]); + MONT_MAC(v0, v1, v2, u[21], m[3]); + MONT_MAC(w0, w1, w2, u[22], m[2]); + MONT_MAC(t0, t1, t2, u[23], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[24] = t0 * n0; + MONT_MAC(t0, t1, t2, u[24], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MAC(t0, t1, t2, a[0], b[25]); + MONT_MUL(v0, v1, v2, a[1], b[24]); + MONT_MUL(w0, w1, w2, a[2], b[23]); + MONT_MAC(t0, t1, t2, a[3], b[22]); + MONT_MAC(v0, v1, v2, a[4], b[21]); + MONT_MAC(w0, w1, w2, a[5], b[20]); + MONT_MAC(t0, t1, t2, a[6], b[19]); + MONT_MAC(v0, v1, v2, a[7], b[18]); + MONT_MAC(w0, w1, w2, a[8], b[17]); + MONT_MAC(t0, t1, t2, a[9], b[16]); + MONT_MAC(v0, v1, v2, a[10], b[15]); + MONT_MAC(w0, w1, w2, a[11], b[14]); + MONT_MAC(t0, t1, t2, a[12], b[13]); + MONT_MAC(v0, v1, v2, a[13], b[12]); + MONT_MAC(w0, w1, w2, a[14], b[11]); + MONT_MAC(t0, t1, t2, a[15], b[10]); + MONT_MAC(v0, v1, v2, a[16], b[9]); + MONT_MAC(w0, w1, w2, a[17], b[8]); + MONT_MAC(t0, t1, t2, a[18], b[7]); + MONT_MAC(v0, v1, v2, a[19], b[6]); + MONT_MAC(w0, w1, w2, a[20], b[5]); + MONT_MAC(t0, t1, t2, a[21], b[4]); + MONT_MAC(v0, v1, v2, a[22], b[3]); + MONT_MAC(w0, w1, w2, a[23], b[2]); + MONT_MAC(t0, t1, t2, a[24], b[1]); + MONT_MAC(v0, v1, v2, a[25], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[25]); + MONT_MAC(t0, t1, t2, u[1], m[24]); + MONT_MAC(v0, v1, v2, u[2], m[23]); + MONT_MAC(w0, w1, w2, u[3], m[22]); + MONT_MAC(t0, t1, t2, u[4], m[21]); + MONT_MAC(v0, v1, v2, u[5], m[20]); + MONT_MAC(w0, w1, w2, u[6], m[19]); + MONT_MAC(t0, t1, t2, u[7], m[18]); + MONT_MAC(v0, v1, v2, u[8], m[17]); + MONT_MAC(w0, w1, w2, u[9], m[16]); + MONT_MAC(t0, t1, t2, u[10], m[15]); + MONT_MAC(v0, v1, v2, u[11], m[14]); + MONT_MAC(w0, w1, w2, u[12], m[13]); + MONT_MAC(t0, t1, t2, u[13], m[12]); + MONT_MAC(v0, v1, v2, u[14], m[11]); + MONT_MAC(w0, w1, w2, u[15], m[10]); + MONT_MAC(t0, t1, t2, u[16], m[9]); + MONT_MAC(v0, v1, v2, u[17], m[8]); + MONT_MAC(w0, w1, w2, u[18], m[7]); + MONT_MAC(t0, t1, t2, u[19], m[6]); + MONT_MAC(v0, v1, v2, u[20], m[5]); + MONT_MAC(w0, w1, w2, u[21], m[4]); + MONT_MAC(t0, t1, t2, u[22], m[3]); + MONT_MAC(v0, v1, v2, u[23], m[2]); + MONT_MAC(w0, w1, w2, u[24], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[25] = t0 * n0; + MONT_MAC(t0, t1, t2, u[25], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MAC(t0, t1, t2, a[0], b[26]); + MONT_MUL(v0, v1, v2, a[1], b[25]); + MONT_MUL(w0, w1, w2, a[2], b[24]); + MONT_MAC(t0, t1, t2, a[3], b[23]); + MONT_MAC(v0, v1, v2, a[4], b[22]); + MONT_MAC(w0, w1, w2, a[5], b[21]); + MONT_MAC(t0, t1, t2, a[6], b[20]); + MONT_MAC(v0, v1, v2, a[7], b[19]); + MONT_MAC(w0, w1, w2, a[8], b[18]); + MONT_MAC(t0, t1, t2, a[9], b[17]); + MONT_MAC(v0, v1, v2, a[10], b[16]); + MONT_MAC(w0, w1, w2, a[11], b[15]); + MONT_MAC(t0, t1, t2, a[12], b[14]); + MONT_MAC(v0, v1, v2, a[13], b[13]); + MONT_MAC(w0, w1, w2, a[14], b[12]); + MONT_MAC(t0, t1, t2, a[15], b[11]); + MONT_MAC(v0, v1, v2, a[16], b[10]); + MONT_MAC(w0, w1, w2, a[17], b[9]); + MONT_MAC(t0, t1, t2, a[18], b[8]); + MONT_MAC(v0, v1, v2, a[19], b[7]); + MONT_MAC(w0, w1, w2, a[20], b[6]); + MONT_MAC(t0, t1, t2, a[21], b[5]); + MONT_MAC(v0, v1, v2, a[22], b[4]); + MONT_MAC(w0, w1, w2, a[23], b[3]); + MONT_MAC(t0, t1, t2, a[24], b[2]); + MONT_MAC(v0, v1, v2, a[25], b[1]); + MONT_MAC(w0, w1, w2, a[26], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[26]); + MONT_MAC(v0, v1, v2, u[1], m[25]); + MONT_MAC(w0, w1, w2, u[2], m[24]); + MONT_MAC(t0, t1, t2, u[3], m[23]); + MONT_MAC(v0, v1, v2, u[4], m[22]); + MONT_MAC(w0, w1, w2, u[5], m[21]); + MONT_MAC(t0, t1, t2, u[6], m[20]); + MONT_MAC(v0, v1, v2, u[7], m[19]); + MONT_MAC(w0, w1, w2, u[8], m[18]); + MONT_MAC(t0, t1, t2, u[9], m[17]); + MONT_MAC(v0, v1, v2, u[10], m[16]); + MONT_MAC(w0, w1, w2, u[11], m[15]); + MONT_MAC(t0, t1, t2, u[12], m[14]); + MONT_MAC(v0, v1, v2, u[13], m[13]); + MONT_MAC(w0, w1, w2, u[14], m[12]); + MONT_MAC(t0, t1, t2, u[15], m[11]); + MONT_MAC(v0, v1, v2, u[16], m[10]); + MONT_MAC(w0, w1, w2, u[17], m[9]); + MONT_MAC(t0, t1, t2, u[18], m[8]); + MONT_MAC(v0, v1, v2, u[19], m[7]); + MONT_MAC(w0, w1, w2, u[20], m[6]); + MONT_MAC(t0, t1, t2, u[21], m[5]); + MONT_MAC(v0, v1, v2, u[22], m[4]); + MONT_MAC(w0, w1, w2, u[23], m[3]); + MONT_MAC(t0, t1, t2, u[24], m[2]); + MONT_MAC(v0, v1, v2, u[25], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[26] = t0 * n0; + MONT_MAC(t0, t1, t2, u[26], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MAC(t0, t1, t2, a[0], b[27]); + MONT_MUL(v0, v1, v2, a[1], b[26]); + MONT_MUL(w0, w1, w2, a[2], b[25]); + MONT_MAC(t0, t1, t2, a[3], b[24]); + MONT_MAC(v0, v1, v2, a[4], b[23]); + MONT_MAC(w0, w1, w2, a[5], b[22]); + MONT_MAC(t0, t1, t2, a[6], b[21]); + MONT_MAC(v0, v1, v2, a[7], b[20]); + MONT_MAC(w0, w1, w2, a[8], b[19]); + MONT_MAC(t0, t1, t2, a[9], b[18]); + MONT_MAC(v0, v1, v2, a[10], b[17]); + MONT_MAC(w0, w1, w2, a[11], b[16]); + MONT_MAC(t0, t1, t2, a[12], b[15]); + MONT_MAC(v0, v1, v2, a[13], b[14]); + MONT_MAC(w0, w1, w2, a[14], b[13]); + MONT_MAC(t0, t1, t2, a[15], b[12]); + MONT_MAC(v0, v1, v2, a[16], b[11]); + MONT_MAC(w0, w1, w2, a[17], b[10]); + MONT_MAC(t0, t1, t2, a[18], b[9]); + MONT_MAC(v0, v1, v2, a[19], b[8]); + MONT_MAC(w0, w1, w2, a[20], b[7]); + MONT_MAC(t0, t1, t2, a[21], b[6]); + MONT_MAC(v0, v1, v2, a[22], b[5]); + MONT_MAC(w0, w1, w2, a[23], b[4]); + MONT_MAC(t0, t1, t2, a[24], b[3]); + MONT_MAC(v0, v1, v2, a[25], b[2]); + MONT_MAC(w0, w1, w2, a[26], b[1]); + MONT_MAC(t0, t1, t2, a[27], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[27]); + MONT_MAC(w0, w1, w2, u[1], m[26]); + MONT_MAC(t0, t1, t2, u[2], m[25]); + MONT_MAC(v0, v1, v2, u[3], m[24]); + MONT_MAC(w0, w1, w2, u[4], m[23]); + MONT_MAC(t0, t1, t2, u[5], m[22]); + MONT_MAC(v0, v1, v2, u[6], m[21]); + MONT_MAC(w0, w1, w2, u[7], m[20]); + MONT_MAC(t0, t1, t2, u[8], m[19]); + MONT_MAC(v0, v1, v2, u[9], m[18]); + MONT_MAC(w0, w1, w2, u[10], m[17]); + MONT_MAC(t0, t1, t2, u[11], m[16]); + MONT_MAC(v0, v1, v2, u[12], m[15]); + MONT_MAC(w0, w1, w2, u[13], m[14]); + MONT_MAC(t0, t1, t2, u[14], m[13]); + MONT_MAC(v0, v1, v2, u[15], m[12]); + MONT_MAC(w0, w1, w2, u[16], m[11]); + MONT_MAC(t0, t1, t2, u[17], m[10]); + MONT_MAC(v0, v1, v2, u[18], m[9]); + MONT_MAC(w0, w1, w2, u[19], m[8]); + MONT_MAC(t0, t1, t2, u[20], m[7]); + MONT_MAC(v0, v1, v2, u[21], m[6]); + MONT_MAC(w0, w1, w2, u[22], m[5]); + MONT_MAC(t0, t1, t2, u[23], m[4]); + MONT_MAC(v0, v1, v2, u[24], m[3]); + MONT_MAC(w0, w1, w2, u[25], m[2]); + MONT_MAC(t0, t1, t2, u[26], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[27] = t0 * n0; + MONT_MAC(t0, t1, t2, u[27], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MAC(t0, t1, t2, a[0], b[28]); + MONT_MUL(v0, v1, v2, a[1], b[27]); + MONT_MUL(w0, w1, w2, a[2], b[26]); + MONT_MAC(t0, t1, t2, a[3], b[25]); + MONT_MAC(v0, v1, v2, a[4], b[24]); + MONT_MAC(w0, w1, w2, a[5], b[23]); + MONT_MAC(t0, t1, t2, a[6], b[22]); + MONT_MAC(v0, v1, v2, a[7], b[21]); + MONT_MAC(w0, w1, w2, a[8], b[20]); + MONT_MAC(t0, t1, t2, a[9], b[19]); + MONT_MAC(v0, v1, v2, a[10], b[18]); + MONT_MAC(w0, w1, w2, a[11], b[17]); + MONT_MAC(t0, t1, t2, a[12], b[16]); + MONT_MAC(v0, v1, v2, a[13], b[15]); + MONT_MAC(w0, w1, w2, a[14], b[14]); + MONT_MAC(t0, t1, t2, a[15], b[13]); + MONT_MAC(v0, v1, v2, a[16], b[12]); + MONT_MAC(w0, w1, w2, a[17], b[11]); + MONT_MAC(t0, t1, t2, a[18], b[10]); + MONT_MAC(v0, v1, v2, a[19], b[9]); + MONT_MAC(w0, w1, w2, a[20], b[8]); + MONT_MAC(t0, t1, t2, a[21], b[7]); + MONT_MAC(v0, v1, v2, a[22], b[6]); + MONT_MAC(w0, w1, w2, a[23], b[5]); + MONT_MAC(t0, t1, t2, a[24], b[4]); + MONT_MAC(v0, v1, v2, a[25], b[3]); + MONT_MAC(w0, w1, w2, a[26], b[2]); + MONT_MAC(t0, t1, t2, a[27], b[1]); + MONT_MAC(v0, v1, v2, a[28], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[28]); + MONT_MAC(t0, t1, t2, u[1], m[27]); + MONT_MAC(v0, v1, v2, u[2], m[26]); + MONT_MAC(w0, w1, w2, u[3], m[25]); + MONT_MAC(t0, t1, t2, u[4], m[24]); + MONT_MAC(v0, v1, v2, u[5], m[23]); + MONT_MAC(w0, w1, w2, u[6], m[22]); + MONT_MAC(t0, t1, t2, u[7], m[21]); + MONT_MAC(v0, v1, v2, u[8], m[20]); + MONT_MAC(w0, w1, w2, u[9], m[19]); + MONT_MAC(t0, t1, t2, u[10], m[18]); + MONT_MAC(v0, v1, v2, u[11], m[17]); + MONT_MAC(w0, w1, w2, u[12], m[16]); + MONT_MAC(t0, t1, t2, u[13], m[15]); + MONT_MAC(v0, v1, v2, u[14], m[14]); + MONT_MAC(w0, w1, w2, u[15], m[13]); + MONT_MAC(t0, t1, t2, u[16], m[12]); + MONT_MAC(v0, v1, v2, u[17], m[11]); + MONT_MAC(w0, w1, w2, u[18], m[10]); + MONT_MAC(t0, t1, t2, u[19], m[9]); + MONT_MAC(v0, v1, v2, u[20], m[8]); + MONT_MAC(w0, w1, w2, u[21], m[7]); + MONT_MAC(t0, t1, t2, u[22], m[6]); + MONT_MAC(v0, v1, v2, u[23], m[5]); + MONT_MAC(w0, w1, w2, u[24], m[4]); + MONT_MAC(t0, t1, t2, u[25], m[3]); + MONT_MAC(v0, v1, v2, u[26], m[2]); + MONT_MAC(w0, w1, w2, u[27], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[28] = t0 * n0; + MONT_MAC(t0, t1, t2, u[28], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MAC(t0, t1, t2, a[0], b[29]); + MONT_MUL(v0, v1, v2, a[1], b[28]); + MONT_MUL(w0, w1, w2, a[2], b[27]); + MONT_MAC(t0, t1, t2, a[3], b[26]); + MONT_MAC(v0, v1, v2, a[4], b[25]); + MONT_MAC(w0, w1, w2, a[5], b[24]); + MONT_MAC(t0, t1, t2, a[6], b[23]); + MONT_MAC(v0, v1, v2, a[7], b[22]); + MONT_MAC(w0, w1, w2, a[8], b[21]); + MONT_MAC(t0, t1, t2, a[9], b[20]); + MONT_MAC(v0, v1, v2, a[10], b[19]); + MONT_MAC(w0, w1, w2, a[11], b[18]); + MONT_MAC(t0, t1, t2, a[12], b[17]); + MONT_MAC(v0, v1, v2, a[13], b[16]); + MONT_MAC(w0, w1, w2, a[14], b[15]); + MONT_MAC(t0, t1, t2, a[15], b[14]); + MONT_MAC(v0, v1, v2, a[16], b[13]); + MONT_MAC(w0, w1, w2, a[17], b[12]); + MONT_MAC(t0, t1, t2, a[18], b[11]); + MONT_MAC(v0, v1, v2, a[19], b[10]); + MONT_MAC(w0, w1, w2, a[20], b[9]); + MONT_MAC(t0, t1, t2, a[21], b[8]); + MONT_MAC(v0, v1, v2, a[22], b[7]); + MONT_MAC(w0, w1, w2, a[23], b[6]); + MONT_MAC(t0, t1, t2, a[24], b[5]); + MONT_MAC(v0, v1, v2, a[25], b[4]); + MONT_MAC(w0, w1, w2, a[26], b[3]); + MONT_MAC(t0, t1, t2, a[27], b[2]); + MONT_MAC(v0, v1, v2, a[28], b[1]); + MONT_MAC(w0, w1, w2, a[29], b[0]); + MONT_MAC(t0, t1, t2, u[0], m[29]); + MONT_MAC(v0, v1, v2, u[1], m[28]); + MONT_MAC(w0, w1, w2, u[2], m[27]); + MONT_MAC(t0, t1, t2, u[3], m[26]); + MONT_MAC(v0, v1, v2, u[4], m[25]); + MONT_MAC(w0, w1, w2, u[5], m[24]); + MONT_MAC(t0, t1, t2, u[6], m[23]); + MONT_MAC(v0, v1, v2, u[7], m[22]); + MONT_MAC(w0, w1, w2, u[8], m[21]); + MONT_MAC(t0, t1, t2, u[9], m[20]); + MONT_MAC(v0, v1, v2, u[10], m[19]); + MONT_MAC(w0, w1, w2, u[11], m[18]); + MONT_MAC(t0, t1, t2, u[12], m[17]); + MONT_MAC(v0, v1, v2, u[13], m[16]); + MONT_MAC(w0, w1, w2, u[14], m[15]); + MONT_MAC(t0, t1, t2, u[15], m[14]); + MONT_MAC(v0, v1, v2, u[16], m[13]); + MONT_MAC(w0, w1, w2, u[17], m[12]); + MONT_MAC(t0, t1, t2, u[18], m[11]); + MONT_MAC(v0, v1, v2, u[19], m[10]); + MONT_MAC(w0, w1, w2, u[20], m[9]); + MONT_MAC(t0, t1, t2, u[21], m[8]); + MONT_MAC(v0, v1, v2, u[22], m[7]); + MONT_MAC(w0, w1, w2, u[23], m[6]); + MONT_MAC(t0, t1, t2, u[24], m[5]); + MONT_MAC(v0, v1, v2, u[25], m[4]); + MONT_MAC(w0, w1, w2, u[26], m[3]); + MONT_MAC(t0, t1, t2, u[27], m[2]); + MONT_MAC(v0, v1, v2, u[28], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[29] = t0 * n0; + MONT_MAC(t0, t1, t2, u[29], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MAC(t0, t1, t2, a[0], b[30]); + MONT_MUL(v0, v1, v2, a[1], b[29]); + MONT_MUL(w0, w1, w2, a[2], b[28]); + MONT_MAC(t0, t1, t2, a[3], b[27]); + MONT_MAC(v0, v1, v2, a[4], b[26]); + MONT_MAC(w0, w1, w2, a[5], b[25]); + MONT_MAC(t0, t1, t2, a[6], b[24]); + MONT_MAC(v0, v1, v2, a[7], b[23]); + MONT_MAC(w0, w1, w2, a[8], b[22]); + MONT_MAC(t0, t1, t2, a[9], b[21]); + MONT_MAC(v0, v1, v2, a[10], b[20]); + MONT_MAC(w0, w1, w2, a[11], b[19]); + MONT_MAC(t0, t1, t2, a[12], b[18]); + MONT_MAC(v0, v1, v2, a[13], b[17]); + MONT_MAC(w0, w1, w2, a[14], b[16]); + MONT_MAC(t0, t1, t2, a[15], b[15]); + MONT_MAC(v0, v1, v2, a[16], b[14]); + MONT_MAC(w0, w1, w2, a[17], b[13]); + MONT_MAC(t0, t1, t2, a[18], b[12]); + MONT_MAC(v0, v1, v2, a[19], b[11]); + MONT_MAC(w0, w1, w2, a[20], b[10]); + MONT_MAC(t0, t1, t2, a[21], b[9]); + MONT_MAC(v0, v1, v2, a[22], b[8]); + MONT_MAC(w0, w1, w2, a[23], b[7]); + MONT_MAC(t0, t1, t2, a[24], b[6]); + MONT_MAC(v0, v1, v2, a[25], b[5]); + MONT_MAC(w0, w1, w2, a[26], b[4]); + MONT_MAC(t0, t1, t2, a[27], b[3]); + MONT_MAC(v0, v1, v2, a[28], b[2]); + MONT_MAC(w0, w1, w2, a[29], b[1]); + MONT_MAC(t0, t1, t2, a[30], b[0]); + MONT_MAC(v0, v1, v2, u[0], m[30]); + MONT_MAC(w0, w1, w2, u[1], m[29]); + MONT_MAC(t0, t1, t2, u[2], m[28]); + MONT_MAC(v0, v1, v2, u[3], m[27]); + MONT_MAC(w0, w1, w2, u[4], m[26]); + MONT_MAC(t0, t1, t2, u[5], m[25]); + MONT_MAC(v0, v1, v2, u[6], m[24]); + MONT_MAC(w0, w1, w2, u[7], m[23]); + MONT_MAC(t0, t1, t2, u[8], m[22]); + MONT_MAC(v0, v1, v2, u[9], m[21]); + MONT_MAC(w0, w1, w2, u[10], m[20]); + MONT_MAC(t0, t1, t2, u[11], m[19]); + MONT_MAC(v0, v1, v2, u[12], m[18]); + MONT_MAC(w0, w1, w2, u[13], m[17]); + MONT_MAC(t0, t1, t2, u[14], m[16]); + MONT_MAC(v0, v1, v2, u[15], m[15]); + MONT_MAC(w0, w1, w2, u[16], m[14]); + MONT_MAC(t0, t1, t2, u[17], m[13]); + MONT_MAC(v0, v1, v2, u[18], m[12]); + MONT_MAC(w0, w1, w2, u[19], m[11]); + MONT_MAC(t0, t1, t2, u[20], m[10]); + MONT_MAC(v0, v1, v2, u[21], m[9]); + MONT_MAC(w0, w1, w2, u[22], m[8]); + MONT_MAC(t0, t1, t2, u[23], m[7]); + MONT_MAC(v0, v1, v2, u[24], m[6]); + MONT_MAC(w0, w1, w2, u[25], m[5]); + MONT_MAC(t0, t1, t2, u[26], m[4]); + MONT_MAC(v0, v1, v2, u[27], m[3]); + MONT_MAC(w0, w1, w2, u[28], m[2]); + MONT_MAC(t0, t1, t2, u[29], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[30] = t0 * n0; + MONT_MAC(t0, t1, t2, u[30], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 31 */ + MONT_MAC(t0, t1, t2, a[0], b[31]); + MONT_MUL(v0, v1, v2, a[1], b[30]); + MONT_MUL(w0, w1, w2, a[2], b[29]); + MONT_MAC(t0, t1, t2, a[3], b[28]); + MONT_MAC(v0, v1, v2, a[4], b[27]); + MONT_MAC(w0, w1, w2, a[5], b[26]); + MONT_MAC(t0, t1, t2, a[6], b[25]); + MONT_MAC(v0, v1, v2, a[7], b[24]); + MONT_MAC(w0, w1, w2, a[8], b[23]); + MONT_MAC(t0, t1, t2, a[9], b[22]); + MONT_MAC(v0, v1, v2, a[10], b[21]); + MONT_MAC(w0, w1, w2, a[11], b[20]); + MONT_MAC(t0, t1, t2, a[12], b[19]); + MONT_MAC(v0, v1, v2, a[13], b[18]); + MONT_MAC(w0, w1, w2, a[14], b[17]); + MONT_MAC(t0, t1, t2, a[15], b[16]); + MONT_MAC(v0, v1, v2, a[16], b[15]); + MONT_MAC(w0, w1, w2, a[17], b[14]); + MONT_MAC(t0, t1, t2, a[18], b[13]); + MONT_MAC(v0, v1, v2, a[19], b[12]); + MONT_MAC(w0, w1, w2, a[20], b[11]); + MONT_MAC(t0, t1, t2, a[21], b[10]); + MONT_MAC(v0, v1, v2, a[22], b[9]); + MONT_MAC(w0, w1, w2, a[23], b[8]); + MONT_MAC(t0, t1, t2, a[24], b[7]); + MONT_MAC(v0, v1, v2, a[25], b[6]); + MONT_MAC(w0, w1, w2, a[26], b[5]); + MONT_MAC(t0, t1, t2, a[27], b[4]); + MONT_MAC(v0, v1, v2, a[28], b[3]); + MONT_MAC(w0, w1, w2, a[29], b[2]); + MONT_MAC(t0, t1, t2, a[30], b[1]); + MONT_MAC(v0, v1, v2, a[31], b[0]); + MONT_MAC(w0, w1, w2, u[0], m[31]); + MONT_MAC(t0, t1, t2, u[1], m[30]); + MONT_MAC(v0, v1, v2, u[2], m[29]); + MONT_MAC(w0, w1, w2, u[3], m[28]); + MONT_MAC(t0, t1, t2, u[4], m[27]); + MONT_MAC(v0, v1, v2, u[5], m[26]); + MONT_MAC(w0, w1, w2, u[6], m[25]); + MONT_MAC(t0, t1, t2, u[7], m[24]); + MONT_MAC(v0, v1, v2, u[8], m[23]); + MONT_MAC(w0, w1, w2, u[9], m[22]); + MONT_MAC(t0, t1, t2, u[10], m[21]); + MONT_MAC(v0, v1, v2, u[11], m[20]); + MONT_MAC(w0, w1, w2, u[12], m[19]); + MONT_MAC(t0, t1, t2, u[13], m[18]); + MONT_MAC(v0, v1, v2, u[14], m[17]); + MONT_MAC(w0, w1, w2, u[15], m[16]); + MONT_MAC(t0, t1, t2, u[16], m[15]); + MONT_MAC(v0, v1, v2, u[17], m[14]); + MONT_MAC(w0, w1, w2, u[18], m[13]); + MONT_MAC(t0, t1, t2, u[19], m[12]); + MONT_MAC(v0, v1, v2, u[20], m[11]); + MONT_MAC(w0, w1, w2, u[21], m[10]); + MONT_MAC(t0, t1, t2, u[22], m[9]); + MONT_MAC(v0, v1, v2, u[23], m[8]); + MONT_MAC(w0, w1, w2, u[24], m[7]); + MONT_MAC(t0, t1, t2, u[25], m[6]); + MONT_MAC(v0, v1, v2, u[26], m[5]); + MONT_MAC(w0, w1, w2, u[27], m[4]); + MONT_MAC(t0, t1, t2, u[28], m[3]); + MONT_MAC(v0, v1, v2, u[29], m[2]); + MONT_MAC(w0, w1, w2, u[30], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + u[31] = t0 * n0; + MONT_MAC(t0, t1, t2, u[31], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 32 */ + MONT_MAC(t0, t1, t2, a[1], b[31]); + MONT_MUL(v0, v1, v2, a[2], b[30]); + MONT_MUL(w0, w1, w2, a[3], b[29]); + MONT_MAC(t0, t1, t2, a[4], b[28]); + MONT_MAC(v0, v1, v2, a[5], b[27]); + MONT_MAC(w0, w1, w2, a[6], b[26]); + MONT_MAC(t0, t1, t2, a[7], b[25]); + MONT_MAC(v0, v1, v2, a[8], b[24]); + MONT_MAC(w0, w1, w2, a[9], b[23]); + MONT_MAC(t0, t1, t2, a[10], b[22]); + MONT_MAC(v0, v1, v2, a[11], b[21]); + MONT_MAC(w0, w1, w2, a[12], b[20]); + MONT_MAC(t0, t1, t2, a[13], b[19]); + MONT_MAC(v0, v1, v2, a[14], b[18]); + MONT_MAC(w0, w1, w2, a[15], b[17]); + MONT_MAC(t0, t1, t2, a[16], b[16]); + MONT_MAC(v0, v1, v2, a[17], b[15]); + MONT_MAC(w0, w1, w2, a[18], b[14]); + MONT_MAC(t0, t1, t2, a[19], b[13]); + MONT_MAC(v0, v1, v2, a[20], b[12]); + MONT_MAC(w0, w1, w2, a[21], b[11]); + MONT_MAC(t0, t1, t2, a[22], b[10]); + MONT_MAC(v0, v1, v2, a[23], b[9]); + MONT_MAC(w0, w1, w2, a[24], b[8]); + MONT_MAC(t0, t1, t2, a[25], b[7]); + MONT_MAC(v0, v1, v2, a[26], b[6]); + MONT_MAC(w0, w1, w2, a[27], b[5]); + MONT_MAC(t0, t1, t2, a[28], b[4]); + MONT_MAC(v0, v1, v2, a[29], b[3]); + MONT_MAC(w0, w1, w2, a[30], b[2]); + MONT_MAC(t0, t1, t2, a[31], b[1]); + MONT_MAC(v0, v1, v2, u[1], m[31]); + MONT_MAC(w0, w1, w2, u[2], m[30]); + MONT_MAC(t0, t1, t2, u[3], m[29]); + MONT_MAC(v0, v1, v2, u[4], m[28]); + MONT_MAC(w0, w1, w2, u[5], m[27]); + MONT_MAC(t0, t1, t2, u[6], m[26]); + MONT_MAC(v0, v1, v2, u[7], m[25]); + MONT_MAC(w0, w1, w2, u[8], m[24]); + MONT_MAC(t0, t1, t2, u[9], m[23]); + MONT_MAC(v0, v1, v2, u[10], m[22]); + MONT_MAC(w0, w1, w2, u[11], m[21]); + MONT_MAC(t0, t1, t2, u[12], m[20]); + MONT_MAC(v0, v1, v2, u[13], m[19]); + MONT_MAC(w0, w1, w2, u[14], m[18]); + MONT_MAC(t0, t1, t2, u[15], m[17]); + MONT_MAC(v0, v1, v2, u[16], m[16]); + MONT_MAC(w0, w1, w2, u[17], m[15]); + MONT_MAC(t0, t1, t2, u[18], m[14]); + MONT_MAC(v0, v1, v2, u[19], m[13]); + MONT_MAC(w0, w1, w2, u[20], m[12]); + MONT_MAC(t0, t1, t2, u[21], m[11]); + MONT_MAC(v0, v1, v2, u[22], m[10]); + MONT_MAC(w0, w1, w2, u[23], m[9]); + MONT_MAC(t0, t1, t2, u[24], m[8]); + MONT_MAC(v0, v1, v2, u[25], m[7]); + MONT_MAC(w0, w1, w2, u[26], m[6]); + MONT_MAC(t0, t1, t2, u[27], m[5]); + MONT_MAC(v0, v1, v2, u[28], m[4]); + MONT_MAC(w0, w1, w2, u[29], m[3]); + MONT_MAC(t0, t1, t2, u[30], m[2]); + MONT_MAC(v0, v1, v2, u[31], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 33 */ + MONT_MAC(t0, t1, t2, a[2], b[31]); + MONT_MUL(v0, v1, v2, a[3], b[30]); + MONT_MUL(w0, w1, w2, a[4], b[29]); + MONT_MAC(t0, t1, t2, a[5], b[28]); + MONT_MAC(v0, v1, v2, a[6], b[27]); + MONT_MAC(w0, w1, w2, a[7], b[26]); + MONT_MAC(t0, t1, t2, a[8], b[25]); + MONT_MAC(v0, v1, v2, a[9], b[24]); + MONT_MAC(w0, w1, w2, a[10], b[23]); + MONT_MAC(t0, t1, t2, a[11], b[22]); + MONT_MAC(v0, v1, v2, a[12], b[21]); + MONT_MAC(w0, w1, w2, a[13], b[20]); + MONT_MAC(t0, t1, t2, a[14], b[19]); + MONT_MAC(v0, v1, v2, a[15], b[18]); + MONT_MAC(w0, w1, w2, a[16], b[17]); + MONT_MAC(t0, t1, t2, a[17], b[16]); + MONT_MAC(v0, v1, v2, a[18], b[15]); + MONT_MAC(w0, w1, w2, a[19], b[14]); + MONT_MAC(t0, t1, t2, a[20], b[13]); + MONT_MAC(v0, v1, v2, a[21], b[12]); + MONT_MAC(w0, w1, w2, a[22], b[11]); + MONT_MAC(t0, t1, t2, a[23], b[10]); + MONT_MAC(v0, v1, v2, a[24], b[9]); + MONT_MAC(w0, w1, w2, a[25], b[8]); + MONT_MAC(t0, t1, t2, a[26], b[7]); + MONT_MAC(v0, v1, v2, a[27], b[6]); + MONT_MAC(w0, w1, w2, a[28], b[5]); + MONT_MAC(t0, t1, t2, a[29], b[4]); + MONT_MAC(v0, v1, v2, a[30], b[3]); + MONT_MAC(w0, w1, w2, a[31], b[2]); + MONT_MAC(t0, t1, t2, u[2], m[31]); + MONT_MAC(v0, v1, v2, u[3], m[30]); + MONT_MAC(w0, w1, w2, u[4], m[29]); + MONT_MAC(t0, t1, t2, u[5], m[28]); + MONT_MAC(v0, v1, v2, u[6], m[27]); + MONT_MAC(w0, w1, w2, u[7], m[26]); + MONT_MAC(t0, t1, t2, u[8], m[25]); + MONT_MAC(v0, v1, v2, u[9], m[24]); + MONT_MAC(w0, w1, w2, u[10], m[23]); + MONT_MAC(t0, t1, t2, u[11], m[22]); + MONT_MAC(v0, v1, v2, u[12], m[21]); + MONT_MAC(w0, w1, w2, u[13], m[20]); + MONT_MAC(t0, t1, t2, u[14], m[19]); + MONT_MAC(v0, v1, v2, u[15], m[18]); + MONT_MAC(w0, w1, w2, u[16], m[17]); + MONT_MAC(t0, t1, t2, u[17], m[16]); + MONT_MAC(v0, v1, v2, u[18], m[15]); + MONT_MAC(w0, w1, w2, u[19], m[14]); + MONT_MAC(t0, t1, t2, u[20], m[13]); + MONT_MAC(v0, v1, v2, u[21], m[12]); + MONT_MAC(w0, w1, w2, u[22], m[11]); + MONT_MAC(t0, t1, t2, u[23], m[10]); + MONT_MAC(v0, v1, v2, u[24], m[9]); + MONT_MAC(w0, w1, w2, u[25], m[8]); + MONT_MAC(t0, t1, t2, u[26], m[7]); + MONT_MAC(v0, v1, v2, u[27], m[6]); + MONT_MAC(w0, w1, w2, u[28], m[5]); + MONT_MAC(t0, t1, t2, u[29], m[4]); + MONT_MAC(v0, v1, v2, u[30], m[3]); + MONT_MAC(w0, w1, w2, u[31], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 34 */ + MONT_MAC(t0, t1, t2, a[3], b[31]); + MONT_MUL(v0, v1, v2, a[4], b[30]); + MONT_MUL(w0, w1, w2, a[5], b[29]); + MONT_MAC(t0, t1, t2, a[6], b[28]); + MONT_MAC(v0, v1, v2, a[7], b[27]); + MONT_MAC(w0, w1, w2, a[8], b[26]); + MONT_MAC(t0, t1, t2, a[9], b[25]); + MONT_MAC(v0, v1, v2, a[10], b[24]); + MONT_MAC(w0, w1, w2, a[11], b[23]); + MONT_MAC(t0, t1, t2, a[12], b[22]); + MONT_MAC(v0, v1, v2, a[13], b[21]); + MONT_MAC(w0, w1, w2, a[14], b[20]); + MONT_MAC(t0, t1, t2, a[15], b[19]); + MONT_MAC(v0, v1, v2, a[16], b[18]); + MONT_MAC(w0, w1, w2, a[17], b[17]); + MONT_MAC(t0, t1, t2, a[18], b[16]); + MONT_MAC(v0, v1, v2, a[19], b[15]); + MONT_MAC(w0, w1, w2, a[20], b[14]); + MONT_MAC(t0, t1, t2, a[21], b[13]); + MONT_MAC(v0, v1, v2, a[22], b[12]); + MONT_MAC(w0, w1, w2, a[23], b[11]); + MONT_MAC(t0, t1, t2, a[24], b[10]); + MONT_MAC(v0, v1, v2, a[25], b[9]); + MONT_MAC(w0, w1, w2, a[26], b[8]); + MONT_MAC(t0, t1, t2, a[27], b[7]); + MONT_MAC(v0, v1, v2, a[28], b[6]); + MONT_MAC(w0, w1, w2, a[29], b[5]); + MONT_MAC(t0, t1, t2, a[30], b[4]); + MONT_MAC(v0, v1, v2, a[31], b[3]); + MONT_MAC(w0, w1, w2, u[3], m[31]); + MONT_MAC(t0, t1, t2, u[4], m[30]); + MONT_MAC(v0, v1, v2, u[5], m[29]); + MONT_MAC(w0, w1, w2, u[6], m[28]); + MONT_MAC(t0, t1, t2, u[7], m[27]); + MONT_MAC(v0, v1, v2, u[8], m[26]); + MONT_MAC(w0, w1, w2, u[9], m[25]); + MONT_MAC(t0, t1, t2, u[10], m[24]); + MONT_MAC(v0, v1, v2, u[11], m[23]); + MONT_MAC(w0, w1, w2, u[12], m[22]); + MONT_MAC(t0, t1, t2, u[13], m[21]); + MONT_MAC(v0, v1, v2, u[14], m[20]); + MONT_MAC(w0, w1, w2, u[15], m[19]); + MONT_MAC(t0, t1, t2, u[16], m[18]); + MONT_MAC(v0, v1, v2, u[17], m[17]); + MONT_MAC(w0, w1, w2, u[18], m[16]); + MONT_MAC(t0, t1, t2, u[19], m[15]); + MONT_MAC(v0, v1, v2, u[20], m[14]); + MONT_MAC(w0, w1, w2, u[21], m[13]); + MONT_MAC(t0, t1, t2, u[22], m[12]); + MONT_MAC(v0, v1, v2, u[23], m[11]); + MONT_MAC(w0, w1, w2, u[24], m[10]); + MONT_MAC(t0, t1, t2, u[25], m[9]); + MONT_MAC(v0, v1, v2, u[26], m[8]); + MONT_MAC(w0, w1, w2, u[27], m[7]); + MONT_MAC(t0, t1, t2, u[28], m[6]); + MONT_MAC(v0, v1, v2, u[29], m[5]); + MONT_MAC(w0, w1, w2, u[30], m[4]); + MONT_MAC(t0, t1, t2, u[31], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 35 */ + MONT_MAC(t0, t1, t2, a[4], b[31]); + MONT_MUL(v0, v1, v2, a[5], b[30]); + MONT_MUL(w0, w1, w2, a[6], b[29]); + MONT_MAC(t0, t1, t2, a[7], b[28]); + MONT_MAC(v0, v1, v2, a[8], b[27]); + MONT_MAC(w0, w1, w2, a[9], b[26]); + MONT_MAC(t0, t1, t2, a[10], b[25]); + MONT_MAC(v0, v1, v2, a[11], b[24]); + MONT_MAC(w0, w1, w2, a[12], b[23]); + MONT_MAC(t0, t1, t2, a[13], b[22]); + MONT_MAC(v0, v1, v2, a[14], b[21]); + MONT_MAC(w0, w1, w2, a[15], b[20]); + MONT_MAC(t0, t1, t2, a[16], b[19]); + MONT_MAC(v0, v1, v2, a[17], b[18]); + MONT_MAC(w0, w1, w2, a[18], b[17]); + MONT_MAC(t0, t1, t2, a[19], b[16]); + MONT_MAC(v0, v1, v2, a[20], b[15]); + MONT_MAC(w0, w1, w2, a[21], b[14]); + MONT_MAC(t0, t1, t2, a[22], b[13]); + MONT_MAC(v0, v1, v2, a[23], b[12]); + MONT_MAC(w0, w1, w2, a[24], b[11]); + MONT_MAC(t0, t1, t2, a[25], b[10]); + MONT_MAC(v0, v1, v2, a[26], b[9]); + MONT_MAC(w0, w1, w2, a[27], b[8]); + MONT_MAC(t0, t1, t2, a[28], b[7]); + MONT_MAC(v0, v1, v2, a[29], b[6]); + MONT_MAC(w0, w1, w2, a[30], b[5]); + MONT_MAC(t0, t1, t2, a[31], b[4]); + MONT_MAC(v0, v1, v2, u[4], m[31]); + MONT_MAC(w0, w1, w2, u[5], m[30]); + MONT_MAC(t0, t1, t2, u[6], m[29]); + MONT_MAC(v0, v1, v2, u[7], m[28]); + MONT_MAC(w0, w1, w2, u[8], m[27]); + MONT_MAC(t0, t1, t2, u[9], m[26]); + MONT_MAC(v0, v1, v2, u[10], m[25]); + MONT_MAC(w0, w1, w2, u[11], m[24]); + MONT_MAC(t0, t1, t2, u[12], m[23]); + MONT_MAC(v0, v1, v2, u[13], m[22]); + MONT_MAC(w0, w1, w2, u[14], m[21]); + MONT_MAC(t0, t1, t2, u[15], m[20]); + MONT_MAC(v0, v1, v2, u[16], m[19]); + MONT_MAC(w0, w1, w2, u[17], m[18]); + MONT_MAC(t0, t1, t2, u[18], m[17]); + MONT_MAC(v0, v1, v2, u[19], m[16]); + MONT_MAC(w0, w1, w2, u[20], m[15]); + MONT_MAC(t0, t1, t2, u[21], m[14]); + MONT_MAC(v0, v1, v2, u[22], m[13]); + MONT_MAC(w0, w1, w2, u[23], m[12]); + MONT_MAC(t0, t1, t2, u[24], m[11]); + MONT_MAC(v0, v1, v2, u[25], m[10]); + MONT_MAC(w0, w1, w2, u[26], m[9]); + MONT_MAC(t0, t1, t2, u[27], m[8]); + MONT_MAC(v0, v1, v2, u[28], m[7]); + MONT_MAC(w0, w1, w2, u[29], m[6]); + MONT_MAC(t0, t1, t2, u[30], m[5]); + MONT_MAC(v0, v1, v2, u[31], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 36 */ + MONT_MAC(t0, t1, t2, a[5], b[31]); + MONT_MUL(v0, v1, v2, a[6], b[30]); + MONT_MUL(w0, w1, w2, a[7], b[29]); + MONT_MAC(t0, t1, t2, a[8], b[28]); + MONT_MAC(v0, v1, v2, a[9], b[27]); + MONT_MAC(w0, w1, w2, a[10], b[26]); + MONT_MAC(t0, t1, t2, a[11], b[25]); + MONT_MAC(v0, v1, v2, a[12], b[24]); + MONT_MAC(w0, w1, w2, a[13], b[23]); + MONT_MAC(t0, t1, t2, a[14], b[22]); + MONT_MAC(v0, v1, v2, a[15], b[21]); + MONT_MAC(w0, w1, w2, a[16], b[20]); + MONT_MAC(t0, t1, t2, a[17], b[19]); + MONT_MAC(v0, v1, v2, a[18], b[18]); + MONT_MAC(w0, w1, w2, a[19], b[17]); + MONT_MAC(t0, t1, t2, a[20], b[16]); + MONT_MAC(v0, v1, v2, a[21], b[15]); + MONT_MAC(w0, w1, w2, a[22], b[14]); + MONT_MAC(t0, t1, t2, a[23], b[13]); + MONT_MAC(v0, v1, v2, a[24], b[12]); + MONT_MAC(w0, w1, w2, a[25], b[11]); + MONT_MAC(t0, t1, t2, a[26], b[10]); + MONT_MAC(v0, v1, v2, a[27], b[9]); + MONT_MAC(w0, w1, w2, a[28], b[8]); + MONT_MAC(t0, t1, t2, a[29], b[7]); + MONT_MAC(v0, v1, v2, a[30], b[6]); + MONT_MAC(w0, w1, w2, a[31], b[5]); + MONT_MAC(t0, t1, t2, u[5], m[31]); + MONT_MAC(v0, v1, v2, u[6], m[30]); + MONT_MAC(w0, w1, w2, u[7], m[29]); + MONT_MAC(t0, t1, t2, u[8], m[28]); + MONT_MAC(v0, v1, v2, u[9], m[27]); + MONT_MAC(w0, w1, w2, u[10], m[26]); + MONT_MAC(t0, t1, t2, u[11], m[25]); + MONT_MAC(v0, v1, v2, u[12], m[24]); + MONT_MAC(w0, w1, w2, u[13], m[23]); + MONT_MAC(t0, t1, t2, u[14], m[22]); + MONT_MAC(v0, v1, v2, u[15], m[21]); + MONT_MAC(w0, w1, w2, u[16], m[20]); + MONT_MAC(t0, t1, t2, u[17], m[19]); + MONT_MAC(v0, v1, v2, u[18], m[18]); + MONT_MAC(w0, w1, w2, u[19], m[17]); + MONT_MAC(t0, t1, t2, u[20], m[16]); + MONT_MAC(v0, v1, v2, u[21], m[15]); + MONT_MAC(w0, w1, w2, u[22], m[14]); + MONT_MAC(t0, t1, t2, u[23], m[13]); + MONT_MAC(v0, v1, v2, u[24], m[12]); + MONT_MAC(w0, w1, w2, u[25], m[11]); + MONT_MAC(t0, t1, t2, u[26], m[10]); + MONT_MAC(v0, v1, v2, u[27], m[9]); + MONT_MAC(w0, w1, w2, u[28], m[8]); + MONT_MAC(t0, t1, t2, u[29], m[7]); + MONT_MAC(v0, v1, v2, u[30], m[6]); + MONT_MAC(w0, w1, w2, u[31], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 37 */ + MONT_MAC(t0, t1, t2, a[6], b[31]); + MONT_MUL(v0, v1, v2, a[7], b[30]); + MONT_MUL(w0, w1, w2, a[8], b[29]); + MONT_MAC(t0, t1, t2, a[9], b[28]); + MONT_MAC(v0, v1, v2, a[10], b[27]); + MONT_MAC(w0, w1, w2, a[11], b[26]); + MONT_MAC(t0, t1, t2, a[12], b[25]); + MONT_MAC(v0, v1, v2, a[13], b[24]); + MONT_MAC(w0, w1, w2, a[14], b[23]); + MONT_MAC(t0, t1, t2, a[15], b[22]); + MONT_MAC(v0, v1, v2, a[16], b[21]); + MONT_MAC(w0, w1, w2, a[17], b[20]); + MONT_MAC(t0, t1, t2, a[18], b[19]); + MONT_MAC(v0, v1, v2, a[19], b[18]); + MONT_MAC(w0, w1, w2, a[20], b[17]); + MONT_MAC(t0, t1, t2, a[21], b[16]); + MONT_MAC(v0, v1, v2, a[22], b[15]); + MONT_MAC(w0, w1, w2, a[23], b[14]); + MONT_MAC(t0, t1, t2, a[24], b[13]); + MONT_MAC(v0, v1, v2, a[25], b[12]); + MONT_MAC(w0, w1, w2, a[26], b[11]); + MONT_MAC(t0, t1, t2, a[27], b[10]); + MONT_MAC(v0, v1, v2, a[28], b[9]); + MONT_MAC(w0, w1, w2, a[29], b[8]); + MONT_MAC(t0, t1, t2, a[30], b[7]); + MONT_MAC(v0, v1, v2, a[31], b[6]); + MONT_MAC(w0, w1, w2, u[6], m[31]); + MONT_MAC(t0, t1, t2, u[7], m[30]); + MONT_MAC(v0, v1, v2, u[8], m[29]); + MONT_MAC(w0, w1, w2, u[9], m[28]); + MONT_MAC(t0, t1, t2, u[10], m[27]); + MONT_MAC(v0, v1, v2, u[11], m[26]); + MONT_MAC(w0, w1, w2, u[12], m[25]); + MONT_MAC(t0, t1, t2, u[13], m[24]); + MONT_MAC(v0, v1, v2, u[14], m[23]); + MONT_MAC(w0, w1, w2, u[15], m[22]); + MONT_MAC(t0, t1, t2, u[16], m[21]); + MONT_MAC(v0, v1, v2, u[17], m[20]); + MONT_MAC(w0, w1, w2, u[18], m[19]); + MONT_MAC(t0, t1, t2, u[19], m[18]); + MONT_MAC(v0, v1, v2, u[20], m[17]); + MONT_MAC(w0, w1, w2, u[21], m[16]); + MONT_MAC(t0, t1, t2, u[22], m[15]); + MONT_MAC(v0, v1, v2, u[23], m[14]); + MONT_MAC(w0, w1, w2, u[24], m[13]); + MONT_MAC(t0, t1, t2, u[25], m[12]); + MONT_MAC(v0, v1, v2, u[26], m[11]); + MONT_MAC(w0, w1, w2, u[27], m[10]); + MONT_MAC(t0, t1, t2, u[28], m[9]); + MONT_MAC(v0, v1, v2, u[29], m[8]); + MONT_MAC(w0, w1, w2, u[30], m[7]); + MONT_MAC(t0, t1, t2, u[31], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 38 */ + MONT_MAC(t0, t1, t2, a[7], b[31]); + MONT_MUL(v0, v1, v2, a[8], b[30]); + MONT_MUL(w0, w1, w2, a[9], b[29]); + MONT_MAC(t0, t1, t2, a[10], b[28]); + MONT_MAC(v0, v1, v2, a[11], b[27]); + MONT_MAC(w0, w1, w2, a[12], b[26]); + MONT_MAC(t0, t1, t2, a[13], b[25]); + MONT_MAC(v0, v1, v2, a[14], b[24]); + MONT_MAC(w0, w1, w2, a[15], b[23]); + MONT_MAC(t0, t1, t2, a[16], b[22]); + MONT_MAC(v0, v1, v2, a[17], b[21]); + MONT_MAC(w0, w1, w2, a[18], b[20]); + MONT_MAC(t0, t1, t2, a[19], b[19]); + MONT_MAC(v0, v1, v2, a[20], b[18]); + MONT_MAC(w0, w1, w2, a[21], b[17]); + MONT_MAC(t0, t1, t2, a[22], b[16]); + MONT_MAC(v0, v1, v2, a[23], b[15]); + MONT_MAC(w0, w1, w2, a[24], b[14]); + MONT_MAC(t0, t1, t2, a[25], b[13]); + MONT_MAC(v0, v1, v2, a[26], b[12]); + MONT_MAC(w0, w1, w2, a[27], b[11]); + MONT_MAC(t0, t1, t2, a[28], b[10]); + MONT_MAC(v0, v1, v2, a[29], b[9]); + MONT_MAC(w0, w1, w2, a[30], b[8]); + MONT_MAC(t0, t1, t2, a[31], b[7]); + MONT_MAC(v0, v1, v2, u[7], m[31]); + MONT_MAC(w0, w1, w2, u[8], m[30]); + MONT_MAC(t0, t1, t2, u[9], m[29]); + MONT_MAC(v0, v1, v2, u[10], m[28]); + MONT_MAC(w0, w1, w2, u[11], m[27]); + MONT_MAC(t0, t1, t2, u[12], m[26]); + MONT_MAC(v0, v1, v2, u[13], m[25]); + MONT_MAC(w0, w1, w2, u[14], m[24]); + MONT_MAC(t0, t1, t2, u[15], m[23]); + MONT_MAC(v0, v1, v2, u[16], m[22]); + MONT_MAC(w0, w1, w2, u[17], m[21]); + MONT_MAC(t0, t1, t2, u[18], m[20]); + MONT_MAC(v0, v1, v2, u[19], m[19]); + MONT_MAC(w0, w1, w2, u[20], m[18]); + MONT_MAC(t0, t1, t2, u[21], m[17]); + MONT_MAC(v0, v1, v2, u[22], m[16]); + MONT_MAC(w0, w1, w2, u[23], m[15]); + MONT_MAC(t0, t1, t2, u[24], m[14]); + MONT_MAC(v0, v1, v2, u[25], m[13]); + MONT_MAC(w0, w1, w2, u[26], m[12]); + MONT_MAC(t0, t1, t2, u[27], m[11]); + MONT_MAC(v0, v1, v2, u[28], m[10]); + MONT_MAC(w0, w1, w2, u[29], m[9]); + MONT_MAC(t0, t1, t2, u[30], m[8]); + MONT_MAC(v0, v1, v2, u[31], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 39 */ + MONT_MAC(t0, t1, t2, a[8], b[31]); + MONT_MUL(v0, v1, v2, a[9], b[30]); + MONT_MUL(w0, w1, w2, a[10], b[29]); + MONT_MAC(t0, t1, t2, a[11], b[28]); + MONT_MAC(v0, v1, v2, a[12], b[27]); + MONT_MAC(w0, w1, w2, a[13], b[26]); + MONT_MAC(t0, t1, t2, a[14], b[25]); + MONT_MAC(v0, v1, v2, a[15], b[24]); + MONT_MAC(w0, w1, w2, a[16], b[23]); + MONT_MAC(t0, t1, t2, a[17], b[22]); + MONT_MAC(v0, v1, v2, a[18], b[21]); + MONT_MAC(w0, w1, w2, a[19], b[20]); + MONT_MAC(t0, t1, t2, a[20], b[19]); + MONT_MAC(v0, v1, v2, a[21], b[18]); + MONT_MAC(w0, w1, w2, a[22], b[17]); + MONT_MAC(t0, t1, t2, a[23], b[16]); + MONT_MAC(v0, v1, v2, a[24], b[15]); + MONT_MAC(w0, w1, w2, a[25], b[14]); + MONT_MAC(t0, t1, t2, a[26], b[13]); + MONT_MAC(v0, v1, v2, a[27], b[12]); + MONT_MAC(w0, w1, w2, a[28], b[11]); + MONT_MAC(t0, t1, t2, a[29], b[10]); + MONT_MAC(v0, v1, v2, a[30], b[9]); + MONT_MAC(w0, w1, w2, a[31], b[8]); + MONT_MAC(t0, t1, t2, u[8], m[31]); + MONT_MAC(v0, v1, v2, u[9], m[30]); + MONT_MAC(w0, w1, w2, u[10], m[29]); + MONT_MAC(t0, t1, t2, u[11], m[28]); + MONT_MAC(v0, v1, v2, u[12], m[27]); + MONT_MAC(w0, w1, w2, u[13], m[26]); + MONT_MAC(t0, t1, t2, u[14], m[25]); + MONT_MAC(v0, v1, v2, u[15], m[24]); + MONT_MAC(w0, w1, w2, u[16], m[23]); + MONT_MAC(t0, t1, t2, u[17], m[22]); + MONT_MAC(v0, v1, v2, u[18], m[21]); + MONT_MAC(w0, w1, w2, u[19], m[20]); + MONT_MAC(t0, t1, t2, u[20], m[19]); + MONT_MAC(v0, v1, v2, u[21], m[18]); + MONT_MAC(w0, w1, w2, u[22], m[17]); + MONT_MAC(t0, t1, t2, u[23], m[16]); + MONT_MAC(v0, v1, v2, u[24], m[15]); + MONT_MAC(w0, w1, w2, u[25], m[14]); + MONT_MAC(t0, t1, t2, u[26], m[13]); + MONT_MAC(v0, v1, v2, u[27], m[12]); + MONT_MAC(w0, w1, w2, u[28], m[11]); + MONT_MAC(t0, t1, t2, u[29], m[10]); + MONT_MAC(v0, v1, v2, u[30], m[9]); + MONT_MAC(w0, w1, w2, u[31], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 40 */ + MONT_MAC(t0, t1, t2, a[9], b[31]); + MONT_MUL(v0, v1, v2, a[10], b[30]); + MONT_MUL(w0, w1, w2, a[11], b[29]); + MONT_MAC(t0, t1, t2, a[12], b[28]); + MONT_MAC(v0, v1, v2, a[13], b[27]); + MONT_MAC(w0, w1, w2, a[14], b[26]); + MONT_MAC(t0, t1, t2, a[15], b[25]); + MONT_MAC(v0, v1, v2, a[16], b[24]); + MONT_MAC(w0, w1, w2, a[17], b[23]); + MONT_MAC(t0, t1, t2, a[18], b[22]); + MONT_MAC(v0, v1, v2, a[19], b[21]); + MONT_MAC(w0, w1, w2, a[20], b[20]); + MONT_MAC(t0, t1, t2, a[21], b[19]); + MONT_MAC(v0, v1, v2, a[22], b[18]); + MONT_MAC(w0, w1, w2, a[23], b[17]); + MONT_MAC(t0, t1, t2, a[24], b[16]); + MONT_MAC(v0, v1, v2, a[25], b[15]); + MONT_MAC(w0, w1, w2, a[26], b[14]); + MONT_MAC(t0, t1, t2, a[27], b[13]); + MONT_MAC(v0, v1, v2, a[28], b[12]); + MONT_MAC(w0, w1, w2, a[29], b[11]); + MONT_MAC(t0, t1, t2, a[30], b[10]); + MONT_MAC(v0, v1, v2, a[31], b[9]); + MONT_MAC(w0, w1, w2, u[9], m[31]); + MONT_MAC(t0, t1, t2, u[10], m[30]); + MONT_MAC(v0, v1, v2, u[11], m[29]); + MONT_MAC(w0, w1, w2, u[12], m[28]); + MONT_MAC(t0, t1, t2, u[13], m[27]); + MONT_MAC(v0, v1, v2, u[14], m[26]); + MONT_MAC(w0, w1, w2, u[15], m[25]); + MONT_MAC(t0, t1, t2, u[16], m[24]); + MONT_MAC(v0, v1, v2, u[17], m[23]); + MONT_MAC(w0, w1, w2, u[18], m[22]); + MONT_MAC(t0, t1, t2, u[19], m[21]); + MONT_MAC(v0, v1, v2, u[20], m[20]); + MONT_MAC(w0, w1, w2, u[21], m[19]); + MONT_MAC(t0, t1, t2, u[22], m[18]); + MONT_MAC(v0, v1, v2, u[23], m[17]); + MONT_MAC(w0, w1, w2, u[24], m[16]); + MONT_MAC(t0, t1, t2, u[25], m[15]); + MONT_MAC(v0, v1, v2, u[26], m[14]); + MONT_MAC(w0, w1, w2, u[27], m[13]); + MONT_MAC(t0, t1, t2, u[28], m[12]); + MONT_MAC(v0, v1, v2, u[29], m[11]); + MONT_MAC(w0, w1, w2, u[30], m[10]); + MONT_MAC(t0, t1, t2, u[31], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 41 */ + MONT_MAC(t0, t1, t2, a[10], b[31]); + MONT_MUL(v0, v1, v2, a[11], b[30]); + MONT_MUL(w0, w1, w2, a[12], b[29]); + MONT_MAC(t0, t1, t2, a[13], b[28]); + MONT_MAC(v0, v1, v2, a[14], b[27]); + MONT_MAC(w0, w1, w2, a[15], b[26]); + MONT_MAC(t0, t1, t2, a[16], b[25]); + MONT_MAC(v0, v1, v2, a[17], b[24]); + MONT_MAC(w0, w1, w2, a[18], b[23]); + MONT_MAC(t0, t1, t2, a[19], b[22]); + MONT_MAC(v0, v1, v2, a[20], b[21]); + MONT_MAC(w0, w1, w2, a[21], b[20]); + MONT_MAC(t0, t1, t2, a[22], b[19]); + MONT_MAC(v0, v1, v2, a[23], b[18]); + MONT_MAC(w0, w1, w2, a[24], b[17]); + MONT_MAC(t0, t1, t2, a[25], b[16]); + MONT_MAC(v0, v1, v2, a[26], b[15]); + MONT_MAC(w0, w1, w2, a[27], b[14]); + MONT_MAC(t0, t1, t2, a[28], b[13]); + MONT_MAC(v0, v1, v2, a[29], b[12]); + MONT_MAC(w0, w1, w2, a[30], b[11]); + MONT_MAC(t0, t1, t2, a[31], b[10]); + MONT_MAC(v0, v1, v2, u[10], m[31]); + MONT_MAC(w0, w1, w2, u[11], m[30]); + MONT_MAC(t0, t1, t2, u[12], m[29]); + MONT_MAC(v0, v1, v2, u[13], m[28]); + MONT_MAC(w0, w1, w2, u[14], m[27]); + MONT_MAC(t0, t1, t2, u[15], m[26]); + MONT_MAC(v0, v1, v2, u[16], m[25]); + MONT_MAC(w0, w1, w2, u[17], m[24]); + MONT_MAC(t0, t1, t2, u[18], m[23]); + MONT_MAC(v0, v1, v2, u[19], m[22]); + MONT_MAC(w0, w1, w2, u[20], m[21]); + MONT_MAC(t0, t1, t2, u[21], m[20]); + MONT_MAC(v0, v1, v2, u[22], m[19]); + MONT_MAC(w0, w1, w2, u[23], m[18]); + MONT_MAC(t0, t1, t2, u[24], m[17]); + MONT_MAC(v0, v1, v2, u[25], m[16]); + MONT_MAC(w0, w1, w2, u[26], m[15]); + MONT_MAC(t0, t1, t2, u[27], m[14]); + MONT_MAC(v0, v1, v2, u[28], m[13]); + MONT_MAC(w0, w1, w2, u[29], m[12]); + MONT_MAC(t0, t1, t2, u[30], m[11]); + MONT_MAC(v0, v1, v2, u[31], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 42 */ + MONT_MAC(t0, t1, t2, a[11], b[31]); + MONT_MUL(v0, v1, v2, a[12], b[30]); + MONT_MUL(w0, w1, w2, a[13], b[29]); + MONT_MAC(t0, t1, t2, a[14], b[28]); + MONT_MAC(v0, v1, v2, a[15], b[27]); + MONT_MAC(w0, w1, w2, a[16], b[26]); + MONT_MAC(t0, t1, t2, a[17], b[25]); + MONT_MAC(v0, v1, v2, a[18], b[24]); + MONT_MAC(w0, w1, w2, a[19], b[23]); + MONT_MAC(t0, t1, t2, a[20], b[22]); + MONT_MAC(v0, v1, v2, a[21], b[21]); + MONT_MAC(w0, w1, w2, a[22], b[20]); + MONT_MAC(t0, t1, t2, a[23], b[19]); + MONT_MAC(v0, v1, v2, a[24], b[18]); + MONT_MAC(w0, w1, w2, a[25], b[17]); + MONT_MAC(t0, t1, t2, a[26], b[16]); + MONT_MAC(v0, v1, v2, a[27], b[15]); + MONT_MAC(w0, w1, w2, a[28], b[14]); + MONT_MAC(t0, t1, t2, a[29], b[13]); + MONT_MAC(v0, v1, v2, a[30], b[12]); + MONT_MAC(w0, w1, w2, a[31], b[11]); + MONT_MAC(t0, t1, t2, u[11], m[31]); + MONT_MAC(v0, v1, v2, u[12], m[30]); + MONT_MAC(w0, w1, w2, u[13], m[29]); + MONT_MAC(t0, t1, t2, u[14], m[28]); + MONT_MAC(v0, v1, v2, u[15], m[27]); + MONT_MAC(w0, w1, w2, u[16], m[26]); + MONT_MAC(t0, t1, t2, u[17], m[25]); + MONT_MAC(v0, v1, v2, u[18], m[24]); + MONT_MAC(w0, w1, w2, u[19], m[23]); + MONT_MAC(t0, t1, t2, u[20], m[22]); + MONT_MAC(v0, v1, v2, u[21], m[21]); + MONT_MAC(w0, w1, w2, u[22], m[20]); + MONT_MAC(t0, t1, t2, u[23], m[19]); + MONT_MAC(v0, v1, v2, u[24], m[18]); + MONT_MAC(w0, w1, w2, u[25], m[17]); + MONT_MAC(t0, t1, t2, u[26], m[16]); + MONT_MAC(v0, v1, v2, u[27], m[15]); + MONT_MAC(w0, w1, w2, u[28], m[14]); + MONT_MAC(t0, t1, t2, u[29], m[13]); + MONT_MAC(v0, v1, v2, u[30], m[12]); + MONT_MAC(w0, w1, w2, u[31], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 43 */ + MONT_MAC(t0, t1, t2, a[12], b[31]); + MONT_MUL(v0, v1, v2, a[13], b[30]); + MONT_MUL(w0, w1, w2, a[14], b[29]); + MONT_MAC(t0, t1, t2, a[15], b[28]); + MONT_MAC(v0, v1, v2, a[16], b[27]); + MONT_MAC(w0, w1, w2, a[17], b[26]); + MONT_MAC(t0, t1, t2, a[18], b[25]); + MONT_MAC(v0, v1, v2, a[19], b[24]); + MONT_MAC(w0, w1, w2, a[20], b[23]); + MONT_MAC(t0, t1, t2, a[21], b[22]); + MONT_MAC(v0, v1, v2, a[22], b[21]); + MONT_MAC(w0, w1, w2, a[23], b[20]); + MONT_MAC(t0, t1, t2, a[24], b[19]); + MONT_MAC(v0, v1, v2, a[25], b[18]); + MONT_MAC(w0, w1, w2, a[26], b[17]); + MONT_MAC(t0, t1, t2, a[27], b[16]); + MONT_MAC(v0, v1, v2, a[28], b[15]); + MONT_MAC(w0, w1, w2, a[29], b[14]); + MONT_MAC(t0, t1, t2, a[30], b[13]); + MONT_MAC(v0, v1, v2, a[31], b[12]); + MONT_MAC(w0, w1, w2, u[12], m[31]); + MONT_MAC(t0, t1, t2, u[13], m[30]); + MONT_MAC(v0, v1, v2, u[14], m[29]); + MONT_MAC(w0, w1, w2, u[15], m[28]); + MONT_MAC(t0, t1, t2, u[16], m[27]); + MONT_MAC(v0, v1, v2, u[17], m[26]); + MONT_MAC(w0, w1, w2, u[18], m[25]); + MONT_MAC(t0, t1, t2, u[19], m[24]); + MONT_MAC(v0, v1, v2, u[20], m[23]); + MONT_MAC(w0, w1, w2, u[21], m[22]); + MONT_MAC(t0, t1, t2, u[22], m[21]); + MONT_MAC(v0, v1, v2, u[23], m[20]); + MONT_MAC(w0, w1, w2, u[24], m[19]); + MONT_MAC(t0, t1, t2, u[25], m[18]); + MONT_MAC(v0, v1, v2, u[26], m[17]); + MONT_MAC(w0, w1, w2, u[27], m[16]); + MONT_MAC(t0, t1, t2, u[28], m[15]); + MONT_MAC(v0, v1, v2, u[29], m[14]); + MONT_MAC(w0, w1, w2, u[30], m[13]); + MONT_MAC(t0, t1, t2, u[31], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 44 */ + MONT_MAC(t0, t1, t2, a[13], b[31]); + MONT_MUL(v0, v1, v2, a[14], b[30]); + MONT_MUL(w0, w1, w2, a[15], b[29]); + MONT_MAC(t0, t1, t2, a[16], b[28]); + MONT_MAC(v0, v1, v2, a[17], b[27]); + MONT_MAC(w0, w1, w2, a[18], b[26]); + MONT_MAC(t0, t1, t2, a[19], b[25]); + MONT_MAC(v0, v1, v2, a[20], b[24]); + MONT_MAC(w0, w1, w2, a[21], b[23]); + MONT_MAC(t0, t1, t2, a[22], b[22]); + MONT_MAC(v0, v1, v2, a[23], b[21]); + MONT_MAC(w0, w1, w2, a[24], b[20]); + MONT_MAC(t0, t1, t2, a[25], b[19]); + MONT_MAC(v0, v1, v2, a[26], b[18]); + MONT_MAC(w0, w1, w2, a[27], b[17]); + MONT_MAC(t0, t1, t2, a[28], b[16]); + MONT_MAC(v0, v1, v2, a[29], b[15]); + MONT_MAC(w0, w1, w2, a[30], b[14]); + MONT_MAC(t0, t1, t2, a[31], b[13]); + MONT_MAC(v0, v1, v2, u[13], m[31]); + MONT_MAC(w0, w1, w2, u[14], m[30]); + MONT_MAC(t0, t1, t2, u[15], m[29]); + MONT_MAC(v0, v1, v2, u[16], m[28]); + MONT_MAC(w0, w1, w2, u[17], m[27]); + MONT_MAC(t0, t1, t2, u[18], m[26]); + MONT_MAC(v0, v1, v2, u[19], m[25]); + MONT_MAC(w0, w1, w2, u[20], m[24]); + MONT_MAC(t0, t1, t2, u[21], m[23]); + MONT_MAC(v0, v1, v2, u[22], m[22]); + MONT_MAC(w0, w1, w2, u[23], m[21]); + MONT_MAC(t0, t1, t2, u[24], m[20]); + MONT_MAC(v0, v1, v2, u[25], m[19]); + MONT_MAC(w0, w1, w2, u[26], m[18]); + MONT_MAC(t0, t1, t2, u[27], m[17]); + MONT_MAC(v0, v1, v2, u[28], m[16]); + MONT_MAC(w0, w1, w2, u[29], m[15]); + MONT_MAC(t0, t1, t2, u[30], m[14]); + MONT_MAC(v0, v1, v2, u[31], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 45 */ + MONT_MAC(t0, t1, t2, a[14], b[31]); + MONT_MUL(v0, v1, v2, a[15], b[30]); + MONT_MUL(w0, w1, w2, a[16], b[29]); + MONT_MAC(t0, t1, t2, a[17], b[28]); + MONT_MAC(v0, v1, v2, a[18], b[27]); + MONT_MAC(w0, w1, w2, a[19], b[26]); + MONT_MAC(t0, t1, t2, a[20], b[25]); + MONT_MAC(v0, v1, v2, a[21], b[24]); + MONT_MAC(w0, w1, w2, a[22], b[23]); + MONT_MAC(t0, t1, t2, a[23], b[22]); + MONT_MAC(v0, v1, v2, a[24], b[21]); + MONT_MAC(w0, w1, w2, a[25], b[20]); + MONT_MAC(t0, t1, t2, a[26], b[19]); + MONT_MAC(v0, v1, v2, a[27], b[18]); + MONT_MAC(w0, w1, w2, a[28], b[17]); + MONT_MAC(t0, t1, t2, a[29], b[16]); + MONT_MAC(v0, v1, v2, a[30], b[15]); + MONT_MAC(w0, w1, w2, a[31], b[14]); + MONT_MAC(t0, t1, t2, u[14], m[31]); + MONT_MAC(v0, v1, v2, u[15], m[30]); + MONT_MAC(w0, w1, w2, u[16], m[29]); + MONT_MAC(t0, t1, t2, u[17], m[28]); + MONT_MAC(v0, v1, v2, u[18], m[27]); + MONT_MAC(w0, w1, w2, u[19], m[26]); + MONT_MAC(t0, t1, t2, u[20], m[25]); + MONT_MAC(v0, v1, v2, u[21], m[24]); + MONT_MAC(w0, w1, w2, u[22], m[23]); + MONT_MAC(t0, t1, t2, u[23], m[22]); + MONT_MAC(v0, v1, v2, u[24], m[21]); + MONT_MAC(w0, w1, w2, u[25], m[20]); + MONT_MAC(t0, t1, t2, u[26], m[19]); + MONT_MAC(v0, v1, v2, u[27], m[18]); + MONT_MAC(w0, w1, w2, u[28], m[17]); + MONT_MAC(t0, t1, t2, u[29], m[16]); + MONT_MAC(v0, v1, v2, u[30], m[15]); + MONT_MAC(w0, w1, w2, u[31], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 46 */ + MONT_MAC(t0, t1, t2, a[15], b[31]); + MONT_MUL(v0, v1, v2, a[16], b[30]); + MONT_MUL(w0, w1, w2, a[17], b[29]); + MONT_MAC(t0, t1, t2, a[18], b[28]); + MONT_MAC(v0, v1, v2, a[19], b[27]); + MONT_MAC(w0, w1, w2, a[20], b[26]); + MONT_MAC(t0, t1, t2, a[21], b[25]); + MONT_MAC(v0, v1, v2, a[22], b[24]); + MONT_MAC(w0, w1, w2, a[23], b[23]); + MONT_MAC(t0, t1, t2, a[24], b[22]); + MONT_MAC(v0, v1, v2, a[25], b[21]); + MONT_MAC(w0, w1, w2, a[26], b[20]); + MONT_MAC(t0, t1, t2, a[27], b[19]); + MONT_MAC(v0, v1, v2, a[28], b[18]); + MONT_MAC(w0, w1, w2, a[29], b[17]); + MONT_MAC(t0, t1, t2, a[30], b[16]); + MONT_MAC(v0, v1, v2, a[31], b[15]); + MONT_MAC(w0, w1, w2, u[15], m[31]); + MONT_MAC(t0, t1, t2, u[16], m[30]); + MONT_MAC(v0, v1, v2, u[17], m[29]); + MONT_MAC(w0, w1, w2, u[18], m[28]); + MONT_MAC(t0, t1, t2, u[19], m[27]); + MONT_MAC(v0, v1, v2, u[20], m[26]); + MONT_MAC(w0, w1, w2, u[21], m[25]); + MONT_MAC(t0, t1, t2, u[22], m[24]); + MONT_MAC(v0, v1, v2, u[23], m[23]); + MONT_MAC(w0, w1, w2, u[24], m[22]); + MONT_MAC(t0, t1, t2, u[25], m[21]); + MONT_MAC(v0, v1, v2, u[26], m[20]); + MONT_MAC(w0, w1, w2, u[27], m[19]); + MONT_MAC(t0, t1, t2, u[28], m[18]); + MONT_MAC(v0, v1, v2, u[29], m[17]); + MONT_MAC(w0, w1, w2, u[30], m[16]); + MONT_MAC(t0, t1, t2, u[31], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 47 */ + MONT_MAC(t0, t1, t2, a[16], b[31]); + MONT_MUL(v0, v1, v2, a[17], b[30]); + MONT_MUL(w0, w1, w2, a[18], b[29]); + MONT_MAC(t0, t1, t2, a[19], b[28]); + MONT_MAC(v0, v1, v2, a[20], b[27]); + MONT_MAC(w0, w1, w2, a[21], b[26]); + MONT_MAC(t0, t1, t2, a[22], b[25]); + MONT_MAC(v0, v1, v2, a[23], b[24]); + MONT_MAC(w0, w1, w2, a[24], b[23]); + MONT_MAC(t0, t1, t2, a[25], b[22]); + MONT_MAC(v0, v1, v2, a[26], b[21]); + MONT_MAC(w0, w1, w2, a[27], b[20]); + MONT_MAC(t0, t1, t2, a[28], b[19]); + MONT_MAC(v0, v1, v2, a[29], b[18]); + MONT_MAC(w0, w1, w2, a[30], b[17]); + MONT_MAC(t0, t1, t2, a[31], b[16]); + MONT_MAC(v0, v1, v2, u[16], m[31]); + MONT_MAC(w0, w1, w2, u[17], m[30]); + MONT_MAC(t0, t1, t2, u[18], m[29]); + MONT_MAC(v0, v1, v2, u[19], m[28]); + MONT_MAC(w0, w1, w2, u[20], m[27]); + MONT_MAC(t0, t1, t2, u[21], m[26]); + MONT_MAC(v0, v1, v2, u[22], m[25]); + MONT_MAC(w0, w1, w2, u[23], m[24]); + MONT_MAC(t0, t1, t2, u[24], m[23]); + MONT_MAC(v0, v1, v2, u[25], m[22]); + MONT_MAC(w0, w1, w2, u[26], m[21]); + MONT_MAC(t0, t1, t2, u[27], m[20]); + MONT_MAC(v0, v1, v2, u[28], m[19]); + MONT_MAC(w0, w1, w2, u[29], m[18]); + MONT_MAC(t0, t1, t2, u[30], m[17]); + MONT_MAC(v0, v1, v2, u[31], m[16]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[15] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 48 */ + MONT_MAC(t0, t1, t2, a[17], b[31]); + MONT_MUL(v0, v1, v2, a[18], b[30]); + MONT_MUL(w0, w1, w2, a[19], b[29]); + MONT_MAC(t0, t1, t2, a[20], b[28]); + MONT_MAC(v0, v1, v2, a[21], b[27]); + MONT_MAC(w0, w1, w2, a[22], b[26]); + MONT_MAC(t0, t1, t2, a[23], b[25]); + MONT_MAC(v0, v1, v2, a[24], b[24]); + MONT_MAC(w0, w1, w2, a[25], b[23]); + MONT_MAC(t0, t1, t2, a[26], b[22]); + MONT_MAC(v0, v1, v2, a[27], b[21]); + MONT_MAC(w0, w1, w2, a[28], b[20]); + MONT_MAC(t0, t1, t2, a[29], b[19]); + MONT_MAC(v0, v1, v2, a[30], b[18]); + MONT_MAC(w0, w1, w2, a[31], b[17]); + MONT_MAC(t0, t1, t2, u[17], m[31]); + MONT_MAC(v0, v1, v2, u[18], m[30]); + MONT_MAC(w0, w1, w2, u[19], m[29]); + MONT_MAC(t0, t1, t2, u[20], m[28]); + MONT_MAC(v0, v1, v2, u[21], m[27]); + MONT_MAC(w0, w1, w2, u[22], m[26]); + MONT_MAC(t0, t1, t2, u[23], m[25]); + MONT_MAC(v0, v1, v2, u[24], m[24]); + MONT_MAC(w0, w1, w2, u[25], m[23]); + MONT_MAC(t0, t1, t2, u[26], m[22]); + MONT_MAC(v0, v1, v2, u[27], m[21]); + MONT_MAC(w0, w1, w2, u[28], m[20]); + MONT_MAC(t0, t1, t2, u[29], m[19]); + MONT_MAC(v0, v1, v2, u[30], m[18]); + MONT_MAC(w0, w1, w2, u[31], m[17]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[16] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 49 */ + MONT_MAC(t0, t1, t2, a[18], b[31]); + MONT_MUL(v0, v1, v2, a[19], b[30]); + MONT_MUL(w0, w1, w2, a[20], b[29]); + MONT_MAC(t0, t1, t2, a[21], b[28]); + MONT_MAC(v0, v1, v2, a[22], b[27]); + MONT_MAC(w0, w1, w2, a[23], b[26]); + MONT_MAC(t0, t1, t2, a[24], b[25]); + MONT_MAC(v0, v1, v2, a[25], b[24]); + MONT_MAC(w0, w1, w2, a[26], b[23]); + MONT_MAC(t0, t1, t2, a[27], b[22]); + MONT_MAC(v0, v1, v2, a[28], b[21]); + MONT_MAC(w0, w1, w2, a[29], b[20]); + MONT_MAC(t0, t1, t2, a[30], b[19]); + MONT_MAC(v0, v1, v2, a[31], b[18]); + MONT_MAC(w0, w1, w2, u[18], m[31]); + MONT_MAC(t0, t1, t2, u[19], m[30]); + MONT_MAC(v0, v1, v2, u[20], m[29]); + MONT_MAC(w0, w1, w2, u[21], m[28]); + MONT_MAC(t0, t1, t2, u[22], m[27]); + MONT_MAC(v0, v1, v2, u[23], m[26]); + MONT_MAC(w0, w1, w2, u[24], m[25]); + MONT_MAC(t0, t1, t2, u[25], m[24]); + MONT_MAC(v0, v1, v2, u[26], m[23]); + MONT_MAC(w0, w1, w2, u[27], m[22]); + MONT_MAC(t0, t1, t2, u[28], m[21]); + MONT_MAC(v0, v1, v2, u[29], m[20]); + MONT_MAC(w0, w1, w2, u[30], m[19]); + MONT_MAC(t0, t1, t2, u[31], m[18]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[17] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 50 */ + MONT_MAC(t0, t1, t2, a[19], b[31]); + MONT_MUL(v0, v1, v2, a[20], b[30]); + MONT_MUL(w0, w1, w2, a[21], b[29]); + MONT_MAC(t0, t1, t2, a[22], b[28]); + MONT_MAC(v0, v1, v2, a[23], b[27]); + MONT_MAC(w0, w1, w2, a[24], b[26]); + MONT_MAC(t0, t1, t2, a[25], b[25]); + MONT_MAC(v0, v1, v2, a[26], b[24]); + MONT_MAC(w0, w1, w2, a[27], b[23]); + MONT_MAC(t0, t1, t2, a[28], b[22]); + MONT_MAC(v0, v1, v2, a[29], b[21]); + MONT_MAC(w0, w1, w2, a[30], b[20]); + MONT_MAC(t0, t1, t2, a[31], b[19]); + MONT_MAC(v0, v1, v2, u[19], m[31]); + MONT_MAC(w0, w1, w2, u[20], m[30]); + MONT_MAC(t0, t1, t2, u[21], m[29]); + MONT_MAC(v0, v1, v2, u[22], m[28]); + MONT_MAC(w0, w1, w2, u[23], m[27]); + MONT_MAC(t0, t1, t2, u[24], m[26]); + MONT_MAC(v0, v1, v2, u[25], m[25]); + MONT_MAC(w0, w1, w2, u[26], m[24]); + MONT_MAC(t0, t1, t2, u[27], m[23]); + MONT_MAC(v0, v1, v2, u[28], m[22]); + MONT_MAC(w0, w1, w2, u[29], m[21]); + MONT_MAC(t0, t1, t2, u[30], m[20]); + MONT_MAC(v0, v1, v2, u[31], m[19]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[18] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 51 */ + MONT_MAC(t0, t1, t2, a[20], b[31]); + MONT_MUL(v0, v1, v2, a[21], b[30]); + MONT_MUL(w0, w1, w2, a[22], b[29]); + MONT_MAC(t0, t1, t2, a[23], b[28]); + MONT_MAC(v0, v1, v2, a[24], b[27]); + MONT_MAC(w0, w1, w2, a[25], b[26]); + MONT_MAC(t0, t1, t2, a[26], b[25]); + MONT_MAC(v0, v1, v2, a[27], b[24]); + MONT_MAC(w0, w1, w2, a[28], b[23]); + MONT_MAC(t0, t1, t2, a[29], b[22]); + MONT_MAC(v0, v1, v2, a[30], b[21]); + MONT_MAC(w0, w1, w2, a[31], b[20]); + MONT_MAC(t0, t1, t2, u[20], m[31]); + MONT_MAC(v0, v1, v2, u[21], m[30]); + MONT_MAC(w0, w1, w2, u[22], m[29]); + MONT_MAC(t0, t1, t2, u[23], m[28]); + MONT_MAC(v0, v1, v2, u[24], m[27]); + MONT_MAC(w0, w1, w2, u[25], m[26]); + MONT_MAC(t0, t1, t2, u[26], m[25]); + MONT_MAC(v0, v1, v2, u[27], m[24]); + MONT_MAC(w0, w1, w2, u[28], m[23]); + MONT_MAC(t0, t1, t2, u[29], m[22]); + MONT_MAC(v0, v1, v2, u[30], m[21]); + MONT_MAC(w0, w1, w2, u[31], m[20]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[19] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 52 */ + MONT_MAC(t0, t1, t2, a[21], b[31]); + MONT_MUL(v0, v1, v2, a[22], b[30]); + MONT_MUL(w0, w1, w2, a[23], b[29]); + MONT_MAC(t0, t1, t2, a[24], b[28]); + MONT_MAC(v0, v1, v2, a[25], b[27]); + MONT_MAC(w0, w1, w2, a[26], b[26]); + MONT_MAC(t0, t1, t2, a[27], b[25]); + MONT_MAC(v0, v1, v2, a[28], b[24]); + MONT_MAC(w0, w1, w2, a[29], b[23]); + MONT_MAC(t0, t1, t2, a[30], b[22]); + MONT_MAC(v0, v1, v2, a[31], b[21]); + MONT_MAC(w0, w1, w2, u[21], m[31]); + MONT_MAC(t0, t1, t2, u[22], m[30]); + MONT_MAC(v0, v1, v2, u[23], m[29]); + MONT_MAC(w0, w1, w2, u[24], m[28]); + MONT_MAC(t0, t1, t2, u[25], m[27]); + MONT_MAC(v0, v1, v2, u[26], m[26]); + MONT_MAC(w0, w1, w2, u[27], m[25]); + MONT_MAC(t0, t1, t2, u[28], m[24]); + MONT_MAC(v0, v1, v2, u[29], m[23]); + MONT_MAC(w0, w1, w2, u[30], m[22]); + MONT_MAC(t0, t1, t2, u[31], m[21]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[20] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 53 */ + MONT_MAC(t0, t1, t2, a[22], b[31]); + MONT_MUL(v0, v1, v2, a[23], b[30]); + MONT_MUL(w0, w1, w2, a[24], b[29]); + MONT_MAC(t0, t1, t2, a[25], b[28]); + MONT_MAC(v0, v1, v2, a[26], b[27]); + MONT_MAC(w0, w1, w2, a[27], b[26]); + MONT_MAC(t0, t1, t2, a[28], b[25]); + MONT_MAC(v0, v1, v2, a[29], b[24]); + MONT_MAC(w0, w1, w2, a[30], b[23]); + MONT_MAC(t0, t1, t2, a[31], b[22]); + MONT_MAC(v0, v1, v2, u[22], m[31]); + MONT_MAC(w0, w1, w2, u[23], m[30]); + MONT_MAC(t0, t1, t2, u[24], m[29]); + MONT_MAC(v0, v1, v2, u[25], m[28]); + MONT_MAC(w0, w1, w2, u[26], m[27]); + MONT_MAC(t0, t1, t2, u[27], m[26]); + MONT_MAC(v0, v1, v2, u[28], m[25]); + MONT_MAC(w0, w1, w2, u[29], m[24]); + MONT_MAC(t0, t1, t2, u[30], m[23]); + MONT_MAC(v0, v1, v2, u[31], m[22]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[21] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 54 */ + MONT_MAC(t0, t1, t2, a[23], b[31]); + MONT_MUL(v0, v1, v2, a[24], b[30]); + MONT_MUL(w0, w1, w2, a[25], b[29]); + MONT_MAC(t0, t1, t2, a[26], b[28]); + MONT_MAC(v0, v1, v2, a[27], b[27]); + MONT_MAC(w0, w1, w2, a[28], b[26]); + MONT_MAC(t0, t1, t2, a[29], b[25]); + MONT_MAC(v0, v1, v2, a[30], b[24]); + MONT_MAC(w0, w1, w2, a[31], b[23]); + MONT_MAC(t0, t1, t2, u[23], m[31]); + MONT_MAC(v0, v1, v2, u[24], m[30]); + MONT_MAC(w0, w1, w2, u[25], m[29]); + MONT_MAC(t0, t1, t2, u[26], m[28]); + MONT_MAC(v0, v1, v2, u[27], m[27]); + MONT_MAC(w0, w1, w2, u[28], m[26]); + MONT_MAC(t0, t1, t2, u[29], m[25]); + MONT_MAC(v0, v1, v2, u[30], m[24]); + MONT_MAC(w0, w1, w2, u[31], m[23]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[22] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 55 */ + MONT_MAC(t0, t1, t2, a[24], b[31]); + MONT_MUL(v0, v1, v2, a[25], b[30]); + MONT_MUL(w0, w1, w2, a[26], b[29]); + MONT_MAC(t0, t1, t2, a[27], b[28]); + MONT_MAC(v0, v1, v2, a[28], b[27]); + MONT_MAC(w0, w1, w2, a[29], b[26]); + MONT_MAC(t0, t1, t2, a[30], b[25]); + MONT_MAC(v0, v1, v2, a[31], b[24]); + MONT_MAC(w0, w1, w2, u[24], m[31]); + MONT_MAC(t0, t1, t2, u[25], m[30]); + MONT_MAC(v0, v1, v2, u[26], m[29]); + MONT_MAC(w0, w1, w2, u[27], m[28]); + MONT_MAC(t0, t1, t2, u[28], m[27]); + MONT_MAC(v0, v1, v2, u[29], m[26]); + MONT_MAC(w0, w1, w2, u[30], m[25]); + MONT_MAC(t0, t1, t2, u[31], m[24]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[23] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 56 */ + MONT_MAC(t0, t1, t2, a[25], b[31]); + MONT_MUL(v0, v1, v2, a[26], b[30]); + MONT_MUL(w0, w1, w2, a[27], b[29]); + MONT_MAC(t0, t1, t2, a[28], b[28]); + MONT_MAC(v0, v1, v2, a[29], b[27]); + MONT_MAC(w0, w1, w2, a[30], b[26]); + MONT_MAC(t0, t1, t2, a[31], b[25]); + MONT_MAC(v0, v1, v2, u[25], m[31]); + MONT_MAC(w0, w1, w2, u[26], m[30]); + MONT_MAC(t0, t1, t2, u[27], m[29]); + MONT_MAC(v0, v1, v2, u[28], m[28]); + MONT_MAC(w0, w1, w2, u[29], m[27]); + MONT_MAC(t0, t1, t2, u[30], m[26]); + MONT_MAC(v0, v1, v2, u[31], m[25]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[24] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 57 */ + MONT_MAC(t0, t1, t2, a[26], b[31]); + MONT_MUL(v0, v1, v2, a[27], b[30]); + MONT_MUL(w0, w1, w2, a[28], b[29]); + MONT_MAC(t0, t1, t2, a[29], b[28]); + MONT_MAC(v0, v1, v2, a[30], b[27]); + MONT_MAC(w0, w1, w2, a[31], b[26]); + MONT_MAC(t0, t1, t2, u[26], m[31]); + MONT_MAC(v0, v1, v2, u[27], m[30]); + MONT_MAC(w0, w1, w2, u[28], m[29]); + MONT_MAC(t0, t1, t2, u[29], m[28]); + MONT_MAC(v0, v1, v2, u[30], m[27]); + MONT_MAC(w0, w1, w2, u[31], m[26]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[25] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 58 */ + MONT_MAC(t0, t1, t2, a[27], b[31]); + MONT_MUL(v0, v1, v2, a[28], b[30]); + MONT_MUL(w0, w1, w2, a[29], b[29]); + MONT_MAC(t0, t1, t2, a[30], b[28]); + MONT_MAC(v0, v1, v2, a[31], b[27]); + MONT_MAC(w0, w1, w2, u[27], m[31]); + MONT_MAC(t0, t1, t2, u[28], m[30]); + MONT_MAC(v0, v1, v2, u[29], m[29]); + MONT_MAC(w0, w1, w2, u[30], m[28]); + MONT_MAC(t0, t1, t2, u[31], m[27]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[26] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 59 */ + MONT_MAC(t0, t1, t2, a[28], b[31]); + MONT_MUL(v0, v1, v2, a[29], b[30]); + MONT_MUL(w0, w1, w2, a[30], b[29]); + MONT_MAC(t0, t1, t2, a[31], b[28]); + MONT_MAC(v0, v1, v2, u[28], m[31]); + MONT_MAC(w0, w1, w2, u[29], m[30]); + MONT_MAC(t0, t1, t2, u[30], m[29]); + MONT_MAC(v0, v1, v2, u[31], m[28]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[27] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 60 */ + MONT_MAC(t0, t1, t2, a[29], b[31]); + MONT_MUL(v0, v1, v2, a[30], b[30]); + MONT_MUL(w0, w1, w2, a[31], b[29]); + MONT_MAC(t0, t1, t2, u[29], m[31]); + MONT_MAC(v0, v1, v2, u[30], m[30]); + MONT_MAC(w0, w1, w2, u[31], m[29]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[28] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 61 */ + MONT_MAC(t0, t1, t2, a[30], b[31]); + MONT_MUL(v0, v1, v2, a[31], b[30]); + MONT_MUL(w0, w1, w2, u[30], m[31]); + MONT_MAC(t0, t1, t2, u[31], m[30]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + z[29] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 62 */ + MONT_MAC(t0, t1, t2, a[31], b[31]); + MONT_MUL(v0, v1, v2, u[31], m[31]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[30] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[31] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 32); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 32); +} + +/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ +static void crypton_mont_sqr32(limb_t *r, const limb_t *a, + const limb_t *m, limb_t n0, limb_t *t) +{ + limb_t *u = t, *z = t + 32; + limb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take; + limb_t s0, s1, s2, c0, c1, c2; + + /* column 0 */ + MONT_MAC(t0, t1, t2, a[0], a[0]); + u[0] = t0 * n0; + MONT_MAC(t0, t1, t2, u[0], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 1 */ + MONT_MUL(s0, s1, s2, a[0], a[1]); + MONT_MAC(t0, t1, t2, u[0], m[1]); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[1] = t0 * n0; + MONT_MAC(t0, t1, t2, u[1], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 2 */ + MONT_MUL(s0, s1, s2, a[0], a[2]); + MONT_MAC(t0, t1, t2, a[1], a[1]); + MONT_MUL(v0, v1, v2, u[0], m[2]); + MONT_MUL(w0, w1, w2, u[1], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[2] = t0 * n0; + MONT_MAC(t0, t1, t2, u[2], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 3 */ + MONT_MUL(s0, s1, s2, a[0], a[3]); + MONT_MUL(c0, c1, c2, a[1], a[2]); + MONT_MAC(t0, t1, t2, u[0], m[3]); + MONT_MUL(v0, v1, v2, u[1], m[2]); + MONT_MUL(w0, w1, w2, u[2], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[3] = t0 * n0; + MONT_MAC(t0, t1, t2, u[3], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 4 */ + MONT_MUL(s0, s1, s2, a[0], a[4]); + MONT_MUL(c0, c1, c2, a[1], a[3]); + MONT_MAC(t0, t1, t2, a[2], a[2]); + MONT_MUL(v0, v1, v2, u[0], m[4]); + MONT_MUL(w0, w1, w2, u[1], m[3]); + MONT_MAC(t0, t1, t2, u[2], m[2]); + MONT_MAC(v0, v1, v2, u[3], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[4] = t0 * n0; + MONT_MAC(t0, t1, t2, u[4], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 5 */ + MONT_MUL(s0, s1, s2, a[0], a[5]); + MONT_MUL(c0, c1, c2, a[1], a[4]); + MONT_MAC(s0, s1, s2, a[2], a[3]); + MONT_MAC(t0, t1, t2, u[0], m[5]); + MONT_MUL(v0, v1, v2, u[1], m[4]); + MONT_MUL(w0, w1, w2, u[2], m[3]); + MONT_MAC(t0, t1, t2, u[3], m[2]); + MONT_MAC(v0, v1, v2, u[4], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[5] = t0 * n0; + MONT_MAC(t0, t1, t2, u[5], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 6 */ + MONT_MUL(s0, s1, s2, a[0], a[6]); + MONT_MUL(c0, c1, c2, a[1], a[5]); + MONT_MAC(s0, s1, s2, a[2], a[4]); + MONT_MAC(t0, t1, t2, a[3], a[3]); + MONT_MUL(v0, v1, v2, u[0], m[6]); + MONT_MUL(w0, w1, w2, u[1], m[5]); + MONT_MAC(t0, t1, t2, u[2], m[4]); + MONT_MAC(v0, v1, v2, u[3], m[3]); + MONT_MAC(w0, w1, w2, u[4], m[2]); + MONT_MAC(t0, t1, t2, u[5], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[6] = t0 * n0; + MONT_MAC(t0, t1, t2, u[6], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 7 */ + MONT_MUL(s0, s1, s2, a[0], a[7]); + MONT_MUL(c0, c1, c2, a[1], a[6]); + MONT_MAC(s0, s1, s2, a[2], a[5]); + MONT_MAC(c0, c1, c2, a[3], a[4]); + MONT_MAC(t0, t1, t2, u[0], m[7]); + MONT_MUL(v0, v1, v2, u[1], m[6]); + MONT_MUL(w0, w1, w2, u[2], m[5]); + MONT_MAC(t0, t1, t2, u[3], m[4]); + MONT_MAC(v0, v1, v2, u[4], m[3]); + MONT_MAC(w0, w1, w2, u[5], m[2]); + MONT_MAC(t0, t1, t2, u[6], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[7] = t0 * n0; + MONT_MAC(t0, t1, t2, u[7], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 8 */ + MONT_MUL(s0, s1, s2, a[0], a[8]); + MONT_MUL(c0, c1, c2, a[1], a[7]); + MONT_MAC(s0, s1, s2, a[2], a[6]); + MONT_MAC(c0, c1, c2, a[3], a[5]); + MONT_MAC(t0, t1, t2, a[4], a[4]); + MONT_MUL(v0, v1, v2, u[0], m[8]); + MONT_MUL(w0, w1, w2, u[1], m[7]); + MONT_MAC(t0, t1, t2, u[2], m[6]); + MONT_MAC(v0, v1, v2, u[3], m[5]); + MONT_MAC(w0, w1, w2, u[4], m[4]); + MONT_MAC(t0, t1, t2, u[5], m[3]); + MONT_MAC(v0, v1, v2, u[6], m[2]); + MONT_MAC(w0, w1, w2, u[7], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[8] = t0 * n0; + MONT_MAC(t0, t1, t2, u[8], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 9 */ + MONT_MUL(s0, s1, s2, a[0], a[9]); + MONT_MUL(c0, c1, c2, a[1], a[8]); + MONT_MAC(s0, s1, s2, a[2], a[7]); + MONT_MAC(c0, c1, c2, a[3], a[6]); + MONT_MAC(s0, s1, s2, a[4], a[5]); + MONT_MAC(t0, t1, t2, u[0], m[9]); + MONT_MUL(v0, v1, v2, u[1], m[8]); + MONT_MUL(w0, w1, w2, u[2], m[7]); + MONT_MAC(t0, t1, t2, u[3], m[6]); + MONT_MAC(v0, v1, v2, u[4], m[5]); + MONT_MAC(w0, w1, w2, u[5], m[4]); + MONT_MAC(t0, t1, t2, u[6], m[3]); + MONT_MAC(v0, v1, v2, u[7], m[2]); + MONT_MAC(w0, w1, w2, u[8], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[9] = t0 * n0; + MONT_MAC(t0, t1, t2, u[9], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 10 */ + MONT_MUL(s0, s1, s2, a[0], a[10]); + MONT_MUL(c0, c1, c2, a[1], a[9]); + MONT_MAC(s0, s1, s2, a[2], a[8]); + MONT_MAC(c0, c1, c2, a[3], a[7]); + MONT_MAC(s0, s1, s2, a[4], a[6]); + MONT_MAC(t0, t1, t2, a[5], a[5]); + MONT_MUL(v0, v1, v2, u[0], m[10]); + MONT_MUL(w0, w1, w2, u[1], m[9]); + MONT_MAC(t0, t1, t2, u[2], m[8]); + MONT_MAC(v0, v1, v2, u[3], m[7]); + MONT_MAC(w0, w1, w2, u[4], m[6]); + MONT_MAC(t0, t1, t2, u[5], m[5]); + MONT_MAC(v0, v1, v2, u[6], m[4]); + MONT_MAC(w0, w1, w2, u[7], m[3]); + MONT_MAC(t0, t1, t2, u[8], m[2]); + MONT_MAC(v0, v1, v2, u[9], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[10] = t0 * n0; + MONT_MAC(t0, t1, t2, u[10], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 11 */ + MONT_MUL(s0, s1, s2, a[0], a[11]); + MONT_MUL(c0, c1, c2, a[1], a[10]); + MONT_MAC(s0, s1, s2, a[2], a[9]); + MONT_MAC(c0, c1, c2, a[3], a[8]); + MONT_MAC(s0, s1, s2, a[4], a[7]); + MONT_MAC(c0, c1, c2, a[5], a[6]); + MONT_MAC(t0, t1, t2, u[0], m[11]); + MONT_MUL(v0, v1, v2, u[1], m[10]); + MONT_MUL(w0, w1, w2, u[2], m[9]); + MONT_MAC(t0, t1, t2, u[3], m[8]); + MONT_MAC(v0, v1, v2, u[4], m[7]); + MONT_MAC(w0, w1, w2, u[5], m[6]); + MONT_MAC(t0, t1, t2, u[6], m[5]); + MONT_MAC(v0, v1, v2, u[7], m[4]); + MONT_MAC(w0, w1, w2, u[8], m[3]); + MONT_MAC(t0, t1, t2, u[9], m[2]); + MONT_MAC(v0, v1, v2, u[10], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[11] = t0 * n0; + MONT_MAC(t0, t1, t2, u[11], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 12 */ + MONT_MUL(s0, s1, s2, a[0], a[12]); + MONT_MUL(c0, c1, c2, a[1], a[11]); + MONT_MAC(s0, s1, s2, a[2], a[10]); + MONT_MAC(c0, c1, c2, a[3], a[9]); + MONT_MAC(s0, s1, s2, a[4], a[8]); + MONT_MAC(c0, c1, c2, a[5], a[7]); + MONT_MAC(t0, t1, t2, a[6], a[6]); + MONT_MUL(v0, v1, v2, u[0], m[12]); + MONT_MUL(w0, w1, w2, u[1], m[11]); + MONT_MAC(t0, t1, t2, u[2], m[10]); + MONT_MAC(v0, v1, v2, u[3], m[9]); + MONT_MAC(w0, w1, w2, u[4], m[8]); + MONT_MAC(t0, t1, t2, u[5], m[7]); + MONT_MAC(v0, v1, v2, u[6], m[6]); + MONT_MAC(w0, w1, w2, u[7], m[5]); + MONT_MAC(t0, t1, t2, u[8], m[4]); + MONT_MAC(v0, v1, v2, u[9], m[3]); + MONT_MAC(w0, w1, w2, u[10], m[2]); + MONT_MAC(t0, t1, t2, u[11], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[12] = t0 * n0; + MONT_MAC(t0, t1, t2, u[12], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 13 */ + MONT_MUL(s0, s1, s2, a[0], a[13]); + MONT_MUL(c0, c1, c2, a[1], a[12]); + MONT_MAC(s0, s1, s2, a[2], a[11]); + MONT_MAC(c0, c1, c2, a[3], a[10]); + MONT_MAC(s0, s1, s2, a[4], a[9]); + MONT_MAC(c0, c1, c2, a[5], a[8]); + MONT_MAC(s0, s1, s2, a[6], a[7]); + MONT_MAC(t0, t1, t2, u[0], m[13]); + MONT_MUL(v0, v1, v2, u[1], m[12]); + MONT_MUL(w0, w1, w2, u[2], m[11]); + MONT_MAC(t0, t1, t2, u[3], m[10]); + MONT_MAC(v0, v1, v2, u[4], m[9]); + MONT_MAC(w0, w1, w2, u[5], m[8]); + MONT_MAC(t0, t1, t2, u[6], m[7]); + MONT_MAC(v0, v1, v2, u[7], m[6]); + MONT_MAC(w0, w1, w2, u[8], m[5]); + MONT_MAC(t0, t1, t2, u[9], m[4]); + MONT_MAC(v0, v1, v2, u[10], m[3]); + MONT_MAC(w0, w1, w2, u[11], m[2]); + MONT_MAC(t0, t1, t2, u[12], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[13] = t0 * n0; + MONT_MAC(t0, t1, t2, u[13], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 14 */ + MONT_MUL(s0, s1, s2, a[0], a[14]); + MONT_MUL(c0, c1, c2, a[1], a[13]); + MONT_MAC(s0, s1, s2, a[2], a[12]); + MONT_MAC(c0, c1, c2, a[3], a[11]); + MONT_MAC(s0, s1, s2, a[4], a[10]); + MONT_MAC(c0, c1, c2, a[5], a[9]); + MONT_MAC(s0, s1, s2, a[6], a[8]); + MONT_MAC(t0, t1, t2, a[7], a[7]); + MONT_MUL(v0, v1, v2, u[0], m[14]); + MONT_MUL(w0, w1, w2, u[1], m[13]); + MONT_MAC(t0, t1, t2, u[2], m[12]); + MONT_MAC(v0, v1, v2, u[3], m[11]); + MONT_MAC(w0, w1, w2, u[4], m[10]); + MONT_MAC(t0, t1, t2, u[5], m[9]); + MONT_MAC(v0, v1, v2, u[6], m[8]); + MONT_MAC(w0, w1, w2, u[7], m[7]); + MONT_MAC(t0, t1, t2, u[8], m[6]); + MONT_MAC(v0, v1, v2, u[9], m[5]); + MONT_MAC(w0, w1, w2, u[10], m[4]); + MONT_MAC(t0, t1, t2, u[11], m[3]); + MONT_MAC(v0, v1, v2, u[12], m[2]); + MONT_MAC(w0, w1, w2, u[13], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[14] = t0 * n0; + MONT_MAC(t0, t1, t2, u[14], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 15 */ + MONT_MUL(s0, s1, s2, a[0], a[15]); + MONT_MUL(c0, c1, c2, a[1], a[14]); + MONT_MAC(s0, s1, s2, a[2], a[13]); + MONT_MAC(c0, c1, c2, a[3], a[12]); + MONT_MAC(s0, s1, s2, a[4], a[11]); + MONT_MAC(c0, c1, c2, a[5], a[10]); + MONT_MAC(s0, s1, s2, a[6], a[9]); + MONT_MAC(c0, c1, c2, a[7], a[8]); + MONT_MAC(t0, t1, t2, u[0], m[15]); + MONT_MUL(v0, v1, v2, u[1], m[14]); + MONT_MUL(w0, w1, w2, u[2], m[13]); + MONT_MAC(t0, t1, t2, u[3], m[12]); + MONT_MAC(v0, v1, v2, u[4], m[11]); + MONT_MAC(w0, w1, w2, u[5], m[10]); + MONT_MAC(t0, t1, t2, u[6], m[9]); + MONT_MAC(v0, v1, v2, u[7], m[8]); + MONT_MAC(w0, w1, w2, u[8], m[7]); + MONT_MAC(t0, t1, t2, u[9], m[6]); + MONT_MAC(v0, v1, v2, u[10], m[5]); + MONT_MAC(w0, w1, w2, u[11], m[4]); + MONT_MAC(t0, t1, t2, u[12], m[3]); + MONT_MAC(v0, v1, v2, u[13], m[2]); + MONT_MAC(w0, w1, w2, u[14], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[15] = t0 * n0; + MONT_MAC(t0, t1, t2, u[15], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 16 */ + MONT_MUL(s0, s1, s2, a[0], a[16]); + MONT_MUL(c0, c1, c2, a[1], a[15]); + MONT_MAC(s0, s1, s2, a[2], a[14]); + MONT_MAC(c0, c1, c2, a[3], a[13]); + MONT_MAC(s0, s1, s2, a[4], a[12]); + MONT_MAC(c0, c1, c2, a[5], a[11]); + MONT_MAC(s0, s1, s2, a[6], a[10]); + MONT_MAC(c0, c1, c2, a[7], a[9]); + MONT_MAC(t0, t1, t2, a[8], a[8]); + MONT_MUL(v0, v1, v2, u[0], m[16]); + MONT_MUL(w0, w1, w2, u[1], m[15]); + MONT_MAC(t0, t1, t2, u[2], m[14]); + MONT_MAC(v0, v1, v2, u[3], m[13]); + MONT_MAC(w0, w1, w2, u[4], m[12]); + MONT_MAC(t0, t1, t2, u[5], m[11]); + MONT_MAC(v0, v1, v2, u[6], m[10]); + MONT_MAC(w0, w1, w2, u[7], m[9]); + MONT_MAC(t0, t1, t2, u[8], m[8]); + MONT_MAC(v0, v1, v2, u[9], m[7]); + MONT_MAC(w0, w1, w2, u[10], m[6]); + MONT_MAC(t0, t1, t2, u[11], m[5]); + MONT_MAC(v0, v1, v2, u[12], m[4]); + MONT_MAC(w0, w1, w2, u[13], m[3]); + MONT_MAC(t0, t1, t2, u[14], m[2]); + MONT_MAC(v0, v1, v2, u[15], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[16] = t0 * n0; + MONT_MAC(t0, t1, t2, u[16], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 17 */ + MONT_MUL(s0, s1, s2, a[0], a[17]); + MONT_MUL(c0, c1, c2, a[1], a[16]); + MONT_MAC(s0, s1, s2, a[2], a[15]); + MONT_MAC(c0, c1, c2, a[3], a[14]); + MONT_MAC(s0, s1, s2, a[4], a[13]); + MONT_MAC(c0, c1, c2, a[5], a[12]); + MONT_MAC(s0, s1, s2, a[6], a[11]); + MONT_MAC(c0, c1, c2, a[7], a[10]); + MONT_MAC(s0, s1, s2, a[8], a[9]); + MONT_MAC(t0, t1, t2, u[0], m[17]); + MONT_MUL(v0, v1, v2, u[1], m[16]); + MONT_MUL(w0, w1, w2, u[2], m[15]); + MONT_MAC(t0, t1, t2, u[3], m[14]); + MONT_MAC(v0, v1, v2, u[4], m[13]); + MONT_MAC(w0, w1, w2, u[5], m[12]); + MONT_MAC(t0, t1, t2, u[6], m[11]); + MONT_MAC(v0, v1, v2, u[7], m[10]); + MONT_MAC(w0, w1, w2, u[8], m[9]); + MONT_MAC(t0, t1, t2, u[9], m[8]); + MONT_MAC(v0, v1, v2, u[10], m[7]); + MONT_MAC(w0, w1, w2, u[11], m[6]); + MONT_MAC(t0, t1, t2, u[12], m[5]); + MONT_MAC(v0, v1, v2, u[13], m[4]); + MONT_MAC(w0, w1, w2, u[14], m[3]); + MONT_MAC(t0, t1, t2, u[15], m[2]); + MONT_MAC(v0, v1, v2, u[16], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[17] = t0 * n0; + MONT_MAC(t0, t1, t2, u[17], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 18 */ + MONT_MUL(s0, s1, s2, a[0], a[18]); + MONT_MUL(c0, c1, c2, a[1], a[17]); + MONT_MAC(s0, s1, s2, a[2], a[16]); + MONT_MAC(c0, c1, c2, a[3], a[15]); + MONT_MAC(s0, s1, s2, a[4], a[14]); + MONT_MAC(c0, c1, c2, a[5], a[13]); + MONT_MAC(s0, s1, s2, a[6], a[12]); + MONT_MAC(c0, c1, c2, a[7], a[11]); + MONT_MAC(s0, s1, s2, a[8], a[10]); + MONT_MAC(t0, t1, t2, a[9], a[9]); + MONT_MUL(v0, v1, v2, u[0], m[18]); + MONT_MUL(w0, w1, w2, u[1], m[17]); + MONT_MAC(t0, t1, t2, u[2], m[16]); + MONT_MAC(v0, v1, v2, u[3], m[15]); + MONT_MAC(w0, w1, w2, u[4], m[14]); + MONT_MAC(t0, t1, t2, u[5], m[13]); + MONT_MAC(v0, v1, v2, u[6], m[12]); + MONT_MAC(w0, w1, w2, u[7], m[11]); + MONT_MAC(t0, t1, t2, u[8], m[10]); + MONT_MAC(v0, v1, v2, u[9], m[9]); + MONT_MAC(w0, w1, w2, u[10], m[8]); + MONT_MAC(t0, t1, t2, u[11], m[7]); + MONT_MAC(v0, v1, v2, u[12], m[6]); + MONT_MAC(w0, w1, w2, u[13], m[5]); + MONT_MAC(t0, t1, t2, u[14], m[4]); + MONT_MAC(v0, v1, v2, u[15], m[3]); + MONT_MAC(w0, w1, w2, u[16], m[2]); + MONT_MAC(t0, t1, t2, u[17], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[18] = t0 * n0; + MONT_MAC(t0, t1, t2, u[18], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 19 */ + MONT_MUL(s0, s1, s2, a[0], a[19]); + MONT_MUL(c0, c1, c2, a[1], a[18]); + MONT_MAC(s0, s1, s2, a[2], a[17]); + MONT_MAC(c0, c1, c2, a[3], a[16]); + MONT_MAC(s0, s1, s2, a[4], a[15]); + MONT_MAC(c0, c1, c2, a[5], a[14]); + MONT_MAC(s0, s1, s2, a[6], a[13]); + MONT_MAC(c0, c1, c2, a[7], a[12]); + MONT_MAC(s0, s1, s2, a[8], a[11]); + MONT_MAC(c0, c1, c2, a[9], a[10]); + MONT_MAC(t0, t1, t2, u[0], m[19]); + MONT_MUL(v0, v1, v2, u[1], m[18]); + MONT_MUL(w0, w1, w2, u[2], m[17]); + MONT_MAC(t0, t1, t2, u[3], m[16]); + MONT_MAC(v0, v1, v2, u[4], m[15]); + MONT_MAC(w0, w1, w2, u[5], m[14]); + MONT_MAC(t0, t1, t2, u[6], m[13]); + MONT_MAC(v0, v1, v2, u[7], m[12]); + MONT_MAC(w0, w1, w2, u[8], m[11]); + MONT_MAC(t0, t1, t2, u[9], m[10]); + MONT_MAC(v0, v1, v2, u[10], m[9]); + MONT_MAC(w0, w1, w2, u[11], m[8]); + MONT_MAC(t0, t1, t2, u[12], m[7]); + MONT_MAC(v0, v1, v2, u[13], m[6]); + MONT_MAC(w0, w1, w2, u[14], m[5]); + MONT_MAC(t0, t1, t2, u[15], m[4]); + MONT_MAC(v0, v1, v2, u[16], m[3]); + MONT_MAC(w0, w1, w2, u[17], m[2]); + MONT_MAC(t0, t1, t2, u[18], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[19] = t0 * n0; + MONT_MAC(t0, t1, t2, u[19], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 20 */ + MONT_MUL(s0, s1, s2, a[0], a[20]); + MONT_MUL(c0, c1, c2, a[1], a[19]); + MONT_MAC(s0, s1, s2, a[2], a[18]); + MONT_MAC(c0, c1, c2, a[3], a[17]); + MONT_MAC(s0, s1, s2, a[4], a[16]); + MONT_MAC(c0, c1, c2, a[5], a[15]); + MONT_MAC(s0, s1, s2, a[6], a[14]); + MONT_MAC(c0, c1, c2, a[7], a[13]); + MONT_MAC(s0, s1, s2, a[8], a[12]); + MONT_MAC(c0, c1, c2, a[9], a[11]); + MONT_MAC(t0, t1, t2, a[10], a[10]); + MONT_MUL(v0, v1, v2, u[0], m[20]); + MONT_MUL(w0, w1, w2, u[1], m[19]); + MONT_MAC(t0, t1, t2, u[2], m[18]); + MONT_MAC(v0, v1, v2, u[3], m[17]); + MONT_MAC(w0, w1, w2, u[4], m[16]); + MONT_MAC(t0, t1, t2, u[5], m[15]); + MONT_MAC(v0, v1, v2, u[6], m[14]); + MONT_MAC(w0, w1, w2, u[7], m[13]); + MONT_MAC(t0, t1, t2, u[8], m[12]); + MONT_MAC(v0, v1, v2, u[9], m[11]); + MONT_MAC(w0, w1, w2, u[10], m[10]); + MONT_MAC(t0, t1, t2, u[11], m[9]); + MONT_MAC(v0, v1, v2, u[12], m[8]); + MONT_MAC(w0, w1, w2, u[13], m[7]); + MONT_MAC(t0, t1, t2, u[14], m[6]); + MONT_MAC(v0, v1, v2, u[15], m[5]); + MONT_MAC(w0, w1, w2, u[16], m[4]); + MONT_MAC(t0, t1, t2, u[17], m[3]); + MONT_MAC(v0, v1, v2, u[18], m[2]); + MONT_MAC(w0, w1, w2, u[19], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[20] = t0 * n0; + MONT_MAC(t0, t1, t2, u[20], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 21 */ + MONT_MUL(s0, s1, s2, a[0], a[21]); + MONT_MUL(c0, c1, c2, a[1], a[20]); + MONT_MAC(s0, s1, s2, a[2], a[19]); + MONT_MAC(c0, c1, c2, a[3], a[18]); + MONT_MAC(s0, s1, s2, a[4], a[17]); + MONT_MAC(c0, c1, c2, a[5], a[16]); + MONT_MAC(s0, s1, s2, a[6], a[15]); + MONT_MAC(c0, c1, c2, a[7], a[14]); + MONT_MAC(s0, s1, s2, a[8], a[13]); + MONT_MAC(c0, c1, c2, a[9], a[12]); + MONT_MAC(s0, s1, s2, a[10], a[11]); + MONT_MAC(t0, t1, t2, u[0], m[21]); + MONT_MUL(v0, v1, v2, u[1], m[20]); + MONT_MUL(w0, w1, w2, u[2], m[19]); + MONT_MAC(t0, t1, t2, u[3], m[18]); + MONT_MAC(v0, v1, v2, u[4], m[17]); + MONT_MAC(w0, w1, w2, u[5], m[16]); + MONT_MAC(t0, t1, t2, u[6], m[15]); + MONT_MAC(v0, v1, v2, u[7], m[14]); + MONT_MAC(w0, w1, w2, u[8], m[13]); + MONT_MAC(t0, t1, t2, u[9], m[12]); + MONT_MAC(v0, v1, v2, u[10], m[11]); + MONT_MAC(w0, w1, w2, u[11], m[10]); + MONT_MAC(t0, t1, t2, u[12], m[9]); + MONT_MAC(v0, v1, v2, u[13], m[8]); + MONT_MAC(w0, w1, w2, u[14], m[7]); + MONT_MAC(t0, t1, t2, u[15], m[6]); + MONT_MAC(v0, v1, v2, u[16], m[5]); + MONT_MAC(w0, w1, w2, u[17], m[4]); + MONT_MAC(t0, t1, t2, u[18], m[3]); + MONT_MAC(v0, v1, v2, u[19], m[2]); + MONT_MAC(w0, w1, w2, u[20], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[21] = t0 * n0; + MONT_MAC(t0, t1, t2, u[21], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 22 */ + MONT_MUL(s0, s1, s2, a[0], a[22]); + MONT_MUL(c0, c1, c2, a[1], a[21]); + MONT_MAC(s0, s1, s2, a[2], a[20]); + MONT_MAC(c0, c1, c2, a[3], a[19]); + MONT_MAC(s0, s1, s2, a[4], a[18]); + MONT_MAC(c0, c1, c2, a[5], a[17]); + MONT_MAC(s0, s1, s2, a[6], a[16]); + MONT_MAC(c0, c1, c2, a[7], a[15]); + MONT_MAC(s0, s1, s2, a[8], a[14]); + MONT_MAC(c0, c1, c2, a[9], a[13]); + MONT_MAC(s0, s1, s2, a[10], a[12]); + MONT_MAC(t0, t1, t2, a[11], a[11]); + MONT_MUL(v0, v1, v2, u[0], m[22]); + MONT_MUL(w0, w1, w2, u[1], m[21]); + MONT_MAC(t0, t1, t2, u[2], m[20]); + MONT_MAC(v0, v1, v2, u[3], m[19]); + MONT_MAC(w0, w1, w2, u[4], m[18]); + MONT_MAC(t0, t1, t2, u[5], m[17]); + MONT_MAC(v0, v1, v2, u[6], m[16]); + MONT_MAC(w0, w1, w2, u[7], m[15]); + MONT_MAC(t0, t1, t2, u[8], m[14]); + MONT_MAC(v0, v1, v2, u[9], m[13]); + MONT_MAC(w0, w1, w2, u[10], m[12]); + MONT_MAC(t0, t1, t2, u[11], m[11]); + MONT_MAC(v0, v1, v2, u[12], m[10]); + MONT_MAC(w0, w1, w2, u[13], m[9]); + MONT_MAC(t0, t1, t2, u[14], m[8]); + MONT_MAC(v0, v1, v2, u[15], m[7]); + MONT_MAC(w0, w1, w2, u[16], m[6]); + MONT_MAC(t0, t1, t2, u[17], m[5]); + MONT_MAC(v0, v1, v2, u[18], m[4]); + MONT_MAC(w0, w1, w2, u[19], m[3]); + MONT_MAC(t0, t1, t2, u[20], m[2]); + MONT_MAC(v0, v1, v2, u[21], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[22] = t0 * n0; + MONT_MAC(t0, t1, t2, u[22], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 23 */ + MONT_MUL(s0, s1, s2, a[0], a[23]); + MONT_MUL(c0, c1, c2, a[1], a[22]); + MONT_MAC(s0, s1, s2, a[2], a[21]); + MONT_MAC(c0, c1, c2, a[3], a[20]); + MONT_MAC(s0, s1, s2, a[4], a[19]); + MONT_MAC(c0, c1, c2, a[5], a[18]); + MONT_MAC(s0, s1, s2, a[6], a[17]); + MONT_MAC(c0, c1, c2, a[7], a[16]); + MONT_MAC(s0, s1, s2, a[8], a[15]); + MONT_MAC(c0, c1, c2, a[9], a[14]); + MONT_MAC(s0, s1, s2, a[10], a[13]); + MONT_MAC(c0, c1, c2, a[11], a[12]); + MONT_MAC(t0, t1, t2, u[0], m[23]); + MONT_MUL(v0, v1, v2, u[1], m[22]); + MONT_MUL(w0, w1, w2, u[2], m[21]); + MONT_MAC(t0, t1, t2, u[3], m[20]); + MONT_MAC(v0, v1, v2, u[4], m[19]); + MONT_MAC(w0, w1, w2, u[5], m[18]); + MONT_MAC(t0, t1, t2, u[6], m[17]); + MONT_MAC(v0, v1, v2, u[7], m[16]); + MONT_MAC(w0, w1, w2, u[8], m[15]); + MONT_MAC(t0, t1, t2, u[9], m[14]); + MONT_MAC(v0, v1, v2, u[10], m[13]); + MONT_MAC(w0, w1, w2, u[11], m[12]); + MONT_MAC(t0, t1, t2, u[12], m[11]); + MONT_MAC(v0, v1, v2, u[13], m[10]); + MONT_MAC(w0, w1, w2, u[14], m[9]); + MONT_MAC(t0, t1, t2, u[15], m[8]); + MONT_MAC(v0, v1, v2, u[16], m[7]); + MONT_MAC(w0, w1, w2, u[17], m[6]); + MONT_MAC(t0, t1, t2, u[18], m[5]); + MONT_MAC(v0, v1, v2, u[19], m[4]); + MONT_MAC(w0, w1, w2, u[20], m[3]); + MONT_MAC(t0, t1, t2, u[21], m[2]); + MONT_MAC(v0, v1, v2, u[22], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[23] = t0 * n0; + MONT_MAC(t0, t1, t2, u[23], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 24 */ + MONT_MUL(s0, s1, s2, a[0], a[24]); + MONT_MUL(c0, c1, c2, a[1], a[23]); + MONT_MAC(s0, s1, s2, a[2], a[22]); + MONT_MAC(c0, c1, c2, a[3], a[21]); + MONT_MAC(s0, s1, s2, a[4], a[20]); + MONT_MAC(c0, c1, c2, a[5], a[19]); + MONT_MAC(s0, s1, s2, a[6], a[18]); + MONT_MAC(c0, c1, c2, a[7], a[17]); + MONT_MAC(s0, s1, s2, a[8], a[16]); + MONT_MAC(c0, c1, c2, a[9], a[15]); + MONT_MAC(s0, s1, s2, a[10], a[14]); + MONT_MAC(c0, c1, c2, a[11], a[13]); + MONT_MAC(t0, t1, t2, a[12], a[12]); + MONT_MUL(v0, v1, v2, u[0], m[24]); + MONT_MUL(w0, w1, w2, u[1], m[23]); + MONT_MAC(t0, t1, t2, u[2], m[22]); + MONT_MAC(v0, v1, v2, u[3], m[21]); + MONT_MAC(w0, w1, w2, u[4], m[20]); + MONT_MAC(t0, t1, t2, u[5], m[19]); + MONT_MAC(v0, v1, v2, u[6], m[18]); + MONT_MAC(w0, w1, w2, u[7], m[17]); + MONT_MAC(t0, t1, t2, u[8], m[16]); + MONT_MAC(v0, v1, v2, u[9], m[15]); + MONT_MAC(w0, w1, w2, u[10], m[14]); + MONT_MAC(t0, t1, t2, u[11], m[13]); + MONT_MAC(v0, v1, v2, u[12], m[12]); + MONT_MAC(w0, w1, w2, u[13], m[11]); + MONT_MAC(t0, t1, t2, u[14], m[10]); + MONT_MAC(v0, v1, v2, u[15], m[9]); + MONT_MAC(w0, w1, w2, u[16], m[8]); + MONT_MAC(t0, t1, t2, u[17], m[7]); + MONT_MAC(v0, v1, v2, u[18], m[6]); + MONT_MAC(w0, w1, w2, u[19], m[5]); + MONT_MAC(t0, t1, t2, u[20], m[4]); + MONT_MAC(v0, v1, v2, u[21], m[3]); + MONT_MAC(w0, w1, w2, u[22], m[2]); + MONT_MAC(t0, t1, t2, u[23], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[24] = t0 * n0; + MONT_MAC(t0, t1, t2, u[24], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 25 */ + MONT_MUL(s0, s1, s2, a[0], a[25]); + MONT_MUL(c0, c1, c2, a[1], a[24]); + MONT_MAC(s0, s1, s2, a[2], a[23]); + MONT_MAC(c0, c1, c2, a[3], a[22]); + MONT_MAC(s0, s1, s2, a[4], a[21]); + MONT_MAC(c0, c1, c2, a[5], a[20]); + MONT_MAC(s0, s1, s2, a[6], a[19]); + MONT_MAC(c0, c1, c2, a[7], a[18]); + MONT_MAC(s0, s1, s2, a[8], a[17]); + MONT_MAC(c0, c1, c2, a[9], a[16]); + MONT_MAC(s0, s1, s2, a[10], a[15]); + MONT_MAC(c0, c1, c2, a[11], a[14]); + MONT_MAC(s0, s1, s2, a[12], a[13]); + MONT_MAC(t0, t1, t2, u[0], m[25]); + MONT_MUL(v0, v1, v2, u[1], m[24]); + MONT_MUL(w0, w1, w2, u[2], m[23]); + MONT_MAC(t0, t1, t2, u[3], m[22]); + MONT_MAC(v0, v1, v2, u[4], m[21]); + MONT_MAC(w0, w1, w2, u[5], m[20]); + MONT_MAC(t0, t1, t2, u[6], m[19]); + MONT_MAC(v0, v1, v2, u[7], m[18]); + MONT_MAC(w0, w1, w2, u[8], m[17]); + MONT_MAC(t0, t1, t2, u[9], m[16]); + MONT_MAC(v0, v1, v2, u[10], m[15]); + MONT_MAC(w0, w1, w2, u[11], m[14]); + MONT_MAC(t0, t1, t2, u[12], m[13]); + MONT_MAC(v0, v1, v2, u[13], m[12]); + MONT_MAC(w0, w1, w2, u[14], m[11]); + MONT_MAC(t0, t1, t2, u[15], m[10]); + MONT_MAC(v0, v1, v2, u[16], m[9]); + MONT_MAC(w0, w1, w2, u[17], m[8]); + MONT_MAC(t0, t1, t2, u[18], m[7]); + MONT_MAC(v0, v1, v2, u[19], m[6]); + MONT_MAC(w0, w1, w2, u[20], m[5]); + MONT_MAC(t0, t1, t2, u[21], m[4]); + MONT_MAC(v0, v1, v2, u[22], m[3]); + MONT_MAC(w0, w1, w2, u[23], m[2]); + MONT_MAC(t0, t1, t2, u[24], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[25] = t0 * n0; + MONT_MAC(t0, t1, t2, u[25], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 26 */ + MONT_MUL(s0, s1, s2, a[0], a[26]); + MONT_MUL(c0, c1, c2, a[1], a[25]); + MONT_MAC(s0, s1, s2, a[2], a[24]); + MONT_MAC(c0, c1, c2, a[3], a[23]); + MONT_MAC(s0, s1, s2, a[4], a[22]); + MONT_MAC(c0, c1, c2, a[5], a[21]); + MONT_MAC(s0, s1, s2, a[6], a[20]); + MONT_MAC(c0, c1, c2, a[7], a[19]); + MONT_MAC(s0, s1, s2, a[8], a[18]); + MONT_MAC(c0, c1, c2, a[9], a[17]); + MONT_MAC(s0, s1, s2, a[10], a[16]); + MONT_MAC(c0, c1, c2, a[11], a[15]); + MONT_MAC(s0, s1, s2, a[12], a[14]); + MONT_MAC(t0, t1, t2, a[13], a[13]); + MONT_MUL(v0, v1, v2, u[0], m[26]); + MONT_MUL(w0, w1, w2, u[1], m[25]); + MONT_MAC(t0, t1, t2, u[2], m[24]); + MONT_MAC(v0, v1, v2, u[3], m[23]); + MONT_MAC(w0, w1, w2, u[4], m[22]); + MONT_MAC(t0, t1, t2, u[5], m[21]); + MONT_MAC(v0, v1, v2, u[6], m[20]); + MONT_MAC(w0, w1, w2, u[7], m[19]); + MONT_MAC(t0, t1, t2, u[8], m[18]); + MONT_MAC(v0, v1, v2, u[9], m[17]); + MONT_MAC(w0, w1, w2, u[10], m[16]); + MONT_MAC(t0, t1, t2, u[11], m[15]); + MONT_MAC(v0, v1, v2, u[12], m[14]); + MONT_MAC(w0, w1, w2, u[13], m[13]); + MONT_MAC(t0, t1, t2, u[14], m[12]); + MONT_MAC(v0, v1, v2, u[15], m[11]); + MONT_MAC(w0, w1, w2, u[16], m[10]); + MONT_MAC(t0, t1, t2, u[17], m[9]); + MONT_MAC(v0, v1, v2, u[18], m[8]); + MONT_MAC(w0, w1, w2, u[19], m[7]); + MONT_MAC(t0, t1, t2, u[20], m[6]); + MONT_MAC(v0, v1, v2, u[21], m[5]); + MONT_MAC(w0, w1, w2, u[22], m[4]); + MONT_MAC(t0, t1, t2, u[23], m[3]); + MONT_MAC(v0, v1, v2, u[24], m[2]); + MONT_MAC(w0, w1, w2, u[25], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[26] = t0 * n0; + MONT_MAC(t0, t1, t2, u[26], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 27 */ + MONT_MUL(s0, s1, s2, a[0], a[27]); + MONT_MUL(c0, c1, c2, a[1], a[26]); + MONT_MAC(s0, s1, s2, a[2], a[25]); + MONT_MAC(c0, c1, c2, a[3], a[24]); + MONT_MAC(s0, s1, s2, a[4], a[23]); + MONT_MAC(c0, c1, c2, a[5], a[22]); + MONT_MAC(s0, s1, s2, a[6], a[21]); + MONT_MAC(c0, c1, c2, a[7], a[20]); + MONT_MAC(s0, s1, s2, a[8], a[19]); + MONT_MAC(c0, c1, c2, a[9], a[18]); + MONT_MAC(s0, s1, s2, a[10], a[17]); + MONT_MAC(c0, c1, c2, a[11], a[16]); + MONT_MAC(s0, s1, s2, a[12], a[15]); + MONT_MAC(c0, c1, c2, a[13], a[14]); + MONT_MAC(t0, t1, t2, u[0], m[27]); + MONT_MUL(v0, v1, v2, u[1], m[26]); + MONT_MUL(w0, w1, w2, u[2], m[25]); + MONT_MAC(t0, t1, t2, u[3], m[24]); + MONT_MAC(v0, v1, v2, u[4], m[23]); + MONT_MAC(w0, w1, w2, u[5], m[22]); + MONT_MAC(t0, t1, t2, u[6], m[21]); + MONT_MAC(v0, v1, v2, u[7], m[20]); + MONT_MAC(w0, w1, w2, u[8], m[19]); + MONT_MAC(t0, t1, t2, u[9], m[18]); + MONT_MAC(v0, v1, v2, u[10], m[17]); + MONT_MAC(w0, w1, w2, u[11], m[16]); + MONT_MAC(t0, t1, t2, u[12], m[15]); + MONT_MAC(v0, v1, v2, u[13], m[14]); + MONT_MAC(w0, w1, w2, u[14], m[13]); + MONT_MAC(t0, t1, t2, u[15], m[12]); + MONT_MAC(v0, v1, v2, u[16], m[11]); + MONT_MAC(w0, w1, w2, u[17], m[10]); + MONT_MAC(t0, t1, t2, u[18], m[9]); + MONT_MAC(v0, v1, v2, u[19], m[8]); + MONT_MAC(w0, w1, w2, u[20], m[7]); + MONT_MAC(t0, t1, t2, u[21], m[6]); + MONT_MAC(v0, v1, v2, u[22], m[5]); + MONT_MAC(w0, w1, w2, u[23], m[4]); + MONT_MAC(t0, t1, t2, u[24], m[3]); + MONT_MAC(v0, v1, v2, u[25], m[2]); + MONT_MAC(w0, w1, w2, u[26], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[27] = t0 * n0; + MONT_MAC(t0, t1, t2, u[27], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 28 */ + MONT_MUL(s0, s1, s2, a[0], a[28]); + MONT_MUL(c0, c1, c2, a[1], a[27]); + MONT_MAC(s0, s1, s2, a[2], a[26]); + MONT_MAC(c0, c1, c2, a[3], a[25]); + MONT_MAC(s0, s1, s2, a[4], a[24]); + MONT_MAC(c0, c1, c2, a[5], a[23]); + MONT_MAC(s0, s1, s2, a[6], a[22]); + MONT_MAC(c0, c1, c2, a[7], a[21]); + MONT_MAC(s0, s1, s2, a[8], a[20]); + MONT_MAC(c0, c1, c2, a[9], a[19]); + MONT_MAC(s0, s1, s2, a[10], a[18]); + MONT_MAC(c0, c1, c2, a[11], a[17]); + MONT_MAC(s0, s1, s2, a[12], a[16]); + MONT_MAC(c0, c1, c2, a[13], a[15]); + MONT_MAC(t0, t1, t2, a[14], a[14]); + MONT_MUL(v0, v1, v2, u[0], m[28]); + MONT_MUL(w0, w1, w2, u[1], m[27]); + MONT_MAC(t0, t1, t2, u[2], m[26]); + MONT_MAC(v0, v1, v2, u[3], m[25]); + MONT_MAC(w0, w1, w2, u[4], m[24]); + MONT_MAC(t0, t1, t2, u[5], m[23]); + MONT_MAC(v0, v1, v2, u[6], m[22]); + MONT_MAC(w0, w1, w2, u[7], m[21]); + MONT_MAC(t0, t1, t2, u[8], m[20]); + MONT_MAC(v0, v1, v2, u[9], m[19]); + MONT_MAC(w0, w1, w2, u[10], m[18]); + MONT_MAC(t0, t1, t2, u[11], m[17]); + MONT_MAC(v0, v1, v2, u[12], m[16]); + MONT_MAC(w0, w1, w2, u[13], m[15]); + MONT_MAC(t0, t1, t2, u[14], m[14]); + MONT_MAC(v0, v1, v2, u[15], m[13]); + MONT_MAC(w0, w1, w2, u[16], m[12]); + MONT_MAC(t0, t1, t2, u[17], m[11]); + MONT_MAC(v0, v1, v2, u[18], m[10]); + MONT_MAC(w0, w1, w2, u[19], m[9]); + MONT_MAC(t0, t1, t2, u[20], m[8]); + MONT_MAC(v0, v1, v2, u[21], m[7]); + MONT_MAC(w0, w1, w2, u[22], m[6]); + MONT_MAC(t0, t1, t2, u[23], m[5]); + MONT_MAC(v0, v1, v2, u[24], m[4]); + MONT_MAC(w0, w1, w2, u[25], m[3]); + MONT_MAC(t0, t1, t2, u[26], m[2]); + MONT_MAC(v0, v1, v2, u[27], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[28] = t0 * n0; + MONT_MAC(t0, t1, t2, u[28], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 29 */ + MONT_MUL(s0, s1, s2, a[0], a[29]); + MONT_MUL(c0, c1, c2, a[1], a[28]); + MONT_MAC(s0, s1, s2, a[2], a[27]); + MONT_MAC(c0, c1, c2, a[3], a[26]); + MONT_MAC(s0, s1, s2, a[4], a[25]); + MONT_MAC(c0, c1, c2, a[5], a[24]); + MONT_MAC(s0, s1, s2, a[6], a[23]); + MONT_MAC(c0, c1, c2, a[7], a[22]); + MONT_MAC(s0, s1, s2, a[8], a[21]); + MONT_MAC(c0, c1, c2, a[9], a[20]); + MONT_MAC(s0, s1, s2, a[10], a[19]); + MONT_MAC(c0, c1, c2, a[11], a[18]); + MONT_MAC(s0, s1, s2, a[12], a[17]); + MONT_MAC(c0, c1, c2, a[13], a[16]); + MONT_MAC(s0, s1, s2, a[14], a[15]); + MONT_MAC(t0, t1, t2, u[0], m[29]); + MONT_MUL(v0, v1, v2, u[1], m[28]); + MONT_MUL(w0, w1, w2, u[2], m[27]); + MONT_MAC(t0, t1, t2, u[3], m[26]); + MONT_MAC(v0, v1, v2, u[4], m[25]); + MONT_MAC(w0, w1, w2, u[5], m[24]); + MONT_MAC(t0, t1, t2, u[6], m[23]); + MONT_MAC(v0, v1, v2, u[7], m[22]); + MONT_MAC(w0, w1, w2, u[8], m[21]); + MONT_MAC(t0, t1, t2, u[9], m[20]); + MONT_MAC(v0, v1, v2, u[10], m[19]); + MONT_MAC(w0, w1, w2, u[11], m[18]); + MONT_MAC(t0, t1, t2, u[12], m[17]); + MONT_MAC(v0, v1, v2, u[13], m[16]); + MONT_MAC(w0, w1, w2, u[14], m[15]); + MONT_MAC(t0, t1, t2, u[15], m[14]); + MONT_MAC(v0, v1, v2, u[16], m[13]); + MONT_MAC(w0, w1, w2, u[17], m[12]); + MONT_MAC(t0, t1, t2, u[18], m[11]); + MONT_MAC(v0, v1, v2, u[19], m[10]); + MONT_MAC(w0, w1, w2, u[20], m[9]); + MONT_MAC(t0, t1, t2, u[21], m[8]); + MONT_MAC(v0, v1, v2, u[22], m[7]); + MONT_MAC(w0, w1, w2, u[23], m[6]); + MONT_MAC(t0, t1, t2, u[24], m[5]); + MONT_MAC(v0, v1, v2, u[25], m[4]); + MONT_MAC(w0, w1, w2, u[26], m[3]); + MONT_MAC(t0, t1, t2, u[27], m[2]); + MONT_MAC(v0, v1, v2, u[28], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[29] = t0 * n0; + MONT_MAC(t0, t1, t2, u[29], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 30 */ + MONT_MUL(s0, s1, s2, a[0], a[30]); + MONT_MUL(c0, c1, c2, a[1], a[29]); + MONT_MAC(s0, s1, s2, a[2], a[28]); + MONT_MAC(c0, c1, c2, a[3], a[27]); + MONT_MAC(s0, s1, s2, a[4], a[26]); + MONT_MAC(c0, c1, c2, a[5], a[25]); + MONT_MAC(s0, s1, s2, a[6], a[24]); + MONT_MAC(c0, c1, c2, a[7], a[23]); + MONT_MAC(s0, s1, s2, a[8], a[22]); + MONT_MAC(c0, c1, c2, a[9], a[21]); + MONT_MAC(s0, s1, s2, a[10], a[20]); + MONT_MAC(c0, c1, c2, a[11], a[19]); + MONT_MAC(s0, s1, s2, a[12], a[18]); + MONT_MAC(c0, c1, c2, a[13], a[17]); + MONT_MAC(s0, s1, s2, a[14], a[16]); + MONT_MAC(t0, t1, t2, a[15], a[15]); + MONT_MUL(v0, v1, v2, u[0], m[30]); + MONT_MUL(w0, w1, w2, u[1], m[29]); + MONT_MAC(t0, t1, t2, u[2], m[28]); + MONT_MAC(v0, v1, v2, u[3], m[27]); + MONT_MAC(w0, w1, w2, u[4], m[26]); + MONT_MAC(t0, t1, t2, u[5], m[25]); + MONT_MAC(v0, v1, v2, u[6], m[24]); + MONT_MAC(w0, w1, w2, u[7], m[23]); + MONT_MAC(t0, t1, t2, u[8], m[22]); + MONT_MAC(v0, v1, v2, u[9], m[21]); + MONT_MAC(w0, w1, w2, u[10], m[20]); + MONT_MAC(t0, t1, t2, u[11], m[19]); + MONT_MAC(v0, v1, v2, u[12], m[18]); + MONT_MAC(w0, w1, w2, u[13], m[17]); + MONT_MAC(t0, t1, t2, u[14], m[16]); + MONT_MAC(v0, v1, v2, u[15], m[15]); + MONT_MAC(w0, w1, w2, u[16], m[14]); + MONT_MAC(t0, t1, t2, u[17], m[13]); + MONT_MAC(v0, v1, v2, u[18], m[12]); + MONT_MAC(w0, w1, w2, u[19], m[11]); + MONT_MAC(t0, t1, t2, u[20], m[10]); + MONT_MAC(v0, v1, v2, u[21], m[9]); + MONT_MAC(w0, w1, w2, u[22], m[8]); + MONT_MAC(t0, t1, t2, u[23], m[7]); + MONT_MAC(v0, v1, v2, u[24], m[6]); + MONT_MAC(w0, w1, w2, u[25], m[5]); + MONT_MAC(t0, t1, t2, u[26], m[4]); + MONT_MAC(v0, v1, v2, u[27], m[3]); + MONT_MAC(w0, w1, w2, u[28], m[2]); + MONT_MAC(t0, t1, t2, u[29], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[30] = t0 * n0; + MONT_MAC(t0, t1, t2, u[30], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 31 */ + MONT_MUL(s0, s1, s2, a[0], a[31]); + MONT_MUL(c0, c1, c2, a[1], a[30]); + MONT_MAC(s0, s1, s2, a[2], a[29]); + MONT_MAC(c0, c1, c2, a[3], a[28]); + MONT_MAC(s0, s1, s2, a[4], a[27]); + MONT_MAC(c0, c1, c2, a[5], a[26]); + MONT_MAC(s0, s1, s2, a[6], a[25]); + MONT_MAC(c0, c1, c2, a[7], a[24]); + MONT_MAC(s0, s1, s2, a[8], a[23]); + MONT_MAC(c0, c1, c2, a[9], a[22]); + MONT_MAC(s0, s1, s2, a[10], a[21]); + MONT_MAC(c0, c1, c2, a[11], a[20]); + MONT_MAC(s0, s1, s2, a[12], a[19]); + MONT_MAC(c0, c1, c2, a[13], a[18]); + MONT_MAC(s0, s1, s2, a[14], a[17]); + MONT_MAC(c0, c1, c2, a[15], a[16]); + MONT_MAC(t0, t1, t2, u[0], m[31]); + MONT_MUL(v0, v1, v2, u[1], m[30]); + MONT_MUL(w0, w1, w2, u[2], m[29]); + MONT_MAC(t0, t1, t2, u[3], m[28]); + MONT_MAC(v0, v1, v2, u[4], m[27]); + MONT_MAC(w0, w1, w2, u[5], m[26]); + MONT_MAC(t0, t1, t2, u[6], m[25]); + MONT_MAC(v0, v1, v2, u[7], m[24]); + MONT_MAC(w0, w1, w2, u[8], m[23]); + MONT_MAC(t0, t1, t2, u[9], m[22]); + MONT_MAC(v0, v1, v2, u[10], m[21]); + MONT_MAC(w0, w1, w2, u[11], m[20]); + MONT_MAC(t0, t1, t2, u[12], m[19]); + MONT_MAC(v0, v1, v2, u[13], m[18]); + MONT_MAC(w0, w1, w2, u[14], m[17]); + MONT_MAC(t0, t1, t2, u[15], m[16]); + MONT_MAC(v0, v1, v2, u[16], m[15]); + MONT_MAC(w0, w1, w2, u[17], m[14]); + MONT_MAC(t0, t1, t2, u[18], m[13]); + MONT_MAC(v0, v1, v2, u[19], m[12]); + MONT_MAC(w0, w1, w2, u[20], m[11]); + MONT_MAC(t0, t1, t2, u[21], m[10]); + MONT_MAC(v0, v1, v2, u[22], m[9]); + MONT_MAC(w0, w1, w2, u[23], m[8]); + MONT_MAC(t0, t1, t2, u[24], m[7]); + MONT_MAC(v0, v1, v2, u[25], m[6]); + MONT_MAC(w0, w1, w2, u[26], m[5]); + MONT_MAC(t0, t1, t2, u[27], m[4]); + MONT_MAC(v0, v1, v2, u[28], m[3]); + MONT_MAC(w0, w1, w2, u[29], m[2]); + MONT_MAC(t0, t1, t2, u[30], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + u[31] = t0 * n0; + MONT_MAC(t0, t1, t2, u[31], m[0]); + t0 = t1; + t1 = t2; + t2 = 0; + /* column 32 */ + MONT_MUL(s0, s1, s2, a[1], a[31]); + MONT_MUL(c0, c1, c2, a[2], a[30]); + MONT_MAC(s0, s1, s2, a[3], a[29]); + MONT_MAC(c0, c1, c2, a[4], a[28]); + MONT_MAC(s0, s1, s2, a[5], a[27]); + MONT_MAC(c0, c1, c2, a[6], a[26]); + MONT_MAC(s0, s1, s2, a[7], a[25]); + MONT_MAC(c0, c1, c2, a[8], a[24]); + MONT_MAC(s0, s1, s2, a[9], a[23]); + MONT_MAC(c0, c1, c2, a[10], a[22]); + MONT_MAC(s0, s1, s2, a[11], a[21]); + MONT_MAC(c0, c1, c2, a[12], a[20]); + MONT_MAC(s0, s1, s2, a[13], a[19]); + MONT_MAC(c0, c1, c2, a[14], a[18]); + MONT_MAC(s0, s1, s2, a[15], a[17]); + MONT_MAC(t0, t1, t2, a[16], a[16]); + MONT_MUL(v0, v1, v2, u[1], m[31]); + MONT_MUL(w0, w1, w2, u[2], m[30]); + MONT_MAC(t0, t1, t2, u[3], m[29]); + MONT_MAC(v0, v1, v2, u[4], m[28]); + MONT_MAC(w0, w1, w2, u[5], m[27]); + MONT_MAC(t0, t1, t2, u[6], m[26]); + MONT_MAC(v0, v1, v2, u[7], m[25]); + MONT_MAC(w0, w1, w2, u[8], m[24]); + MONT_MAC(t0, t1, t2, u[9], m[23]); + MONT_MAC(v0, v1, v2, u[10], m[22]); + MONT_MAC(w0, w1, w2, u[11], m[21]); + MONT_MAC(t0, t1, t2, u[12], m[20]); + MONT_MAC(v0, v1, v2, u[13], m[19]); + MONT_MAC(w0, w1, w2, u[14], m[18]); + MONT_MAC(t0, t1, t2, u[15], m[17]); + MONT_MAC(v0, v1, v2, u[16], m[16]); + MONT_MAC(w0, w1, w2, u[17], m[15]); + MONT_MAC(t0, t1, t2, u[18], m[14]); + MONT_MAC(v0, v1, v2, u[19], m[13]); + MONT_MAC(w0, w1, w2, u[20], m[12]); + MONT_MAC(t0, t1, t2, u[21], m[11]); + MONT_MAC(v0, v1, v2, u[22], m[10]); + MONT_MAC(w0, w1, w2, u[23], m[9]); + MONT_MAC(t0, t1, t2, u[24], m[8]); + MONT_MAC(v0, v1, v2, u[25], m[7]); + MONT_MAC(w0, w1, w2, u[26], m[6]); + MONT_MAC(t0, t1, t2, u[27], m[5]); + MONT_MAC(v0, v1, v2, u[28], m[4]); + MONT_MAC(w0, w1, w2, u[29], m[3]); + MONT_MAC(t0, t1, t2, u[30], m[2]); + MONT_MAC(v0, v1, v2, u[31], m[1]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[0] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 33 */ + MONT_MUL(s0, s1, s2, a[2], a[31]); + MONT_MUL(c0, c1, c2, a[3], a[30]); + MONT_MAC(s0, s1, s2, a[4], a[29]); + MONT_MAC(c0, c1, c2, a[5], a[28]); + MONT_MAC(s0, s1, s2, a[6], a[27]); + MONT_MAC(c0, c1, c2, a[7], a[26]); + MONT_MAC(s0, s1, s2, a[8], a[25]); + MONT_MAC(c0, c1, c2, a[9], a[24]); + MONT_MAC(s0, s1, s2, a[10], a[23]); + MONT_MAC(c0, c1, c2, a[11], a[22]); + MONT_MAC(s0, s1, s2, a[12], a[21]); + MONT_MAC(c0, c1, c2, a[13], a[20]); + MONT_MAC(s0, s1, s2, a[14], a[19]); + MONT_MAC(c0, c1, c2, a[15], a[18]); + MONT_MAC(s0, s1, s2, a[16], a[17]); + MONT_MAC(t0, t1, t2, u[2], m[31]); + MONT_MUL(v0, v1, v2, u[3], m[30]); + MONT_MUL(w0, w1, w2, u[4], m[29]); + MONT_MAC(t0, t1, t2, u[5], m[28]); + MONT_MAC(v0, v1, v2, u[6], m[27]); + MONT_MAC(w0, w1, w2, u[7], m[26]); + MONT_MAC(t0, t1, t2, u[8], m[25]); + MONT_MAC(v0, v1, v2, u[9], m[24]); + MONT_MAC(w0, w1, w2, u[10], m[23]); + MONT_MAC(t0, t1, t2, u[11], m[22]); + MONT_MAC(v0, v1, v2, u[12], m[21]); + MONT_MAC(w0, w1, w2, u[13], m[20]); + MONT_MAC(t0, t1, t2, u[14], m[19]); + MONT_MAC(v0, v1, v2, u[15], m[18]); + MONT_MAC(w0, w1, w2, u[16], m[17]); + MONT_MAC(t0, t1, t2, u[17], m[16]); + MONT_MAC(v0, v1, v2, u[18], m[15]); + MONT_MAC(w0, w1, w2, u[19], m[14]); + MONT_MAC(t0, t1, t2, u[20], m[13]); + MONT_MAC(v0, v1, v2, u[21], m[12]); + MONT_MAC(w0, w1, w2, u[22], m[11]); + MONT_MAC(t0, t1, t2, u[23], m[10]); + MONT_MAC(v0, v1, v2, u[24], m[9]); + MONT_MAC(w0, w1, w2, u[25], m[8]); + MONT_MAC(t0, t1, t2, u[26], m[7]); + MONT_MAC(v0, v1, v2, u[27], m[6]); + MONT_MAC(w0, w1, w2, u[28], m[5]); + MONT_MAC(t0, t1, t2, u[29], m[4]); + MONT_MAC(v0, v1, v2, u[30], m[3]); + MONT_MAC(w0, w1, w2, u[31], m[2]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[1] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 34 */ + MONT_MUL(s0, s1, s2, a[3], a[31]); + MONT_MUL(c0, c1, c2, a[4], a[30]); + MONT_MAC(s0, s1, s2, a[5], a[29]); + MONT_MAC(c0, c1, c2, a[6], a[28]); + MONT_MAC(s0, s1, s2, a[7], a[27]); + MONT_MAC(c0, c1, c2, a[8], a[26]); + MONT_MAC(s0, s1, s2, a[9], a[25]); + MONT_MAC(c0, c1, c2, a[10], a[24]); + MONT_MAC(s0, s1, s2, a[11], a[23]); + MONT_MAC(c0, c1, c2, a[12], a[22]); + MONT_MAC(s0, s1, s2, a[13], a[21]); + MONT_MAC(c0, c1, c2, a[14], a[20]); + MONT_MAC(s0, s1, s2, a[15], a[19]); + MONT_MAC(c0, c1, c2, a[16], a[18]); + MONT_MAC(t0, t1, t2, a[17], a[17]); + MONT_MUL(v0, v1, v2, u[3], m[31]); + MONT_MUL(w0, w1, w2, u[4], m[30]); + MONT_MAC(t0, t1, t2, u[5], m[29]); + MONT_MAC(v0, v1, v2, u[6], m[28]); + MONT_MAC(w0, w1, w2, u[7], m[27]); + MONT_MAC(t0, t1, t2, u[8], m[26]); + MONT_MAC(v0, v1, v2, u[9], m[25]); + MONT_MAC(w0, w1, w2, u[10], m[24]); + MONT_MAC(t0, t1, t2, u[11], m[23]); + MONT_MAC(v0, v1, v2, u[12], m[22]); + MONT_MAC(w0, w1, w2, u[13], m[21]); + MONT_MAC(t0, t1, t2, u[14], m[20]); + MONT_MAC(v0, v1, v2, u[15], m[19]); + MONT_MAC(w0, w1, w2, u[16], m[18]); + MONT_MAC(t0, t1, t2, u[17], m[17]); + MONT_MAC(v0, v1, v2, u[18], m[16]); + MONT_MAC(w0, w1, w2, u[19], m[15]); + MONT_MAC(t0, t1, t2, u[20], m[14]); + MONT_MAC(v0, v1, v2, u[21], m[13]); + MONT_MAC(w0, w1, w2, u[22], m[12]); + MONT_MAC(t0, t1, t2, u[23], m[11]); + MONT_MAC(v0, v1, v2, u[24], m[10]); + MONT_MAC(w0, w1, w2, u[25], m[9]); + MONT_MAC(t0, t1, t2, u[26], m[8]); + MONT_MAC(v0, v1, v2, u[27], m[7]); + MONT_MAC(w0, w1, w2, u[28], m[6]); + MONT_MAC(t0, t1, t2, u[29], m[5]); + MONT_MAC(v0, v1, v2, u[30], m[4]); + MONT_MAC(w0, w1, w2, u[31], m[3]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[2] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 35 */ + MONT_MUL(s0, s1, s2, a[4], a[31]); + MONT_MUL(c0, c1, c2, a[5], a[30]); + MONT_MAC(s0, s1, s2, a[6], a[29]); + MONT_MAC(c0, c1, c2, a[7], a[28]); + MONT_MAC(s0, s1, s2, a[8], a[27]); + MONT_MAC(c0, c1, c2, a[9], a[26]); + MONT_MAC(s0, s1, s2, a[10], a[25]); + MONT_MAC(c0, c1, c2, a[11], a[24]); + MONT_MAC(s0, s1, s2, a[12], a[23]); + MONT_MAC(c0, c1, c2, a[13], a[22]); + MONT_MAC(s0, s1, s2, a[14], a[21]); + MONT_MAC(c0, c1, c2, a[15], a[20]); + MONT_MAC(s0, s1, s2, a[16], a[19]); + MONT_MAC(c0, c1, c2, a[17], a[18]); + MONT_MAC(t0, t1, t2, u[4], m[31]); + MONT_MUL(v0, v1, v2, u[5], m[30]); + MONT_MUL(w0, w1, w2, u[6], m[29]); + MONT_MAC(t0, t1, t2, u[7], m[28]); + MONT_MAC(v0, v1, v2, u[8], m[27]); + MONT_MAC(w0, w1, w2, u[9], m[26]); + MONT_MAC(t0, t1, t2, u[10], m[25]); + MONT_MAC(v0, v1, v2, u[11], m[24]); + MONT_MAC(w0, w1, w2, u[12], m[23]); + MONT_MAC(t0, t1, t2, u[13], m[22]); + MONT_MAC(v0, v1, v2, u[14], m[21]); + MONT_MAC(w0, w1, w2, u[15], m[20]); + MONT_MAC(t0, t1, t2, u[16], m[19]); + MONT_MAC(v0, v1, v2, u[17], m[18]); + MONT_MAC(w0, w1, w2, u[18], m[17]); + MONT_MAC(t0, t1, t2, u[19], m[16]); + MONT_MAC(v0, v1, v2, u[20], m[15]); + MONT_MAC(w0, w1, w2, u[21], m[14]); + MONT_MAC(t0, t1, t2, u[22], m[13]); + MONT_MAC(v0, v1, v2, u[23], m[12]); + MONT_MAC(w0, w1, w2, u[24], m[11]); + MONT_MAC(t0, t1, t2, u[25], m[10]); + MONT_MAC(v0, v1, v2, u[26], m[9]); + MONT_MAC(w0, w1, w2, u[27], m[8]); + MONT_MAC(t0, t1, t2, u[28], m[7]); + MONT_MAC(v0, v1, v2, u[29], m[6]); + MONT_MAC(w0, w1, w2, u[30], m[5]); + MONT_MAC(t0, t1, t2, u[31], m[4]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[3] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 36 */ + MONT_MUL(s0, s1, s2, a[5], a[31]); + MONT_MUL(c0, c1, c2, a[6], a[30]); + MONT_MAC(s0, s1, s2, a[7], a[29]); + MONT_MAC(c0, c1, c2, a[8], a[28]); + MONT_MAC(s0, s1, s2, a[9], a[27]); + MONT_MAC(c0, c1, c2, a[10], a[26]); + MONT_MAC(s0, s1, s2, a[11], a[25]); + MONT_MAC(c0, c1, c2, a[12], a[24]); + MONT_MAC(s0, s1, s2, a[13], a[23]); + MONT_MAC(c0, c1, c2, a[14], a[22]); + MONT_MAC(s0, s1, s2, a[15], a[21]); + MONT_MAC(c0, c1, c2, a[16], a[20]); + MONT_MAC(s0, s1, s2, a[17], a[19]); + MONT_MAC(t0, t1, t2, a[18], a[18]); + MONT_MUL(v0, v1, v2, u[5], m[31]); + MONT_MUL(w0, w1, w2, u[6], m[30]); + MONT_MAC(t0, t1, t2, u[7], m[29]); + MONT_MAC(v0, v1, v2, u[8], m[28]); + MONT_MAC(w0, w1, w2, u[9], m[27]); + MONT_MAC(t0, t1, t2, u[10], m[26]); + MONT_MAC(v0, v1, v2, u[11], m[25]); + MONT_MAC(w0, w1, w2, u[12], m[24]); + MONT_MAC(t0, t1, t2, u[13], m[23]); + MONT_MAC(v0, v1, v2, u[14], m[22]); + MONT_MAC(w0, w1, w2, u[15], m[21]); + MONT_MAC(t0, t1, t2, u[16], m[20]); + MONT_MAC(v0, v1, v2, u[17], m[19]); + MONT_MAC(w0, w1, w2, u[18], m[18]); + MONT_MAC(t0, t1, t2, u[19], m[17]); + MONT_MAC(v0, v1, v2, u[20], m[16]); + MONT_MAC(w0, w1, w2, u[21], m[15]); + MONT_MAC(t0, t1, t2, u[22], m[14]); + MONT_MAC(v0, v1, v2, u[23], m[13]); + MONT_MAC(w0, w1, w2, u[24], m[12]); + MONT_MAC(t0, t1, t2, u[25], m[11]); + MONT_MAC(v0, v1, v2, u[26], m[10]); + MONT_MAC(w0, w1, w2, u[27], m[9]); + MONT_MAC(t0, t1, t2, u[28], m[8]); + MONT_MAC(v0, v1, v2, u[29], m[7]); + MONT_MAC(w0, w1, w2, u[30], m[6]); + MONT_MAC(t0, t1, t2, u[31], m[5]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[4] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 37 */ + MONT_MUL(s0, s1, s2, a[6], a[31]); + MONT_MUL(c0, c1, c2, a[7], a[30]); + MONT_MAC(s0, s1, s2, a[8], a[29]); + MONT_MAC(c0, c1, c2, a[9], a[28]); + MONT_MAC(s0, s1, s2, a[10], a[27]); + MONT_MAC(c0, c1, c2, a[11], a[26]); + MONT_MAC(s0, s1, s2, a[12], a[25]); + MONT_MAC(c0, c1, c2, a[13], a[24]); + MONT_MAC(s0, s1, s2, a[14], a[23]); + MONT_MAC(c0, c1, c2, a[15], a[22]); + MONT_MAC(s0, s1, s2, a[16], a[21]); + MONT_MAC(c0, c1, c2, a[17], a[20]); + MONT_MAC(s0, s1, s2, a[18], a[19]); + MONT_MAC(t0, t1, t2, u[6], m[31]); + MONT_MUL(v0, v1, v2, u[7], m[30]); + MONT_MUL(w0, w1, w2, u[8], m[29]); + MONT_MAC(t0, t1, t2, u[9], m[28]); + MONT_MAC(v0, v1, v2, u[10], m[27]); + MONT_MAC(w0, w1, w2, u[11], m[26]); + MONT_MAC(t0, t1, t2, u[12], m[25]); + MONT_MAC(v0, v1, v2, u[13], m[24]); + MONT_MAC(w0, w1, w2, u[14], m[23]); + MONT_MAC(t0, t1, t2, u[15], m[22]); + MONT_MAC(v0, v1, v2, u[16], m[21]); + MONT_MAC(w0, w1, w2, u[17], m[20]); + MONT_MAC(t0, t1, t2, u[18], m[19]); + MONT_MAC(v0, v1, v2, u[19], m[18]); + MONT_MAC(w0, w1, w2, u[20], m[17]); + MONT_MAC(t0, t1, t2, u[21], m[16]); + MONT_MAC(v0, v1, v2, u[22], m[15]); + MONT_MAC(w0, w1, w2, u[23], m[14]); + MONT_MAC(t0, t1, t2, u[24], m[13]); + MONT_MAC(v0, v1, v2, u[25], m[12]); + MONT_MAC(w0, w1, w2, u[26], m[11]); + MONT_MAC(t0, t1, t2, u[27], m[10]); + MONT_MAC(v0, v1, v2, u[28], m[9]); + MONT_MAC(w0, w1, w2, u[29], m[8]); + MONT_MAC(t0, t1, t2, u[30], m[7]); + MONT_MAC(v0, v1, v2, u[31], m[6]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[5] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 38 */ + MONT_MUL(s0, s1, s2, a[7], a[31]); + MONT_MUL(c0, c1, c2, a[8], a[30]); + MONT_MAC(s0, s1, s2, a[9], a[29]); + MONT_MAC(c0, c1, c2, a[10], a[28]); + MONT_MAC(s0, s1, s2, a[11], a[27]); + MONT_MAC(c0, c1, c2, a[12], a[26]); + MONT_MAC(s0, s1, s2, a[13], a[25]); + MONT_MAC(c0, c1, c2, a[14], a[24]); + MONT_MAC(s0, s1, s2, a[15], a[23]); + MONT_MAC(c0, c1, c2, a[16], a[22]); + MONT_MAC(s0, s1, s2, a[17], a[21]); + MONT_MAC(c0, c1, c2, a[18], a[20]); + MONT_MAC(t0, t1, t2, a[19], a[19]); + MONT_MUL(v0, v1, v2, u[7], m[31]); + MONT_MUL(w0, w1, w2, u[8], m[30]); + MONT_MAC(t0, t1, t2, u[9], m[29]); + MONT_MAC(v0, v1, v2, u[10], m[28]); + MONT_MAC(w0, w1, w2, u[11], m[27]); + MONT_MAC(t0, t1, t2, u[12], m[26]); + MONT_MAC(v0, v1, v2, u[13], m[25]); + MONT_MAC(w0, w1, w2, u[14], m[24]); + MONT_MAC(t0, t1, t2, u[15], m[23]); + MONT_MAC(v0, v1, v2, u[16], m[22]); + MONT_MAC(w0, w1, w2, u[17], m[21]); + MONT_MAC(t0, t1, t2, u[18], m[20]); + MONT_MAC(v0, v1, v2, u[19], m[19]); + MONT_MAC(w0, w1, w2, u[20], m[18]); + MONT_MAC(t0, t1, t2, u[21], m[17]); + MONT_MAC(v0, v1, v2, u[22], m[16]); + MONT_MAC(w0, w1, w2, u[23], m[15]); + MONT_MAC(t0, t1, t2, u[24], m[14]); + MONT_MAC(v0, v1, v2, u[25], m[13]); + MONT_MAC(w0, w1, w2, u[26], m[12]); + MONT_MAC(t0, t1, t2, u[27], m[11]); + MONT_MAC(v0, v1, v2, u[28], m[10]); + MONT_MAC(w0, w1, w2, u[29], m[9]); + MONT_MAC(t0, t1, t2, u[30], m[8]); + MONT_MAC(v0, v1, v2, u[31], m[7]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[6] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 39 */ + MONT_MUL(s0, s1, s2, a[8], a[31]); + MONT_MUL(c0, c1, c2, a[9], a[30]); + MONT_MAC(s0, s1, s2, a[10], a[29]); + MONT_MAC(c0, c1, c2, a[11], a[28]); + MONT_MAC(s0, s1, s2, a[12], a[27]); + MONT_MAC(c0, c1, c2, a[13], a[26]); + MONT_MAC(s0, s1, s2, a[14], a[25]); + MONT_MAC(c0, c1, c2, a[15], a[24]); + MONT_MAC(s0, s1, s2, a[16], a[23]); + MONT_MAC(c0, c1, c2, a[17], a[22]); + MONT_MAC(s0, s1, s2, a[18], a[21]); + MONT_MAC(c0, c1, c2, a[19], a[20]); + MONT_MAC(t0, t1, t2, u[8], m[31]); + MONT_MUL(v0, v1, v2, u[9], m[30]); + MONT_MUL(w0, w1, w2, u[10], m[29]); + MONT_MAC(t0, t1, t2, u[11], m[28]); + MONT_MAC(v0, v1, v2, u[12], m[27]); + MONT_MAC(w0, w1, w2, u[13], m[26]); + MONT_MAC(t0, t1, t2, u[14], m[25]); + MONT_MAC(v0, v1, v2, u[15], m[24]); + MONT_MAC(w0, w1, w2, u[16], m[23]); + MONT_MAC(t0, t1, t2, u[17], m[22]); + MONT_MAC(v0, v1, v2, u[18], m[21]); + MONT_MAC(w0, w1, w2, u[19], m[20]); + MONT_MAC(t0, t1, t2, u[20], m[19]); + MONT_MAC(v0, v1, v2, u[21], m[18]); + MONT_MAC(w0, w1, w2, u[22], m[17]); + MONT_MAC(t0, t1, t2, u[23], m[16]); + MONT_MAC(v0, v1, v2, u[24], m[15]); + MONT_MAC(w0, w1, w2, u[25], m[14]); + MONT_MAC(t0, t1, t2, u[26], m[13]); + MONT_MAC(v0, v1, v2, u[27], m[12]); + MONT_MAC(w0, w1, w2, u[28], m[11]); + MONT_MAC(t0, t1, t2, u[29], m[10]); + MONT_MAC(v0, v1, v2, u[30], m[9]); + MONT_MAC(w0, w1, w2, u[31], m[8]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[7] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 40 */ + MONT_MUL(s0, s1, s2, a[9], a[31]); + MONT_MUL(c0, c1, c2, a[10], a[30]); + MONT_MAC(s0, s1, s2, a[11], a[29]); + MONT_MAC(c0, c1, c2, a[12], a[28]); + MONT_MAC(s0, s1, s2, a[13], a[27]); + MONT_MAC(c0, c1, c2, a[14], a[26]); + MONT_MAC(s0, s1, s2, a[15], a[25]); + MONT_MAC(c0, c1, c2, a[16], a[24]); + MONT_MAC(s0, s1, s2, a[17], a[23]); + MONT_MAC(c0, c1, c2, a[18], a[22]); + MONT_MAC(s0, s1, s2, a[19], a[21]); + MONT_MAC(t0, t1, t2, a[20], a[20]); + MONT_MUL(v0, v1, v2, u[9], m[31]); + MONT_MUL(w0, w1, w2, u[10], m[30]); + MONT_MAC(t0, t1, t2, u[11], m[29]); + MONT_MAC(v0, v1, v2, u[12], m[28]); + MONT_MAC(w0, w1, w2, u[13], m[27]); + MONT_MAC(t0, t1, t2, u[14], m[26]); + MONT_MAC(v0, v1, v2, u[15], m[25]); + MONT_MAC(w0, w1, w2, u[16], m[24]); + MONT_MAC(t0, t1, t2, u[17], m[23]); + MONT_MAC(v0, v1, v2, u[18], m[22]); + MONT_MAC(w0, w1, w2, u[19], m[21]); + MONT_MAC(t0, t1, t2, u[20], m[20]); + MONT_MAC(v0, v1, v2, u[21], m[19]); + MONT_MAC(w0, w1, w2, u[22], m[18]); + MONT_MAC(t0, t1, t2, u[23], m[17]); + MONT_MAC(v0, v1, v2, u[24], m[16]); + MONT_MAC(w0, w1, w2, u[25], m[15]); + MONT_MAC(t0, t1, t2, u[26], m[14]); + MONT_MAC(v0, v1, v2, u[27], m[13]); + MONT_MAC(w0, w1, w2, u[28], m[12]); + MONT_MAC(t0, t1, t2, u[29], m[11]); + MONT_MAC(v0, v1, v2, u[30], m[10]); + MONT_MAC(w0, w1, w2, u[31], m[9]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[8] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 41 */ + MONT_MUL(s0, s1, s2, a[10], a[31]); + MONT_MUL(c0, c1, c2, a[11], a[30]); + MONT_MAC(s0, s1, s2, a[12], a[29]); + MONT_MAC(c0, c1, c2, a[13], a[28]); + MONT_MAC(s0, s1, s2, a[14], a[27]); + MONT_MAC(c0, c1, c2, a[15], a[26]); + MONT_MAC(s0, s1, s2, a[16], a[25]); + MONT_MAC(c0, c1, c2, a[17], a[24]); + MONT_MAC(s0, s1, s2, a[18], a[23]); + MONT_MAC(c0, c1, c2, a[19], a[22]); + MONT_MAC(s0, s1, s2, a[20], a[21]); + MONT_MAC(t0, t1, t2, u[10], m[31]); + MONT_MUL(v0, v1, v2, u[11], m[30]); + MONT_MUL(w0, w1, w2, u[12], m[29]); + MONT_MAC(t0, t1, t2, u[13], m[28]); + MONT_MAC(v0, v1, v2, u[14], m[27]); + MONT_MAC(w0, w1, w2, u[15], m[26]); + MONT_MAC(t0, t1, t2, u[16], m[25]); + MONT_MAC(v0, v1, v2, u[17], m[24]); + MONT_MAC(w0, w1, w2, u[18], m[23]); + MONT_MAC(t0, t1, t2, u[19], m[22]); + MONT_MAC(v0, v1, v2, u[20], m[21]); + MONT_MAC(w0, w1, w2, u[21], m[20]); + MONT_MAC(t0, t1, t2, u[22], m[19]); + MONT_MAC(v0, v1, v2, u[23], m[18]); + MONT_MAC(w0, w1, w2, u[24], m[17]); + MONT_MAC(t0, t1, t2, u[25], m[16]); + MONT_MAC(v0, v1, v2, u[26], m[15]); + MONT_MAC(w0, w1, w2, u[27], m[14]); + MONT_MAC(t0, t1, t2, u[28], m[13]); + MONT_MAC(v0, v1, v2, u[29], m[12]); + MONT_MAC(w0, w1, w2, u[30], m[11]); + MONT_MAC(t0, t1, t2, u[31], m[10]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[9] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 42 */ + MONT_MUL(s0, s1, s2, a[11], a[31]); + MONT_MUL(c0, c1, c2, a[12], a[30]); + MONT_MAC(s0, s1, s2, a[13], a[29]); + MONT_MAC(c0, c1, c2, a[14], a[28]); + MONT_MAC(s0, s1, s2, a[15], a[27]); + MONT_MAC(c0, c1, c2, a[16], a[26]); + MONT_MAC(s0, s1, s2, a[17], a[25]); + MONT_MAC(c0, c1, c2, a[18], a[24]); + MONT_MAC(s0, s1, s2, a[19], a[23]); + MONT_MAC(c0, c1, c2, a[20], a[22]); + MONT_MAC(t0, t1, t2, a[21], a[21]); + MONT_MUL(v0, v1, v2, u[11], m[31]); + MONT_MUL(w0, w1, w2, u[12], m[30]); + MONT_MAC(t0, t1, t2, u[13], m[29]); + MONT_MAC(v0, v1, v2, u[14], m[28]); + MONT_MAC(w0, w1, w2, u[15], m[27]); + MONT_MAC(t0, t1, t2, u[16], m[26]); + MONT_MAC(v0, v1, v2, u[17], m[25]); + MONT_MAC(w0, w1, w2, u[18], m[24]); + MONT_MAC(t0, t1, t2, u[19], m[23]); + MONT_MAC(v0, v1, v2, u[20], m[22]); + MONT_MAC(w0, w1, w2, u[21], m[21]); + MONT_MAC(t0, t1, t2, u[22], m[20]); + MONT_MAC(v0, v1, v2, u[23], m[19]); + MONT_MAC(w0, w1, w2, u[24], m[18]); + MONT_MAC(t0, t1, t2, u[25], m[17]); + MONT_MAC(v0, v1, v2, u[26], m[16]); + MONT_MAC(w0, w1, w2, u[27], m[15]); + MONT_MAC(t0, t1, t2, u[28], m[14]); + MONT_MAC(v0, v1, v2, u[29], m[13]); + MONT_MAC(w0, w1, w2, u[30], m[12]); + MONT_MAC(t0, t1, t2, u[31], m[11]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[10] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 43 */ + MONT_MUL(s0, s1, s2, a[12], a[31]); + MONT_MUL(c0, c1, c2, a[13], a[30]); + MONT_MAC(s0, s1, s2, a[14], a[29]); + MONT_MAC(c0, c1, c2, a[15], a[28]); + MONT_MAC(s0, s1, s2, a[16], a[27]); + MONT_MAC(c0, c1, c2, a[17], a[26]); + MONT_MAC(s0, s1, s2, a[18], a[25]); + MONT_MAC(c0, c1, c2, a[19], a[24]); + MONT_MAC(s0, s1, s2, a[20], a[23]); + MONT_MAC(c0, c1, c2, a[21], a[22]); + MONT_MAC(t0, t1, t2, u[12], m[31]); + MONT_MUL(v0, v1, v2, u[13], m[30]); + MONT_MUL(w0, w1, w2, u[14], m[29]); + MONT_MAC(t0, t1, t2, u[15], m[28]); + MONT_MAC(v0, v1, v2, u[16], m[27]); + MONT_MAC(w0, w1, w2, u[17], m[26]); + MONT_MAC(t0, t1, t2, u[18], m[25]); + MONT_MAC(v0, v1, v2, u[19], m[24]); + MONT_MAC(w0, w1, w2, u[20], m[23]); + MONT_MAC(t0, t1, t2, u[21], m[22]); + MONT_MAC(v0, v1, v2, u[22], m[21]); + MONT_MAC(w0, w1, w2, u[23], m[20]); + MONT_MAC(t0, t1, t2, u[24], m[19]); + MONT_MAC(v0, v1, v2, u[25], m[18]); + MONT_MAC(w0, w1, w2, u[26], m[17]); + MONT_MAC(t0, t1, t2, u[27], m[16]); + MONT_MAC(v0, v1, v2, u[28], m[15]); + MONT_MAC(w0, w1, w2, u[29], m[14]); + MONT_MAC(t0, t1, t2, u[30], m[13]); + MONT_MAC(v0, v1, v2, u[31], m[12]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[11] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 44 */ + MONT_MUL(s0, s1, s2, a[13], a[31]); + MONT_MUL(c0, c1, c2, a[14], a[30]); + MONT_MAC(s0, s1, s2, a[15], a[29]); + MONT_MAC(c0, c1, c2, a[16], a[28]); + MONT_MAC(s0, s1, s2, a[17], a[27]); + MONT_MAC(c0, c1, c2, a[18], a[26]); + MONT_MAC(s0, s1, s2, a[19], a[25]); + MONT_MAC(c0, c1, c2, a[20], a[24]); + MONT_MAC(s0, s1, s2, a[21], a[23]); + MONT_MAC(t0, t1, t2, a[22], a[22]); + MONT_MUL(v0, v1, v2, u[13], m[31]); + MONT_MUL(w0, w1, w2, u[14], m[30]); + MONT_MAC(t0, t1, t2, u[15], m[29]); + MONT_MAC(v0, v1, v2, u[16], m[28]); + MONT_MAC(w0, w1, w2, u[17], m[27]); + MONT_MAC(t0, t1, t2, u[18], m[26]); + MONT_MAC(v0, v1, v2, u[19], m[25]); + MONT_MAC(w0, w1, w2, u[20], m[24]); + MONT_MAC(t0, t1, t2, u[21], m[23]); + MONT_MAC(v0, v1, v2, u[22], m[22]); + MONT_MAC(w0, w1, w2, u[23], m[21]); + MONT_MAC(t0, t1, t2, u[24], m[20]); + MONT_MAC(v0, v1, v2, u[25], m[19]); + MONT_MAC(w0, w1, w2, u[26], m[18]); + MONT_MAC(t0, t1, t2, u[27], m[17]); + MONT_MAC(v0, v1, v2, u[28], m[16]); + MONT_MAC(w0, w1, w2, u[29], m[15]); + MONT_MAC(t0, t1, t2, u[30], m[14]); + MONT_MAC(v0, v1, v2, u[31], m[13]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[12] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 45 */ + MONT_MUL(s0, s1, s2, a[14], a[31]); + MONT_MUL(c0, c1, c2, a[15], a[30]); + MONT_MAC(s0, s1, s2, a[16], a[29]); + MONT_MAC(c0, c1, c2, a[17], a[28]); + MONT_MAC(s0, s1, s2, a[18], a[27]); + MONT_MAC(c0, c1, c2, a[19], a[26]); + MONT_MAC(s0, s1, s2, a[20], a[25]); + MONT_MAC(c0, c1, c2, a[21], a[24]); + MONT_MAC(s0, s1, s2, a[22], a[23]); + MONT_MAC(t0, t1, t2, u[14], m[31]); + MONT_MUL(v0, v1, v2, u[15], m[30]); + MONT_MUL(w0, w1, w2, u[16], m[29]); + MONT_MAC(t0, t1, t2, u[17], m[28]); + MONT_MAC(v0, v1, v2, u[18], m[27]); + MONT_MAC(w0, w1, w2, u[19], m[26]); + MONT_MAC(t0, t1, t2, u[20], m[25]); + MONT_MAC(v0, v1, v2, u[21], m[24]); + MONT_MAC(w0, w1, w2, u[22], m[23]); + MONT_MAC(t0, t1, t2, u[23], m[22]); + MONT_MAC(v0, v1, v2, u[24], m[21]); + MONT_MAC(w0, w1, w2, u[25], m[20]); + MONT_MAC(t0, t1, t2, u[26], m[19]); + MONT_MAC(v0, v1, v2, u[27], m[18]); + MONT_MAC(w0, w1, w2, u[28], m[17]); + MONT_MAC(t0, t1, t2, u[29], m[16]); + MONT_MAC(v0, v1, v2, u[30], m[15]); + MONT_MAC(w0, w1, w2, u[31], m[14]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[13] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 46 */ + MONT_MUL(s0, s1, s2, a[15], a[31]); + MONT_MUL(c0, c1, c2, a[16], a[30]); + MONT_MAC(s0, s1, s2, a[17], a[29]); + MONT_MAC(c0, c1, c2, a[18], a[28]); + MONT_MAC(s0, s1, s2, a[19], a[27]); + MONT_MAC(c0, c1, c2, a[20], a[26]); + MONT_MAC(s0, s1, s2, a[21], a[25]); + MONT_MAC(c0, c1, c2, a[22], a[24]); + MONT_MAC(t0, t1, t2, a[23], a[23]); + MONT_MUL(v0, v1, v2, u[15], m[31]); + MONT_MUL(w0, w1, w2, u[16], m[30]); + MONT_MAC(t0, t1, t2, u[17], m[29]); + MONT_MAC(v0, v1, v2, u[18], m[28]); + MONT_MAC(w0, w1, w2, u[19], m[27]); + MONT_MAC(t0, t1, t2, u[20], m[26]); + MONT_MAC(v0, v1, v2, u[21], m[25]); + MONT_MAC(w0, w1, w2, u[22], m[24]); + MONT_MAC(t0, t1, t2, u[23], m[23]); + MONT_MAC(v0, v1, v2, u[24], m[22]); + MONT_MAC(w0, w1, w2, u[25], m[21]); + MONT_MAC(t0, t1, t2, u[26], m[20]); + MONT_MAC(v0, v1, v2, u[27], m[19]); + MONT_MAC(w0, w1, w2, u[28], m[18]); + MONT_MAC(t0, t1, t2, u[29], m[17]); + MONT_MAC(v0, v1, v2, u[30], m[16]); + MONT_MAC(w0, w1, w2, u[31], m[15]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[14] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 47 */ + MONT_MUL(s0, s1, s2, a[16], a[31]); + MONT_MUL(c0, c1, c2, a[17], a[30]); + MONT_MAC(s0, s1, s2, a[18], a[29]); + MONT_MAC(c0, c1, c2, a[19], a[28]); + MONT_MAC(s0, s1, s2, a[20], a[27]); + MONT_MAC(c0, c1, c2, a[21], a[26]); + MONT_MAC(s0, s1, s2, a[22], a[25]); + MONT_MAC(c0, c1, c2, a[23], a[24]); + MONT_MAC(t0, t1, t2, u[16], m[31]); + MONT_MUL(v0, v1, v2, u[17], m[30]); + MONT_MUL(w0, w1, w2, u[18], m[29]); + MONT_MAC(t0, t1, t2, u[19], m[28]); + MONT_MAC(v0, v1, v2, u[20], m[27]); + MONT_MAC(w0, w1, w2, u[21], m[26]); + MONT_MAC(t0, t1, t2, u[22], m[25]); + MONT_MAC(v0, v1, v2, u[23], m[24]); + MONT_MAC(w0, w1, w2, u[24], m[23]); + MONT_MAC(t0, t1, t2, u[25], m[22]); + MONT_MAC(v0, v1, v2, u[26], m[21]); + MONT_MAC(w0, w1, w2, u[27], m[20]); + MONT_MAC(t0, t1, t2, u[28], m[19]); + MONT_MAC(v0, v1, v2, u[29], m[18]); + MONT_MAC(w0, w1, w2, u[30], m[17]); + MONT_MAC(t0, t1, t2, u[31], m[16]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[15] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 48 */ + MONT_MUL(s0, s1, s2, a[17], a[31]); + MONT_MUL(c0, c1, c2, a[18], a[30]); + MONT_MAC(s0, s1, s2, a[19], a[29]); + MONT_MAC(c0, c1, c2, a[20], a[28]); + MONT_MAC(s0, s1, s2, a[21], a[27]); + MONT_MAC(c0, c1, c2, a[22], a[26]); + MONT_MAC(s0, s1, s2, a[23], a[25]); + MONT_MAC(t0, t1, t2, a[24], a[24]); + MONT_MUL(v0, v1, v2, u[17], m[31]); + MONT_MUL(w0, w1, w2, u[18], m[30]); + MONT_MAC(t0, t1, t2, u[19], m[29]); + MONT_MAC(v0, v1, v2, u[20], m[28]); + MONT_MAC(w0, w1, w2, u[21], m[27]); + MONT_MAC(t0, t1, t2, u[22], m[26]); + MONT_MAC(v0, v1, v2, u[23], m[25]); + MONT_MAC(w0, w1, w2, u[24], m[24]); + MONT_MAC(t0, t1, t2, u[25], m[23]); + MONT_MAC(v0, v1, v2, u[26], m[22]); + MONT_MAC(w0, w1, w2, u[27], m[21]); + MONT_MAC(t0, t1, t2, u[28], m[20]); + MONT_MAC(v0, v1, v2, u[29], m[19]); + MONT_MAC(w0, w1, w2, u[30], m[18]); + MONT_MAC(t0, t1, t2, u[31], m[17]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[16] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 49 */ + MONT_MUL(s0, s1, s2, a[18], a[31]); + MONT_MUL(c0, c1, c2, a[19], a[30]); + MONT_MAC(s0, s1, s2, a[20], a[29]); + MONT_MAC(c0, c1, c2, a[21], a[28]); + MONT_MAC(s0, s1, s2, a[22], a[27]); + MONT_MAC(c0, c1, c2, a[23], a[26]); + MONT_MAC(s0, s1, s2, a[24], a[25]); + MONT_MAC(t0, t1, t2, u[18], m[31]); + MONT_MUL(v0, v1, v2, u[19], m[30]); + MONT_MUL(w0, w1, w2, u[20], m[29]); + MONT_MAC(t0, t1, t2, u[21], m[28]); + MONT_MAC(v0, v1, v2, u[22], m[27]); + MONT_MAC(w0, w1, w2, u[23], m[26]); + MONT_MAC(t0, t1, t2, u[24], m[25]); + MONT_MAC(v0, v1, v2, u[25], m[24]); + MONT_MAC(w0, w1, w2, u[26], m[23]); + MONT_MAC(t0, t1, t2, u[27], m[22]); + MONT_MAC(v0, v1, v2, u[28], m[21]); + MONT_MAC(w0, w1, w2, u[29], m[20]); + MONT_MAC(t0, t1, t2, u[30], m[19]); + MONT_MAC(v0, v1, v2, u[31], m[18]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[17] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 50 */ + MONT_MUL(s0, s1, s2, a[19], a[31]); + MONT_MUL(c0, c1, c2, a[20], a[30]); + MONT_MAC(s0, s1, s2, a[21], a[29]); + MONT_MAC(c0, c1, c2, a[22], a[28]); + MONT_MAC(s0, s1, s2, a[23], a[27]); + MONT_MAC(c0, c1, c2, a[24], a[26]); + MONT_MAC(t0, t1, t2, a[25], a[25]); + MONT_MUL(v0, v1, v2, u[19], m[31]); + MONT_MUL(w0, w1, w2, u[20], m[30]); + MONT_MAC(t0, t1, t2, u[21], m[29]); + MONT_MAC(v0, v1, v2, u[22], m[28]); + MONT_MAC(w0, w1, w2, u[23], m[27]); + MONT_MAC(t0, t1, t2, u[24], m[26]); + MONT_MAC(v0, v1, v2, u[25], m[25]); + MONT_MAC(w0, w1, w2, u[26], m[24]); + MONT_MAC(t0, t1, t2, u[27], m[23]); + MONT_MAC(v0, v1, v2, u[28], m[22]); + MONT_MAC(w0, w1, w2, u[29], m[21]); + MONT_MAC(t0, t1, t2, u[30], m[20]); + MONT_MAC(v0, v1, v2, u[31], m[19]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[18] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 51 */ + MONT_MUL(s0, s1, s2, a[20], a[31]); + MONT_MUL(c0, c1, c2, a[21], a[30]); + MONT_MAC(s0, s1, s2, a[22], a[29]); + MONT_MAC(c0, c1, c2, a[23], a[28]); + MONT_MAC(s0, s1, s2, a[24], a[27]); + MONT_MAC(c0, c1, c2, a[25], a[26]); + MONT_MAC(t0, t1, t2, u[20], m[31]); + MONT_MUL(v0, v1, v2, u[21], m[30]); + MONT_MUL(w0, w1, w2, u[22], m[29]); + MONT_MAC(t0, t1, t2, u[23], m[28]); + MONT_MAC(v0, v1, v2, u[24], m[27]); + MONT_MAC(w0, w1, w2, u[25], m[26]); + MONT_MAC(t0, t1, t2, u[26], m[25]); + MONT_MAC(v0, v1, v2, u[27], m[24]); + MONT_MAC(w0, w1, w2, u[28], m[23]); + MONT_MAC(t0, t1, t2, u[29], m[22]); + MONT_MAC(v0, v1, v2, u[30], m[21]); + MONT_MAC(w0, w1, w2, u[31], m[20]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[19] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 52 */ + MONT_MUL(s0, s1, s2, a[21], a[31]); + MONT_MUL(c0, c1, c2, a[22], a[30]); + MONT_MAC(s0, s1, s2, a[23], a[29]); + MONT_MAC(c0, c1, c2, a[24], a[28]); + MONT_MAC(s0, s1, s2, a[25], a[27]); + MONT_MAC(t0, t1, t2, a[26], a[26]); + MONT_MUL(v0, v1, v2, u[21], m[31]); + MONT_MUL(w0, w1, w2, u[22], m[30]); + MONT_MAC(t0, t1, t2, u[23], m[29]); + MONT_MAC(v0, v1, v2, u[24], m[28]); + MONT_MAC(w0, w1, w2, u[25], m[27]); + MONT_MAC(t0, t1, t2, u[26], m[26]); + MONT_MAC(v0, v1, v2, u[27], m[25]); + MONT_MAC(w0, w1, w2, u[28], m[24]); + MONT_MAC(t0, t1, t2, u[29], m[23]); + MONT_MAC(v0, v1, v2, u[30], m[22]); + MONT_MAC(w0, w1, w2, u[31], m[21]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[20] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 53 */ + MONT_MUL(s0, s1, s2, a[22], a[31]); + MONT_MUL(c0, c1, c2, a[23], a[30]); + MONT_MAC(s0, s1, s2, a[24], a[29]); + MONT_MAC(c0, c1, c2, a[25], a[28]); + MONT_MAC(s0, s1, s2, a[26], a[27]); + MONT_MAC(t0, t1, t2, u[22], m[31]); + MONT_MUL(v0, v1, v2, u[23], m[30]); + MONT_MUL(w0, w1, w2, u[24], m[29]); + MONT_MAC(t0, t1, t2, u[25], m[28]); + MONT_MAC(v0, v1, v2, u[26], m[27]); + MONT_MAC(w0, w1, w2, u[27], m[26]); + MONT_MAC(t0, t1, t2, u[28], m[25]); + MONT_MAC(v0, v1, v2, u[29], m[24]); + MONT_MAC(w0, w1, w2, u[30], m[23]); + MONT_MAC(t0, t1, t2, u[31], m[22]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[21] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 54 */ + MONT_MUL(s0, s1, s2, a[23], a[31]); + MONT_MUL(c0, c1, c2, a[24], a[30]); + MONT_MAC(s0, s1, s2, a[25], a[29]); + MONT_MAC(c0, c1, c2, a[26], a[28]); + MONT_MAC(t0, t1, t2, a[27], a[27]); + MONT_MUL(v0, v1, v2, u[23], m[31]); + MONT_MUL(w0, w1, w2, u[24], m[30]); + MONT_MAC(t0, t1, t2, u[25], m[29]); + MONT_MAC(v0, v1, v2, u[26], m[28]); + MONT_MAC(w0, w1, w2, u[27], m[27]); + MONT_MAC(t0, t1, t2, u[28], m[26]); + MONT_MAC(v0, v1, v2, u[29], m[25]); + MONT_MAC(w0, w1, w2, u[30], m[24]); + MONT_MAC(t0, t1, t2, u[31], m[23]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[22] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 55 */ + MONT_MUL(s0, s1, s2, a[24], a[31]); + MONT_MUL(c0, c1, c2, a[25], a[30]); + MONT_MAC(s0, s1, s2, a[26], a[29]); + MONT_MAC(c0, c1, c2, a[27], a[28]); + MONT_MAC(t0, t1, t2, u[24], m[31]); + MONT_MUL(v0, v1, v2, u[25], m[30]); + MONT_MUL(w0, w1, w2, u[26], m[29]); + MONT_MAC(t0, t1, t2, u[27], m[28]); + MONT_MAC(v0, v1, v2, u[28], m[27]); + MONT_MAC(w0, w1, w2, u[29], m[26]); + MONT_MAC(t0, t1, t2, u[30], m[25]); + MONT_MAC(v0, v1, v2, u[31], m[24]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[23] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 56 */ + MONT_MUL(s0, s1, s2, a[25], a[31]); + MONT_MUL(c0, c1, c2, a[26], a[30]); + MONT_MAC(s0, s1, s2, a[27], a[29]); + MONT_MAC(t0, t1, t2, a[28], a[28]); + MONT_MUL(v0, v1, v2, u[25], m[31]); + MONT_MUL(w0, w1, w2, u[26], m[30]); + MONT_MAC(t0, t1, t2, u[27], m[29]); + MONT_MAC(v0, v1, v2, u[28], m[28]); + MONT_MAC(w0, w1, w2, u[29], m[27]); + MONT_MAC(t0, t1, t2, u[30], m[26]); + MONT_MAC(v0, v1, v2, u[31], m[25]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[24] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 57 */ + MONT_MUL(s0, s1, s2, a[26], a[31]); + MONT_MUL(c0, c1, c2, a[27], a[30]); + MONT_MAC(s0, s1, s2, a[28], a[29]); + MONT_MAC(t0, t1, t2, u[26], m[31]); + MONT_MUL(v0, v1, v2, u[27], m[30]); + MONT_MUL(w0, w1, w2, u[28], m[29]); + MONT_MAC(t0, t1, t2, u[29], m[28]); + MONT_MAC(v0, v1, v2, u[30], m[27]); + MONT_MAC(w0, w1, w2, u[31], m[26]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[25] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 58 */ + MONT_MUL(s0, s1, s2, a[27], a[31]); + MONT_MUL(c0, c1, c2, a[28], a[30]); + MONT_MAC(t0, t1, t2, a[29], a[29]); + MONT_MUL(v0, v1, v2, u[27], m[31]); + MONT_MUL(w0, w1, w2, u[28], m[30]); + MONT_MAC(t0, t1, t2, u[29], m[29]); + MONT_MAC(v0, v1, v2, u[30], m[28]); + MONT_MAC(w0, w1, w2, u[31], m[27]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[26] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 59 */ + MONT_MUL(s0, s1, s2, a[28], a[31]); + MONT_MUL(c0, c1, c2, a[29], a[30]); + MONT_MAC(t0, t1, t2, u[28], m[31]); + MONT_MUL(v0, v1, v2, u[29], m[30]); + MONT_MUL(w0, w1, w2, u[30], m[29]); + MONT_MAC(t0, t1, t2, u[31], m[28]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_ADD(s0, s1, s2, c0, c1, c2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[27] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 60 */ + MONT_MUL(s0, s1, s2, a[29], a[31]); + MONT_MAC(t0, t1, t2, a[30], a[30]); + MONT_MUL(v0, v1, v2, u[29], m[31]); + MONT_MUL(w0, w1, w2, u[30], m[30]); + MONT_MAC(t0, t1, t2, u[31], m[29]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_ADD(t0, t1, t2, w0, w1, w2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[28] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 61 */ + MONT_MUL(s0, s1, s2, a[30], a[31]); + MONT_MAC(t0, t1, t2, u[30], m[31]); + MONT_MUL(v0, v1, v2, u[31], m[30]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + MONT_DBL(s0, s1, s2); + MONT_ADD(t0, t1, t2, s0, s1, s2); + z[29] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + /* column 62 */ + MONT_MAC(t0, t1, t2, a[31], a[31]); + MONT_MUL(v0, v1, v2, u[31], m[31]); + MONT_ADD(t0, t1, t2, v0, v1, v2); + z[30] = t0; + t0 = t1; + t1 = t2; + t2 = 0; + z[31] = t0; + + /* under 2m, with t1 the bit above n limbs: at most one subtraction */ + borrow = sub_n(r, z, m, 32); + take = t1 | (borrow ^ 1); + select_n(r, r, z, take & 1, 32); +} + +#endif diff --git a/cbits/crypton_mont.py b/cbits/crypton_mont.py new file mode 100644 --- /dev/null +++ b/cbits/crypton_mont.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Generate cbits/crypton_mont.h: Montgomery multiplication and squaring for +AArch64 at the limb counts of RSA's CRT halves, fully unrolled and +product-scanning. + +Usage: python3 cbits/crypton_mont.py > cbits/crypton_mont.h + +The result is a column at a time, with the reduction folded into the same +scan (the FIPS method), so the running sum stays in registers instead of +going through memory a row at a time. A column's products are spread over +three accumulators and added together at its end, since one accumulator's +carry chain is what holds the core back, not its multipliers. In a square, +each product off the diagonal is summed once, doubled, and added in. +""" + +SIZES = (16, 24, 32) + +HEADER = """\ +/* Generated by cbits/crypton_mont.py; do not edit. + * + * Montgomery multiplication and squaring for AArch64 at 16, 24 and 32 limbs, + * the halves of RSA-2048, -3072 and -4096. Each answers what mont_mul and + * mont_sqr answer. */ +#ifndef CRYPTON_MONT_H +#define CRYPTON_MONT_H + +#include "crypton_bignum.h" + +/* (t2:t1:t0) += x * y */ +#define MONT_MAC(t0, t1, t2, x, y) \\ + do { \\ + limb_t lo_, hi_; \\ + __asm__("mul\\t%0, %5, %6\\n\\t" \\ + "umulh\\t%1, %5, %6\\n\\t" \\ + "adds\\t%2, %2, %0\\n\\t" \\ + "adcs\\t%3, %3, %1\\n\\t" \\ + "adc\\t%4, %4, xzr" \\ + : "=&r"(lo_), "=&r"(hi_), "+r"(t0), "+r"(t1), "+r"(t2) \\ + : "r"(x), "r"(y) \\ + : "cc"); \\ + } while (0) + +/* (s2:s1:s0) = x * y */ +#define MONT_MUL(s0, s1, s2, x, y) \\ + __asm__("mul\\t%0, %3, %4\\n\\t" \\ + "umulh\\t%1, %3, %4\\n\\t" \\ + "mov\\t%2, xzr" \\ + : "=&r"(s0), "=&r"(s1), "=r"(s2) \\ + : "r"(x), "r"(y)) + +/* (t2:t1:t0) += (s2:s1:s0) */ +#define MONT_ADD(t0, t1, t2, s0, s1, s2) \\ + __asm__("adds\\t%0, %0, %3\\n\\t" \\ + "adcs\\t%1, %1, %4\\n\\t" \\ + "adc\\t%2, %2, %5" \\ + : "+r"(t0), "+r"(t1), "+r"(t2) \\ + : "r"(s0), "r"(s1), "r"(s2) \\ + : "cc") + +/* (s2:s1:s0) *= 2 */ +#define MONT_DBL(s0, s1, s2) \\ + do { \\ + s2 = (s2 << 1) | (s1 >> 63); \\ + s1 = (s1 << 1) | (s0 >> 63); \\ + s0 <<= 1; \\ + } while (0) +""" + +FOOTER = """\ +#endif +""" + + +class Column: + """Products added to accumulators, each set up by its first product.""" + + def __init__(self, w): + self.w = w + self.started = set() + + def add(self, acc, x, y): + if acc == "t" or acc in self.started: + self.w(f"\tMONT_MAC({acc}0, {acc}1, {acc}2, {x}, {y});") + else: + self.w(f"\tMONT_MUL({acc}0, {acc}1, {acc}2, {x}, {y});") + self.started.add(acc) + + def fold(self, into, acc): + if acc in self.started: + self.w(f"\tMONT_ADD({into}0, {into}1, {into}2, {acc}0, {acc}1, {acc}2);") + + +def column(w, singles, cross): + """singles are added once, cross twice: s and c take the cross products, + t, v and w take the rest, and all of them end up in t.""" + col = Column(w) + for k, (x, y) in enumerate(cross): + col.add("s" if k % 2 == 0 else "c", x, y) + for k, (x, y) in enumerate(singles): + col.add("tvw"[k % 3], x, y) + col.fold("t", "v") + col.fold("t", "w") + if cross: + col.fold("s", "c") + w("\tMONT_DBL(s0, s1, s2);") + w("\tMONT_ADD(t0, t1, t2, s0, s1, s2);") + + +def gen(n, square): + out = [] + w = out.append + name = f"crypton_mont_{'sqr' if square else 'mul'}{n}" + args = "const limb_t *a," if square else "const limb_t *a, const limb_t *b," + if square: + w(f"/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */") + else: + w(f"/* r = a * b * R^-1 mod m, for a and b under m; t is 2n limbs of scratch */") + w(f"static void {name}(limb_t *r, {args}") + w(f"{' ' * len(f'static void {name}(')}const limb_t *m, limb_t n0, limb_t *t)") + w("{") + w(f"\tlimb_t *u = t, *z = t + {n};") + w("\tlimb_t t0 = 0, t1 = 0, t2 = 0, v0, v1, v2, w0, w1, w2, borrow, take;") + if square: + w("\tlimb_t s0, s1, s2, c0, c1, c2;") + w("") + for i in range(2 * n - 1): + w(f"\t/* column {i} */") + singles, cross = [], [] + for j in range(max(0, i - n + 1), min(i, n - 1) + 1): + k = i - j + if not square: + singles.append((f"a[{j}]", f"b[{k}]")) + elif j < k: + cross.append((f"a[{j}]", f"a[{k}]")) + elif j == k: + singles.append((f"a[{j}]", f"a[{j}]")) + for j in range(max(0, i - n + 1), min(i, n)): + singles.append((f"u[{j}]", f"m[{i - j}]")) + column(w, singles, cross) + if i < n: + w(f"\tu[{i}] = t0 * n0;") + w(f"\tMONT_MAC(t0, t1, t2, u[{i}], m[0]);") + else: + w(f"\tz[{i - n}] = t0;") + w("\tt0 = t1;") + w("\tt1 = t2;") + w("\tt2 = 0;") + w(f"\tz[{n - 1}] = t0;") + w("") + w("\t/* under 2m, with t1 the bit above n limbs: at most one subtraction */") + w(f"\tborrow = sub_n(r, z, m, {n});") + w("\ttake = t1 | (borrow ^ 1);") + w(f"\tselect_n(r, r, z, take & 1, {n});") + w("}") + w("") + return "\n".join(out) + + +def main(): + print(HEADER) + for n in SIZES: + print(gen(n, False)) + print(gen(n, True)) + print(FOOTER, end="") + + +if __name__ == "__main__": + main() diff --git a/cbits/crypton_mont_sqr.h b/cbits/crypton_mont_sqr.h deleted file mode 100644 --- a/cbits/crypton_mont_sqr.h +++ /dev/null @@ -1,3751 +0,0 @@ -/* Generated by cbits/crypton_mont_sqr.py; do not edit. - * - * Montgomery squaring for AArch64 at 16, 24 and 32 limbs, the halves of - * RSA-2048, -3072 and -4096. Each answers what mont_sqr answers. */ -#ifndef CRYPTON_MONT_SQR_H -#define CRYPTON_MONT_SQR_H - -#include "crypton_bignum.h" - -/* (t2:t1:t0) += x * y */ -#define MONT_SQR_MAC(t0, t1, t2, x, y) \ - do { \ - limb_t lo_, hi_; \ - __asm__("mul\t%0, %5, %6\n\t" \ - "umulh\t%1, %5, %6\n\t" \ - "adds\t%2, %2, %0\n\t" \ - "adcs\t%3, %3, %1\n\t" \ - "adc\t%4, %4, xzr" \ - : "=&r"(lo_), "=&r"(hi_), "+r"(t0), "+r"(t1), "+r"(t2) \ - : "r"(x), "r"(y) \ - : "cc"); \ - } while (0) - -/* (s2:s1:s0) = x * y */ -#define MONT_SQR_MUL(s0, s1, s2, x, y) \ - __asm__("mul\t%0, %3, %4\n\t" \ - "umulh\t%1, %3, %4\n\t" \ - "mov\t%2, xzr" \ - : "=&r"(s0), "=&r"(s1), "=r"(s2) \ - : "r"(x), "r"(y)) - -/* (t2:t1:t0) += 2 * (s2:s1:s0) */ -#define MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2) \ - do { \ - s2 = (s2 << 1) | (s1 >> 63); \ - s1 = (s1 << 1) | (s0 >> 63); \ - s0 <<= 1; \ - __asm__("adds\t%0, %0, %3\n\t" \ - "adcs\t%1, %1, %4\n\t" \ - "adc\t%2, %2, %5" \ - : "+r"(t0), "+r"(t1), "+r"(t2) \ - : "r"(s0), "r"(s1), "r"(s2) \ - : "cc"); \ - } while (0) - -/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ -static void crypton_mont_sqr16(limb_t *r, const limb_t *a, const limb_t *m, - limb_t n0, limb_t *t) -{ - limb_t *u = t, *z = t + 16; - limb_t t0 = 0, t1 = 0, t2 = 0, s0, s1, s2, borrow, take; - - /* column 0 */ - MONT_SQR_MAC(t0, t1, t2, a[0], a[0]); - u[0] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[0], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 1 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[1]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[1]); - u[1] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[1], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 2 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[1], a[1]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[1]); - u[2] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[2], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 3 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[3]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[1]); - u[3] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[3], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 4 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[2], a[2]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[1]); - u[4] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[4], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 5 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[1]); - u[5] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[5], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 6 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[3], a[3]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[1]); - u[6] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[6], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 7 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[1]); - u[7] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[7], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 8 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[4], a[4]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[1]); - u[8] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[8], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 9 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[1]); - u[9] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[9], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 10 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[5], a[5]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[1]); - u[10] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[10], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 11 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[1]); - u[11] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[11], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 12 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[6], a[6]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[1]); - u[12] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[12], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 13 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[1]); - u[13] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[13], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 14 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[7], a[7]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[1]); - u[14] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[14], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 15 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[1]); - u[15] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[15], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 16 */ - MONT_SQR_MUL(s0, s1, s2, a[1], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[8], a[8]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[1]); - z[0] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 17 */ - MONT_SQR_MUL(s0, s1, s2, a[2], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[2], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[2]); - z[1] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 18 */ - MONT_SQR_MUL(s0, s1, s2, a[3], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[9], a[9]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[3]); - z[2] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 19 */ - MONT_SQR_MUL(s0, s1, s2, a[4], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[4], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[4]); - z[3] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 20 */ - MONT_SQR_MUL(s0, s1, s2, a[5], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[10], a[10]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[5]); - z[4] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 21 */ - MONT_SQR_MUL(s0, s1, s2, a[6], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[6], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[6]); - z[5] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 22 */ - MONT_SQR_MUL(s0, s1, s2, a[7], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[11], a[11]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[7]); - z[6] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 23 */ - MONT_SQR_MUL(s0, s1, s2, a[8], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[8], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[8]); - z[7] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 24 */ - MONT_SQR_MUL(s0, s1, s2, a[9], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[12], a[12]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[9]); - z[8] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 25 */ - MONT_SQR_MUL(s0, s1, s2, a[10], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[10], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[10]); - z[9] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 26 */ - MONT_SQR_MUL(s0, s1, s2, a[11], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[13], a[13]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[11]); - z[10] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 27 */ - MONT_SQR_MUL(s0, s1, s2, a[12], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[12], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[12]); - z[11] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 28 */ - MONT_SQR_MUL(s0, s1, s2, a[13], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[14], a[14]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[13]); - z[12] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 29 */ - MONT_SQR_MUL(s0, s1, s2, a[14], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[14], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[14]); - z[13] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 30 */ - MONT_SQR_MAC(t0, t1, t2, a[15], a[15]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[15]); - z[14] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - z[15] = t0; - - /* under 2m, with t1 the bit above n limbs: at most one subtraction */ - borrow = sub_n(r, z, m, 16); - take = t1 | (borrow ^ 1); - select_n(r, r, z, take & 1, 16); -} - -/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ -static void crypton_mont_sqr24(limb_t *r, const limb_t *a, const limb_t *m, - limb_t n0, limb_t *t) -{ - limb_t *u = t, *z = t + 24; - limb_t t0 = 0, t1 = 0, t2 = 0, s0, s1, s2, borrow, take; - - /* column 0 */ - MONT_SQR_MAC(t0, t1, t2, a[0], a[0]); - u[0] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[0], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 1 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[1]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[1]); - u[1] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[1], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 2 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[1], a[1]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[1]); - u[2] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[2], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 3 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[3]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[1]); - u[3] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[3], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 4 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[2], a[2]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[1]); - u[4] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[4], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 5 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[1]); - u[5] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[5], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 6 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[3], a[3]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[1]); - u[6] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[6], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 7 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[1]); - u[7] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[7], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 8 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[4], a[4]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[1]); - u[8] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[8], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 9 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[1]); - u[9] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[9], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 10 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[5], a[5]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[1]); - u[10] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[10], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 11 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[1]); - u[11] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[11], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 12 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[6], a[6]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[1]); - u[12] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[12], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 13 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[1]); - u[13] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[13], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 14 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[7], a[7]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[1]); - u[14] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[14], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 15 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[1]); - u[15] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[15], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 16 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[8], a[8]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[1]); - u[16] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[16], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 17 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[1]); - u[17] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[17], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 18 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[9], a[9]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[1]); - u[18] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[18], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 19 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[1]); - u[19] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[19], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 20 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[10], a[10]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[1]); - u[20] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[20], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 21 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[1]); - u[21] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[21], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 22 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[11], a[11]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[1]); - u[22] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[22], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 23 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[1]); - u[23] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[23], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 24 */ - MONT_SQR_MUL(s0, s1, s2, a[1], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[12], a[12]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[1]); - z[0] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 25 */ - MONT_SQR_MUL(s0, s1, s2, a[2], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[2], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[2]); - z[1] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 26 */ - MONT_SQR_MUL(s0, s1, s2, a[3], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[13], a[13]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[3]); - z[2] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 27 */ - MONT_SQR_MUL(s0, s1, s2, a[4], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[4], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[4]); - z[3] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 28 */ - MONT_SQR_MUL(s0, s1, s2, a[5], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[14], a[14]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[5]); - z[4] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 29 */ - MONT_SQR_MUL(s0, s1, s2, a[6], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[6], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[6]); - z[5] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 30 */ - MONT_SQR_MUL(s0, s1, s2, a[7], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[16]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[15], a[15]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[7]); - z[6] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 31 */ - MONT_SQR_MUL(s0, s1, s2, a[8], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[16]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[8], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[8]); - z[7] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 32 */ - MONT_SQR_MUL(s0, s1, s2, a[9], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[17]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[16], a[16]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[9]); - z[8] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 33 */ - MONT_SQR_MUL(s0, s1, s2, a[10], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[17]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[10], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[10]); - z[9] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 34 */ - MONT_SQR_MUL(s0, s1, s2, a[11], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[18]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[17], a[17]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[11]); - z[10] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 35 */ - MONT_SQR_MUL(s0, s1, s2, a[12], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[18]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[12], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[12]); - z[11] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 36 */ - MONT_SQR_MUL(s0, s1, s2, a[13], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[19]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[18], a[18]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[13]); - z[12] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 37 */ - MONT_SQR_MUL(s0, s1, s2, a[14], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[19]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[14], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[14]); - z[13] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 38 */ - MONT_SQR_MUL(s0, s1, s2, a[15], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[20]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[19], a[19]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[15]); - z[14] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 39 */ - MONT_SQR_MUL(s0, s1, s2, a[16], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[20]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[16], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[16]); - z[15] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 40 */ - MONT_SQR_MUL(s0, s1, s2, a[17], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[21]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[20], a[20]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[17]); - z[16] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 41 */ - MONT_SQR_MUL(s0, s1, s2, a[18], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[21]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[18], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[18]); - z[17] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 42 */ - MONT_SQR_MUL(s0, s1, s2, a[19], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[22]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[21], a[21]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[19]); - z[18] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 43 */ - MONT_SQR_MUL(s0, s1, s2, a[20], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[22]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[20], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[20]); - z[19] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 44 */ - MONT_SQR_MUL(s0, s1, s2, a[21], a[23]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[22], a[22]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[21]); - z[20] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 45 */ - MONT_SQR_MUL(s0, s1, s2, a[22], a[23]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[22], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[22]); - z[21] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 46 */ - MONT_SQR_MAC(t0, t1, t2, a[23], a[23]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[23]); - z[22] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - z[23] = t0; - - /* under 2m, with t1 the bit above n limbs: at most one subtraction */ - borrow = sub_n(r, z, m, 24); - take = t1 | (borrow ^ 1); - select_n(r, r, z, take & 1, 24); -} - -/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */ -static void crypton_mont_sqr32(limb_t *r, const limb_t *a, const limb_t *m, - limb_t n0, limb_t *t) -{ - limb_t *u = t, *z = t + 32; - limb_t t0 = 0, t1 = 0, t2 = 0, s0, s1, s2, borrow, take; - - /* column 0 */ - MONT_SQR_MAC(t0, t1, t2, a[0], a[0]); - u[0] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[0], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 1 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[1]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[1]); - u[1] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[1], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 2 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[1], a[1]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[1]); - u[2] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[2], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 3 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[3]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[2]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[1]); - u[3] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[3], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 4 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[2], a[2]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[1]); - u[4] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[4], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 5 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[4]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[3]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[1]); - u[5] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[5], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 6 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[3], a[3]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[1]); - u[6] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[6], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 7 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[5]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[4]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[1]); - u[7] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[7], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 8 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[4], a[4]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[1]); - u[8] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[8], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 9 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[6]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[5]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[1]); - u[9] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[9], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 10 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[5], a[5]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[1]); - u[10] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[10], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 11 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[7]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[6]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[1]); - u[11] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[11], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 12 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[6], a[6]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[1]); - u[12] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[12], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 13 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[8]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[7]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[1]); - u[13] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[13], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 14 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[7], a[7]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[1]); - u[14] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[14], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 15 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[9]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[8]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[1]); - u[15] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[15], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 16 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[8], a[8]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[1]); - u[16] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[16], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 17 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[10]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[9]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[1]); - u[17] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[17], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 18 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[9], a[9]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[1]); - u[18] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[18], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 19 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[11]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[10]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[1]); - u[19] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[19], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 20 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[10], a[10]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[1]); - u[20] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[20], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 21 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[12]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[11]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[1]); - u[21] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[21], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 22 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[11], a[11]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[1]); - u[22] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[22], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 23 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[13]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[12]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[1]); - u[23] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[23], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 24 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[12], a[12]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[1]); - u[24] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[24], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 25 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[14]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[13]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[1]); - u[25] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[25], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 26 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[13], a[13]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[1]); - u[26] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[26], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 27 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[15]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[14]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[1]); - u[27] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[27], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 28 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[14], a[14]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[1]); - u[28] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[28], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 29 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[16]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[15]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[1]); - u[29] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[29], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 30 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[16]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[15], a[15]); - MONT_SQR_MAC(t0, t1, t2, u[0], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[1]); - u[30] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[30], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 31 */ - MONT_SQR_MUL(s0, s1, s2, a[0], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[1], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[17]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[16]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[0], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[1]); - u[31] = t0 * n0; - MONT_SQR_MAC(t0, t1, t2, u[31], m[0]); - t0 = t1; - t1 = t2; - t2 = 0; - /* column 32 */ - MONT_SQR_MUL(s0, s1, s2, a[1], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[2], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[17]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[16], a[16]); - MONT_SQR_MAC(t0, t1, t2, u[1], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[2], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[2]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[1]); - z[0] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 33 */ - MONT_SQR_MUL(s0, s1, s2, a[2], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[3], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[18]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[17]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[2], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[3]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[2]); - z[1] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 34 */ - MONT_SQR_MUL(s0, s1, s2, a[3], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[4], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[18]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[17], a[17]); - MONT_SQR_MAC(t0, t1, t2, u[3], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[4], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[4]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[3]); - z[2] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 35 */ - MONT_SQR_MUL(s0, s1, s2, a[4], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[5], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[19]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[18]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[4], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[5]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[4]); - z[3] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 36 */ - MONT_SQR_MUL(s0, s1, s2, a[5], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[6], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[19]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[18], a[18]); - MONT_SQR_MAC(t0, t1, t2, u[5], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[6], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[6]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[5]); - z[4] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 37 */ - MONT_SQR_MUL(s0, s1, s2, a[6], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[7], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[20]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[19]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[6], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[7]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[6]); - z[5] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 38 */ - MONT_SQR_MUL(s0, s1, s2, a[7], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[8], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[20]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[19], a[19]); - MONT_SQR_MAC(t0, t1, t2, u[7], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[8], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[8]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[7]); - z[6] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 39 */ - MONT_SQR_MUL(s0, s1, s2, a[8], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[9], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[21]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[20]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[8], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[9]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[8]); - z[7] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 40 */ - MONT_SQR_MUL(s0, s1, s2, a[9], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[10], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[21]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[20], a[20]); - MONT_SQR_MAC(t0, t1, t2, u[9], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[10], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[10]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[9]); - z[8] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 41 */ - MONT_SQR_MUL(s0, s1, s2, a[10], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[11], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[22]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[21]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[10], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[11]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[10]); - z[9] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 42 */ - MONT_SQR_MUL(s0, s1, s2, a[11], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[12], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[22]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[21], a[21]); - MONT_SQR_MAC(t0, t1, t2, u[11], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[12], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[12]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[11]); - z[10] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 43 */ - MONT_SQR_MUL(s0, s1, s2, a[12], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[13], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[23]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[22]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[12], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[13]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[12]); - z[11] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 44 */ - MONT_SQR_MUL(s0, s1, s2, a[13], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[14], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[23]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[22], a[22]); - MONT_SQR_MAC(t0, t1, t2, u[13], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[14], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[14]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[13]); - z[12] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 45 */ - MONT_SQR_MUL(s0, s1, s2, a[14], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[15], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[24]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[23]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[14], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[15]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[14]); - z[13] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 46 */ - MONT_SQR_MUL(s0, s1, s2, a[15], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[16], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[24]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[23], a[23]); - MONT_SQR_MAC(t0, t1, t2, u[15], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[16], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[16]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[15]); - z[14] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 47 */ - MONT_SQR_MUL(s0, s1, s2, a[16], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[17], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[25]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[24]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[16], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[17]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[16]); - z[15] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 48 */ - MONT_SQR_MUL(s0, s1, s2, a[17], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[18], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[25]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[24], a[24]); - MONT_SQR_MAC(t0, t1, t2, u[17], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[18], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[18]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[17]); - z[16] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 49 */ - MONT_SQR_MUL(s0, s1, s2, a[18], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[19], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[26]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[25]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[18], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[19]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[18]); - z[17] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 50 */ - MONT_SQR_MUL(s0, s1, s2, a[19], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[20], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[26]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[25], a[25]); - MONT_SQR_MAC(t0, t1, t2, u[19], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[20], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[20]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[19]); - z[18] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 51 */ - MONT_SQR_MUL(s0, s1, s2, a[20], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[21], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[27]); - MONT_SQR_MAC(s0, s1, s2, a[25], a[26]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[20], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[21]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[20]); - z[19] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 52 */ - MONT_SQR_MUL(s0, s1, s2, a[21], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[22], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[25], a[27]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[26], a[26]); - MONT_SQR_MAC(t0, t1, t2, u[21], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[22], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[22]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[21]); - z[20] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 53 */ - MONT_SQR_MUL(s0, s1, s2, a[22], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[23], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[25], a[28]); - MONT_SQR_MAC(s0, s1, s2, a[26], a[27]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[22], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[23]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[22]); - z[21] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 54 */ - MONT_SQR_MUL(s0, s1, s2, a[23], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[24], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[25], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[26], a[28]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[27], a[27]); - MONT_SQR_MAC(t0, t1, t2, u[23], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[24], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[24]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[23]); - z[22] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 55 */ - MONT_SQR_MUL(s0, s1, s2, a[24], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[25], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[26], a[29]); - MONT_SQR_MAC(s0, s1, s2, a[27], a[28]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[24], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[25]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[24]); - z[23] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 56 */ - MONT_SQR_MUL(s0, s1, s2, a[25], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[26], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[27], a[29]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[28], a[28]); - MONT_SQR_MAC(t0, t1, t2, u[25], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[26], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[26]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[25]); - z[24] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 57 */ - MONT_SQR_MUL(s0, s1, s2, a[26], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[27], a[30]); - MONT_SQR_MAC(s0, s1, s2, a[28], a[29]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[26], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[27]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[26]); - z[25] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 58 */ - MONT_SQR_MUL(s0, s1, s2, a[27], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[28], a[30]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[29], a[29]); - MONT_SQR_MAC(t0, t1, t2, u[27], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[28], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[28]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[27]); - z[26] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 59 */ - MONT_SQR_MUL(s0, s1, s2, a[28], a[31]); - MONT_SQR_MAC(s0, s1, s2, a[29], a[30]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[28], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[29]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[28]); - z[27] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 60 */ - MONT_SQR_MUL(s0, s1, s2, a[29], a[31]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, a[30], a[30]); - MONT_SQR_MAC(t0, t1, t2, u[29], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[30], m[30]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[29]); - z[28] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 61 */ - MONT_SQR_MUL(s0, s1, s2, a[30], a[31]); - MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2); - MONT_SQR_MAC(t0, t1, t2, u[30], m[31]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[30]); - z[29] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - /* column 62 */ - MONT_SQR_MAC(t0, t1, t2, a[31], a[31]); - MONT_SQR_MAC(t0, t1, t2, u[31], m[31]); - z[30] = t0; - t0 = t1; - t1 = t2; - t2 = 0; - z[31] = t0; - - /* under 2m, with t1 the bit above n limbs: at most one subtraction */ - borrow = sub_n(r, z, m, 32); - take = t1 | (borrow ^ 1); - select_n(r, r, z, take & 1, 32); -} - -#endif diff --git a/cbits/crypton_mont_sqr.py b/cbits/crypton_mont_sqr.py deleted file mode 100644 --- a/cbits/crypton_mont_sqr.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -"""Generate cbits/crypton_mont_sqr.h: Montgomery squaring for AArch64 at the -limb counts of RSA's CRT halves, fully unrolled and product-scanning. - -Usage: python3 cbits/crypton_mont_sqr.py > cbits/crypton_mont_sqr.h - -The result is a column at a time, with the reduction folded into the same -scan (the FIPS method), so the running sum stays in three registers instead -of going through memory a row at a time. Each product off the diagonal is -summed once into a column of its own, doubled, and added in. -""" - -SIZES = (16, 24, 32) - -HEADER = """\ -/* Generated by cbits/crypton_mont_sqr.py; do not edit. - * - * Montgomery squaring for AArch64 at 16, 24 and 32 limbs, the halves of - * RSA-2048, -3072 and -4096. Each answers what mont_sqr answers. */ -#ifndef CRYPTON_MONT_SQR_H -#define CRYPTON_MONT_SQR_H - -#include "crypton_bignum.h" - -/* (t2:t1:t0) += x * y */ -#define MONT_SQR_MAC(t0, t1, t2, x, y) \\ - do { \\ - limb_t lo_, hi_; \\ - __asm__("mul\\t%0, %5, %6\\n\\t" \\ - "umulh\\t%1, %5, %6\\n\\t" \\ - "adds\\t%2, %2, %0\\n\\t" \\ - "adcs\\t%3, %3, %1\\n\\t" \\ - "adc\\t%4, %4, xzr" \\ - : "=&r"(lo_), "=&r"(hi_), "+r"(t0), "+r"(t1), "+r"(t2) \\ - : "r"(x), "r"(y) \\ - : "cc"); \\ - } while (0) - -/* (s2:s1:s0) = x * y */ -#define MONT_SQR_MUL(s0, s1, s2, x, y) \\ - __asm__("mul\\t%0, %3, %4\\n\\t" \\ - "umulh\\t%1, %3, %4\\n\\t" \\ - "mov\\t%2, xzr" \\ - : "=&r"(s0), "=&r"(s1), "=r"(s2) \\ - : "r"(x), "r"(y)) - -/* (t2:t1:t0) += 2 * (s2:s1:s0) */ -#define MONT_SQR_ADD2(t0, t1, t2, s0, s1, s2) \\ - do { \\ - s2 = (s2 << 1) | (s1 >> 63); \\ - s1 = (s1 << 1) | (s0 >> 63); \\ - s0 <<= 1; \\ - __asm__("adds\\t%0, %0, %3\\n\\t" \\ - "adcs\\t%1, %1, %4\\n\\t" \\ - "adc\\t%2, %2, %5" \\ - : "+r"(t0), "+r"(t1), "+r"(t2) \\ - : "r"(s0), "r"(s1), "r"(s2) \\ - : "cc"); \\ - } while (0) -""" - -FOOTER = """\ -#endif -""" - - -def gen(n): - out = [] - w = out.append - w(f"/* r = a * a * R^-1 mod m, for a under m; t is 2n limbs of scratch */") - w(f"static void crypton_mont_sqr{n}(limb_t *r, const limb_t *a, const limb_t *m,") - w(f"{' ' * len(f'static void crypton_mont_sqr{n}(')}limb_t n0, limb_t *t)") - w("{") - w(f"\tlimb_t *u = t, *z = t + {n};") - w("\tlimb_t t0 = 0, t1 = 0, t2 = 0, s0, s1, s2, borrow, take;") - w("") - for i in range(2 * n - 1): - w(f"\t/* column {i} */") - pairs = [(j, i - j) for j in range(max(0, i - n + 1), n) if j < i - j] - if pairs: - j, k = pairs[0] - w(f"\tMONT_SQR_MUL(s0, s1, s2, a[{j}], a[{k}]);") - for (j, k) in pairs[1:]: - w(f"\tMONT_SQR_MAC(s0, s1, s2, a[{j}], a[{k}]);") - w("\tMONT_SQR_ADD2(t0, t1, t2, s0, s1, s2);") - if i % 2 == 0: - w(f"\tMONT_SQR_MAC(t0, t1, t2, a[{i // 2}], a[{i // 2}]);") - for j in range(max(0, i - n + 1), min(i, n)): - w(f"\tMONT_SQR_MAC(t0, t1, t2, u[{j}], m[{i - j}]);") - if i < n: - w(f"\tu[{i}] = t0 * n0;") - w(f"\tMONT_SQR_MAC(t0, t1, t2, u[{i}], m[0]);") - else: - w(f"\tz[{i - n}] = t0;") - w("\tt0 = t1;") - w("\tt1 = t2;") - w("\tt2 = 0;") - w(f"\tz[{n - 1}] = t0;") - w("") - w("\t/* under 2m, with t1 the bit above n limbs: at most one subtraction */") - w(f"\tborrow = sub_n(r, z, m, {n});") - w("\ttake = t1 | (borrow ^ 1);") - w(f"\tselect_n(r, r, z, take & 1, {n});") - w("}") - w("") - return "\n".join(out) - - -def main(): - print(HEADER) - for n in SIZES: - print(gen(n)) - print(FOOTER, end="") - - -if __name__ == "__main__": - main() diff --git a/cbits/crypton_powm.c b/cbits/crypton_powm.c --- a/cbits/crypton_powm.c +++ b/cbits/crypton_powm.c @@ -22,13 +22,14 @@ #include #include -/* On AArch64, squaring at the RSA half sizes goes through generated code - * that scans a column at a time with the reduction folded in; see - * cbits/crypton_mont_sqr.py. CRYPTON_NO_MONT_SQR keeps mont_sqr, to compare. */ -#if defined(__aarch64__) && LIMB_BITS == 64 && !defined(CRYPTON_NO_MONT_SQR) \ +/* On AArch64, multiplication and squaring at the RSA half sizes go through + * generated code that scans a column at a time with the reduction folded in; + * see cbits/crypton_mont.py. CRYPTON_NO_MONT_FIXED keeps mont_mul and + * mont_sqr, to compare. */ +#if defined(__aarch64__) && LIMB_BITS == 64 && !defined(CRYPTON_NO_MONT_FIXED) \ && (defined(__GNUC__) || defined(__clang__)) -#define CRYPTON_MONT_SQR 1 -#include +#define CRYPTON_MONT_FIXED 1 +#include #endif /* @@ -175,6 +176,13 @@ (void)scratch; (void)s2n; #endif +#ifdef CRYPTON_MONT_FIXED + switch (n) { + case 16: crypton_mont_mul16(r, a, b, m, n0, t); return; + case 24: crypton_mont_mul24(r, a, b, m, n0, t); return; + case 32: crypton_mont_mul32(r, a, b, m, n0, t); return; + } +#endif mont_mul(r, a, b, m, n0, n, t); } @@ -190,7 +198,7 @@ (void)scratch; (void)s2n; #endif -#ifdef CRYPTON_MONT_SQR +#ifdef CRYPTON_MONT_FIXED switch (n) { case 16: crypton_mont_sqr16(r, a, m, n0, t); return; case 24: crypton_mont_sqr24(r, a, m, n0, t); return; @@ -371,6 +379,10 @@ return n > 0; #elif defined(CRYPTON_POWM_S2N) return n == 32 && powm_s2n_usable(n); +#elif defined(CRYPTON_MONT_FIXED) + /* AArch64: the generated code is ahead of GMP at 2048 bits, and the + * Linux GMP is ahead of the C at 3072 and 4096 */ + return n == 32; #else (void) n; return 0; @@ -387,7 +399,39 @@ const uint8_t *exp, uint32_t explen, const uint8_t *mod, uint32_t modlen) { + return crypton_powm_pub_r2(out, base, baselen, exp, explen, mod, modlen, NULL); +} + +int crypton_powm_r2(uint8_t *out, const uint8_t *mod, uint32_t modlen) +{ uint32_t n = (modlen + LIMB_BYTES - 1) / LIMB_BYTES; + limb_t *space, *m, *r2, *t; + + if (modlen == 0 || n == 0 || (mod[modlen - 1] & 1) == 0 || !powm_pub_wanted(n)) + return 1; + space = calloc(5 * n, sizeof(limb_t)); + if (space == NULL) + return 1; + m = space; + r2 = m + n; + t = r2 + n; + if (from_be(m, n, mod, modlen) != 0) { + free(space); + return 1; + } + mont_r2(r2, m, mont_n0(m[0]), n, t); + to_be(out, modlen, r2, n); + free(space); + return 0; +} + +int crypton_powm_pub_r2(uint8_t *out, + const uint8_t *base, uint32_t baselen, + const uint8_t *exp, uint32_t explen, + const uint8_t *mod, uint32_t modlen, + const uint8_t *r2be) +{ + uint32_t n = (modlen + LIMB_BYTES - 1) / LIMB_BYTES; uint32_t words = (6 + 2) * n + powm_s2n_scratch(n); limb_t *space, *m, *r2, *acc, *tmp, *bm, *one, *t, *scratch, n0; uint32_t top, i; @@ -427,7 +471,10 @@ if (from_be(m, n, mod, modlen) != 0) goto fail; n0 = mont_n0(m[0]); - mont_r2(r2, m, n0, n, t); + if (r2be == NULL) + mont_r2(r2, m, n0, n, t); + else if (from_be(r2, n, r2be, modlen) != 0) + goto fail; /* the base into Montgomery form. The accumulator starts there rather * than at one, because the exponent's top bit is a one by diff --git a/cbits/crypton_powm.h b/cbits/crypton_powm.h --- a/cbits/crypton_powm.h +++ b/cbits/crypton_powm.h @@ -37,6 +37,33 @@ const uint8_t *exp, uint32_t explen, const uint8_t *mod, uint32_t modlen); +/* crypton_powm_pub with R^2 mod m given, big-endian in modlen bytes, as + * crypton_powm_r2 computes it; NULL computes it here as crypton_powm_pub does. + * A caller verifying under one key many times keeps it. */ +int crypton_powm_pub_r2(uint8_t *out, + const uint8_t *base, uint32_t baselen, + const uint8_t *exp, uint32_t explen, + const uint8_t *mod, uint32_t modlen, + const uint8_t *r2be); + +/* R^2 mod m for crypton_powm_pub_r2, or nonzero where crypton_powm_pub would + * decline this modulus, so that nothing is kept for it. */ +int crypton_powm_r2(uint8_t *out, const uint8_t *mod, uint32_t modlen); + +/* Two secure exponentiations at once, as the halves of an RSA private-key + * operation by CRT are: each as crypton_powm_sec, with the same contract. + * + * Only on x86-64 with AVX-512 IFMA, for two moduli of 128, 192 or 256 bytes + * and two exponents of the same length. It declines everywhere else -- + * returning nonzero with both outputs untouched -- so a caller must have + * crypton_powm_sec to fall back to. */ +int crypton_powm_sec_x2(uint8_t *out1, const uint8_t *base1, uint32_t baselen1, + const uint8_t *exp1, uint32_t explen1, + const uint8_t *mod1, uint32_t modlen1, + uint8_t *out2, const uint8_t *base2, uint32_t baselen2, + const uint8_t *exp2, uint32_t explen2, + const uint8_t *mod2, uint32_t modlen2); + /* The window the secure exponentiation walks the exponent in, so that a * build which did not take -DPOWM_WINDOW_BITS cannot pass for one that * did. */ diff --git a/cbits/crypton_powm_ifma.c b/cbits/crypton_powm_ifma.c new file mode 100644 --- /dev/null +++ b/cbits/crypton_powm_ifma.c @@ -0,0 +1,187 @@ +/* Two secure exponentiations at once with AVX-512 IFMA, the CRT halves of + * RSA-2048, -3072 and -4096: crypton_powm_sec's walk with a five-bit window, + * over the multiplication generated by cbits/crypton_powm_ifma.py. */ +#include +#include +#include +#include +#include + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +#include +#include "crypton_powm_ifma.h" + +#define WIN 5 + +typedef void (*amm_fn)(uint64_t *, const uint64_t *, const uint64_t *, const uint64_t *, + uint64_t, uint64_t *, const uint64_t *, const uint64_t *, + const uint64_t *, uint64_t); +typedef void (*sel_fn)(uint64_t *, const uint64_t *, uint64_t, uint64_t *, + const uint64_t *, uint64_t); + +/* limbs of 64 bits to digits of 52, d of them; the rest of the w words zero */ +static void to_digits(uint64_t *dg, uint32_t w, const limb_t *l, uint32_t n, uint32_t d) +{ + uint32_t i; + + memset(dg, 0, w * sizeof(uint64_t)); + for (i = 0; i < d; i++) { + uint32_t bit = 52 * i, li = bit / 64, sh = bit % 64; + uint64_t v = li < n ? l[li] >> sh : 0; + + if (sh > 12 && li + 1 < n) + v |= l[li + 1] << (64 - sh); + dg[i] = v & IFMA_MASK52; + } +} + +static void from_digits(limb_t *l, uint32_t n, const uint64_t *dg, uint32_t d) +{ + uint32_t i; + + memset(l, 0, n * sizeof(limb_t)); + for (i = 0; i < d; i++) { + uint32_t bit = 52 * i, li = bit / 64, sh = bit % 64; + + if (li < n) + l[li] |= dg[i] << sh; + if (sh > 12 && li + 1 < n) + l[li + 1] |= dg[i] >> (64 - sh); + } +} + +/* Window k from the bottom of a big-endian exponent of len bytes. Which + * window is read is the loop counter's business, not the exponent's. */ +static uint64_t window(const uint8_t *e, uint32_t len, uint32_t k) +{ + uint64_t v = 0; + uint32_t j; + + for (j = 0; j < WIN; j++) { + uint32_t bit = k * WIN + j; + + if (bit < 8 * len) + v |= (uint64_t) ((e[len - 1 - bit / 8] >> (bit % 8)) & 1) << j; + } + return v; +} + +/* m and k0 in digits, and R^2 mod m for R = 2^(52d): crypton's R^2 is for + * 2^(64n), so it is doubled the rest of the way */ +static int setup(const uint8_t *mod, uint32_t modlen, uint32_t n, uint32_t d, uint32_t w, + limb_t *ml, uint64_t *md, uint64_t *rr, uint64_t *k0, limb_t *t) +{ + limb_t *r2 = t + 4 * n; + uint32_t j; + + if (from_be(ml, n, mod, modlen) != 0 || (ml[0] & 1) == 0) + return 1; + *k0 = mont_n0(ml[0]); + mont_r2(r2, ml, *k0, n, t); + for (j = 0; j < 2 * (52 * d - 64 * n); j++) + dbl_mod(r2, ml, n, t); + to_digits(md, w, ml, n, d); + to_digits(rr, w, r2, n, d); + return 0; +} + +int crypton_powm_sec_x2(uint8_t *out1, const uint8_t *base1, uint32_t baselen1, + const uint8_t *exp1, uint32_t explen1, + const uint8_t *mod1, uint32_t modlen1, + uint8_t *out2, const uint8_t *base2, uint32_t baselen2, + const uint8_t *exp2, uint32_t explen2, + const uint8_t *mod2, uint32_t modlen2) +{ + uint32_t n, d, w, nwin, k, j; + amm_fn amm; + sel_fn sel; + uint64_t *space, *m1, *m2, *rr1, *rr2, *x1, *x2, *s1, *s2, *one, *t1, *t2, k1, k2; + limb_t *l1, *l2, *lt, *lt2; + size_t words; + + if (modlen1 != modlen2 || explen1 != explen2) + return 1; + if ((crypton_x86_simd_features() & CRYPTON_X86_IFMA) == 0) + return 1; + switch (modlen1) { + case 128: d = 20; amm = ifma_amm20_x2; sel = ifma_sel5; break; + case 192: d = 30; amm = ifma_amm30_x2; sel = ifma_sel8; break; + case 256: d = 40; amm = ifma_amm40_x2; sel = ifma_sel10; break; + default: return 1; + } + n = modlen1 / 8; + w = 4 * ((d + 3) / 4); + + /* nine numbers in digits, two tables of 32, and limbs for the setup */ + words = (size_t) w * (9 + 2 * 32) + (size_t) n * 12; + space = calloc(words, sizeof(uint64_t)); + if (space == NULL) + return 1; + m1 = space; m2 = m1 + w; rr1 = m2 + w; rr2 = rr1 + w; + x1 = rr2 + w; x2 = x1 + w; s1 = x2 + w; s2 = s1 + w; one = s2 + w; + t1 = one + w; t2 = t1 + 32 * w; + l1 = (limb_t *)(t2 + 32 * w); l2 = l1 + n; lt = l2 + n; lt2 = lt + n; + + if (setup(mod1, modlen1, n, d, w, l1, m1, rr1, &k1, lt) != 0 + || setup(mod2, modlen2, n, d, w, l2, m2, rr2, &k2, lt) != 0) + goto fail; + + /* the bases, under their moduli as the contract says, into digits */ + if (from_be(lt, n, base1, baselen1) != 0 || from_be(lt2, n, base2, baselen2) != 0) + goto fail; + to_digits(s1, w, lt, n, d); + to_digits(s2, w, lt2, n, d); + + /* table[0] = R and table[1] = b R in Montgomery form, table[k] = b^k R */ + one[0] = 1; + amm(t1, one, rr1, m1, k1, t2, one, rr2, m2, k2); + amm(t1 + w, s1, rr1, m1, k1, t2 + w, s2, rr2, m2, k2); + for (k = 2; k < 32; k++) + amm(t1 + k * w, t1 + (k - 1) * w, t1 + w, m1, k1, + t2 + k * w, t2 + (k - 1) * w, t2 + w, m2, k2); + + memcpy(x1, t1, w * sizeof(uint64_t)); + memcpy(x2, t2, w * sizeof(uint64_t)); + nwin = (8 * explen1 + WIN - 1) / WIN; + for (k = nwin; k > 0; k--) { + for (j = 0; j < WIN; j++) + amm(x1, x1, x1, m1, k1, x2, x2, x2, m2, k2); + sel(s1, t1, window(exp1, explen1, k - 1), s2, t2, window(exp2, explen2, k - 1)); + amm(x1, x1, s1, m1, k1, x2, x2, s2, m2, k2); + } + + /* out of Montgomery form, then under m with one masked subtraction */ + amm(x1, x1, one, m1, k1, x2, x2, one, m2, k2); + from_digits(lt, n, x1, d); + select_n(lt, lt2, lt, sub_n(lt2, lt, l1, n) ^ 1, n); + to_be(out1, modlen1, lt, n); + from_digits(lt, n, x2, d); + select_n(lt, lt2, lt, sub_n(lt2, lt, l2, n) ^ 1, n); + to_be(out2, modlen2, lt, n); + + crypton_bzero(space, words * sizeof(uint64_t)); + free(space); + return 0; + +fail: + crypton_bzero(space, words * sizeof(uint64_t)); + free(space); + return 1; +} + +#else + +int crypton_powm_sec_x2(uint8_t *out1, const uint8_t *base1, uint32_t baselen1, + const uint8_t *exp1, uint32_t explen1, + const uint8_t *mod1, uint32_t modlen1, + uint8_t *out2, const uint8_t *base2, uint32_t baselen2, + const uint8_t *exp2, uint32_t explen2, + const uint8_t *mod2, uint32_t modlen2) +{ + (void) out1; (void) base1; (void) baselen1; (void) exp1; (void) explen1; + (void) mod1; (void) modlen1; (void) out2; (void) base2; (void) baselen2; + (void) exp2; (void) explen2; (void) mod2; (void) modlen2; + return 1; +} + +#endif diff --git a/cbits/crypton_powm_ifma.h b/cbits/crypton_powm_ifma.h new file mode 100644 --- /dev/null +++ b/cbits/crypton_powm_ifma.h @@ -0,0 +1,513 @@ +/* Generated by cbits/crypton_powm_ifma.py; do not edit. + * + * Almost-Montgomery multiplication in radix 2^52, two at once: r = a b + * 2^-52D mod m, under 2m for a and b under 2m. Digits are 52 bits in 64-bit + * words, each number padded with zero words to a whole number of 256-bit + * registers. m is read from memory; at 20 digits a is held in registers, + * above that it is read from memory too. */ +#ifndef CRYPTON_POWM_IFMA_H +#define CRYPTON_POWM_IFMA_H + +#include +#include + +#define IFMA_MASK52 ((UINT64_C(1) << 52) - 1) +#define IFMA_TARGET __attribute__((target("avx2,avx512f,avx512vl,avx512ifma"))) + +/* 20 lanes to 52-bit digits: what is above 52 bits moves up a lane, + * and the one-bit carries that can then remain are resolved together */ +IFMA_TARGET static inline void ifma_norm5(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4) +{ + const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1); + const __m256i zero = _mm256_setzero_si256(); + uint64_t g = 0, p = 0, c; + __m256i C0 = _mm256_srli_epi64(X0, 52); + __m256i C1 = _mm256_srli_epi64(X1, 52); + __m256i C2 = _mm256_srli_epi64(X2, 52); + __m256i C3 = _mm256_srli_epi64(X3, 52); + __m256i C4 = _mm256_srli_epi64(X4, 52); + X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3)); + X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3)); + X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3)); + X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3)); + X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3)); + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16; + c = ((g << 1) + p) ^ p; + X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk); + _mm256_storeu_si256((__m256i *)(r + 0), X0); + X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk); + _mm256_storeu_si256((__m256i *)(r + 4), X1); + X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk); + _mm256_storeu_si256((__m256i *)(r + 8), X2); + X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk); + _mm256_storeu_si256((__m256i *)(r + 12), X3); + X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk); + _mm256_storeu_si256((__m256i *)(r + 16), X4); +} + +/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping + * one; entries are 20 words apart */ +IFMA_TARGET static void ifma_sel5(uint64_t *o1, const uint64_t *t1, uint64_t k1, + uint64_t *o2, const uint64_t *t2, uint64_t k2) +{ + const __m256i one = _mm256_set1_epi64x(1); + __m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2); + __m256i kv = _mm256_setzero_si256(); + __m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv; + int k; + for (k = 0; k < 32; k++) { + __mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv); + const uint64_t *e1 = t1 + 20 * k, *e2 = t2 + 20 * k; + P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0); + P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4); + P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8); + P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12); + P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16); + kv = _mm256_add_epi64(kv, one); + } + _mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0); + _mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1); + _mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2); + _mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3); + _mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4); +} + +/* 32 lanes to 52-bit digits: what is above 52 bits moves up a lane, + * and the one-bit carries that can then remain are resolved together */ +IFMA_TARGET static inline void ifma_norm8(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4, __m256i X5, __m256i X6, __m256i X7) +{ + const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1); + const __m256i zero = _mm256_setzero_si256(); + uint64_t g = 0, p = 0, c; + __m256i C0 = _mm256_srli_epi64(X0, 52); + __m256i C1 = _mm256_srli_epi64(X1, 52); + __m256i C2 = _mm256_srli_epi64(X2, 52); + __m256i C3 = _mm256_srli_epi64(X3, 52); + __m256i C4 = _mm256_srli_epi64(X4, 52); + __m256i C5 = _mm256_srli_epi64(X5, 52); + __m256i C6 = _mm256_srli_epi64(X6, 52); + __m256i C7 = _mm256_srli_epi64(X7, 52); + X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3)); + X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3)); + X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3)); + X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3)); + X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3)); + X5 = _mm256_add_epi64(_mm256_and_si256(X5, mk), _mm256_alignr_epi64(C5, C4, 3)); + X6 = _mm256_add_epi64(_mm256_and_si256(X6, mk), _mm256_alignr_epi64(C6, C5, 3)); + X7 = _mm256_add_epi64(_mm256_and_si256(X7, mk), _mm256_alignr_epi64(C7, C6, 3)); + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X5, mk) << 20; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X5, mk) << 20; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X6, mk) << 24; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X6, mk) << 24; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X7, mk) << 28; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X7, mk) << 28; + c = ((g << 1) + p) ^ p; + X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk); + _mm256_storeu_si256((__m256i *)(r + 0), X0); + X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk); + _mm256_storeu_si256((__m256i *)(r + 4), X1); + X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk); + _mm256_storeu_si256((__m256i *)(r + 8), X2); + X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk); + _mm256_storeu_si256((__m256i *)(r + 12), X3); + X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk); + _mm256_storeu_si256((__m256i *)(r + 16), X4); + X5 = _mm256_and_si256(_mm256_mask_add_epi64(X5, (__mmask8)(c >> 20 & 15), X5, one), mk); + _mm256_storeu_si256((__m256i *)(r + 20), X5); + X6 = _mm256_and_si256(_mm256_mask_add_epi64(X6, (__mmask8)(c >> 24 & 15), X6, one), mk); + _mm256_storeu_si256((__m256i *)(r + 24), X6); + X7 = _mm256_and_si256(_mm256_mask_add_epi64(X7, (__mmask8)(c >> 28 & 15), X7, one), mk); + _mm256_storeu_si256((__m256i *)(r + 28), X7); +} + +/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping + * one; entries are 32 words apart */ +IFMA_TARGET static void ifma_sel8(uint64_t *o1, const uint64_t *t1, uint64_t k1, + uint64_t *o2, const uint64_t *t2, uint64_t k2) +{ + const __m256i one = _mm256_set1_epi64x(1); + __m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2); + __m256i kv = _mm256_setzero_si256(); + __m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv, P5 = kv, Q5 = kv, P6 = kv, Q6 = kv, P7 = kv, Q7 = kv; + int k; + for (k = 0; k < 32; k++) { + __mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv); + const uint64_t *e1 = t1 + 32 * k, *e2 = t2 + 32 * k; + P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0); + P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4); + P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8); + P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12); + P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16); + P5 = _mm256_mask_loadu_epi64(P5, s1, e1 + 20); Q5 = _mm256_mask_loadu_epi64(Q5, s2, e2 + 20); + P6 = _mm256_mask_loadu_epi64(P6, s1, e1 + 24); Q6 = _mm256_mask_loadu_epi64(Q6, s2, e2 + 24); + P7 = _mm256_mask_loadu_epi64(P7, s1, e1 + 28); Q7 = _mm256_mask_loadu_epi64(Q7, s2, e2 + 28); + kv = _mm256_add_epi64(kv, one); + } + _mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0); + _mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1); + _mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2); + _mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3); + _mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4); + _mm256_storeu_si256((__m256i *)(o1 + 20), P5); _mm256_storeu_si256((__m256i *)(o2 + 20), Q5); + _mm256_storeu_si256((__m256i *)(o1 + 24), P6); _mm256_storeu_si256((__m256i *)(o2 + 24), Q6); + _mm256_storeu_si256((__m256i *)(o1 + 28), P7); _mm256_storeu_si256((__m256i *)(o2 + 28), Q7); +} + +/* 40 lanes to 52-bit digits: what is above 52 bits moves up a lane, + * and the one-bit carries that can then remain are resolved together */ +IFMA_TARGET static inline void ifma_norm10(uint64_t *r, __m256i X0, __m256i X1, __m256i X2, __m256i X3, __m256i X4, __m256i X5, __m256i X6, __m256i X7, __m256i X8, __m256i X9) +{ + const __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1); + const __m256i zero = _mm256_setzero_si256(); + uint64_t g = 0, p = 0, c; + __m256i C0 = _mm256_srli_epi64(X0, 52); + __m256i C1 = _mm256_srli_epi64(X1, 52); + __m256i C2 = _mm256_srli_epi64(X2, 52); + __m256i C3 = _mm256_srli_epi64(X3, 52); + __m256i C4 = _mm256_srli_epi64(X4, 52); + __m256i C5 = _mm256_srli_epi64(X5, 52); + __m256i C6 = _mm256_srli_epi64(X6, 52); + __m256i C7 = _mm256_srli_epi64(X7, 52); + __m256i C8 = _mm256_srli_epi64(X8, 52); + __m256i C9 = _mm256_srli_epi64(X9, 52); + X0 = _mm256_add_epi64(_mm256_and_si256(X0, mk), _mm256_alignr_epi64(C0, zero, 3)); + X1 = _mm256_add_epi64(_mm256_and_si256(X1, mk), _mm256_alignr_epi64(C1, C0, 3)); + X2 = _mm256_add_epi64(_mm256_and_si256(X2, mk), _mm256_alignr_epi64(C2, C1, 3)); + X3 = _mm256_add_epi64(_mm256_and_si256(X3, mk), _mm256_alignr_epi64(C3, C2, 3)); + X4 = _mm256_add_epi64(_mm256_and_si256(X4, mk), _mm256_alignr_epi64(C4, C3, 3)); + X5 = _mm256_add_epi64(_mm256_and_si256(X5, mk), _mm256_alignr_epi64(C5, C4, 3)); + X6 = _mm256_add_epi64(_mm256_and_si256(X6, mk), _mm256_alignr_epi64(C6, C5, 3)); + X7 = _mm256_add_epi64(_mm256_and_si256(X7, mk), _mm256_alignr_epi64(C7, C6, 3)); + X8 = _mm256_add_epi64(_mm256_and_si256(X8, mk), _mm256_alignr_epi64(C8, C7, 3)); + X9 = _mm256_add_epi64(_mm256_and_si256(X9, mk), _mm256_alignr_epi64(C9, C8, 3)); + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X0, mk) << 0; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X0, mk) << 0; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X1, mk) << 4; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X1, mk) << 4; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X2, mk) << 8; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X2, mk) << 8; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X3, mk) << 12; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X3, mk) << 12; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X4, mk) << 16; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X4, mk) << 16; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X5, mk) << 20; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X5, mk) << 20; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X6, mk) << 24; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X6, mk) << 24; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X7, mk) << 28; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X7, mk) << 28; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X8, mk) << 32; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X8, mk) << 32; + g |= (uint64_t) _mm256_cmpgt_epu64_mask(X9, mk) << 36; + p |= (uint64_t) _mm256_cmpeq_epu64_mask(X9, mk) << 36; + c = ((g << 1) + p) ^ p; + X0 = _mm256_and_si256(_mm256_mask_add_epi64(X0, (__mmask8)(c >> 0 & 15), X0, one), mk); + _mm256_storeu_si256((__m256i *)(r + 0), X0); + X1 = _mm256_and_si256(_mm256_mask_add_epi64(X1, (__mmask8)(c >> 4 & 15), X1, one), mk); + _mm256_storeu_si256((__m256i *)(r + 4), X1); + X2 = _mm256_and_si256(_mm256_mask_add_epi64(X2, (__mmask8)(c >> 8 & 15), X2, one), mk); + _mm256_storeu_si256((__m256i *)(r + 8), X2); + X3 = _mm256_and_si256(_mm256_mask_add_epi64(X3, (__mmask8)(c >> 12 & 15), X3, one), mk); + _mm256_storeu_si256((__m256i *)(r + 12), X3); + X4 = _mm256_and_si256(_mm256_mask_add_epi64(X4, (__mmask8)(c >> 16 & 15), X4, one), mk); + _mm256_storeu_si256((__m256i *)(r + 16), X4); + X5 = _mm256_and_si256(_mm256_mask_add_epi64(X5, (__mmask8)(c >> 20 & 15), X5, one), mk); + _mm256_storeu_si256((__m256i *)(r + 20), X5); + X6 = _mm256_and_si256(_mm256_mask_add_epi64(X6, (__mmask8)(c >> 24 & 15), X6, one), mk); + _mm256_storeu_si256((__m256i *)(r + 24), X6); + X7 = _mm256_and_si256(_mm256_mask_add_epi64(X7, (__mmask8)(c >> 28 & 15), X7, one), mk); + _mm256_storeu_si256((__m256i *)(r + 28), X7); + X8 = _mm256_and_si256(_mm256_mask_add_epi64(X8, (__mmask8)(c >> 32 & 15), X8, one), mk); + _mm256_storeu_si256((__m256i *)(r + 32), X8); + X9 = _mm256_and_si256(_mm256_mask_add_epi64(X9, (__mmask8)(c >> 36 & 15), X9, one), mk); + _mm256_storeu_si256((__m256i *)(r + 36), X9); +} + +/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping + * one; entries are 40 words apart */ +IFMA_TARGET static void ifma_sel10(uint64_t *o1, const uint64_t *t1, uint64_t k1, + uint64_t *o2, const uint64_t *t2, uint64_t k2) +{ + const __m256i one = _mm256_set1_epi64x(1); + __m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2); + __m256i kv = _mm256_setzero_si256(); + __m256i P0 = kv, Q0 = kv, P1 = kv, Q1 = kv, P2 = kv, Q2 = kv, P3 = kv, Q3 = kv, P4 = kv, Q4 = kv, P5 = kv, Q5 = kv, P6 = kv, Q6 = kv, P7 = kv, Q7 = kv, P8 = kv, Q8 = kv, P9 = kv, Q9 = kv; + int k; + for (k = 0; k < 32; k++) { + __mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv); + const uint64_t *e1 = t1 + 40 * k, *e2 = t2 + 40 * k; + P0 = _mm256_mask_loadu_epi64(P0, s1, e1 + 0); Q0 = _mm256_mask_loadu_epi64(Q0, s2, e2 + 0); + P1 = _mm256_mask_loadu_epi64(P1, s1, e1 + 4); Q1 = _mm256_mask_loadu_epi64(Q1, s2, e2 + 4); + P2 = _mm256_mask_loadu_epi64(P2, s1, e1 + 8); Q2 = _mm256_mask_loadu_epi64(Q2, s2, e2 + 8); + P3 = _mm256_mask_loadu_epi64(P3, s1, e1 + 12); Q3 = _mm256_mask_loadu_epi64(Q3, s2, e2 + 12); + P4 = _mm256_mask_loadu_epi64(P4, s1, e1 + 16); Q4 = _mm256_mask_loadu_epi64(Q4, s2, e2 + 16); + P5 = _mm256_mask_loadu_epi64(P5, s1, e1 + 20); Q5 = _mm256_mask_loadu_epi64(Q5, s2, e2 + 20); + P6 = _mm256_mask_loadu_epi64(P6, s1, e1 + 24); Q6 = _mm256_mask_loadu_epi64(Q6, s2, e2 + 24); + P7 = _mm256_mask_loadu_epi64(P7, s1, e1 + 28); Q7 = _mm256_mask_loadu_epi64(Q7, s2, e2 + 28); + P8 = _mm256_mask_loadu_epi64(P8, s1, e1 + 32); Q8 = _mm256_mask_loadu_epi64(Q8, s2, e2 + 32); + P9 = _mm256_mask_loadu_epi64(P9, s1, e1 + 36); Q9 = _mm256_mask_loadu_epi64(Q9, s2, e2 + 36); + kv = _mm256_add_epi64(kv, one); + } + _mm256_storeu_si256((__m256i *)(o1 + 0), P0); _mm256_storeu_si256((__m256i *)(o2 + 0), Q0); + _mm256_storeu_si256((__m256i *)(o1 + 4), P1); _mm256_storeu_si256((__m256i *)(o2 + 4), Q1); + _mm256_storeu_si256((__m256i *)(o1 + 8), P2); _mm256_storeu_si256((__m256i *)(o2 + 8), Q2); + _mm256_storeu_si256((__m256i *)(o1 + 12), P3); _mm256_storeu_si256((__m256i *)(o2 + 12), Q3); + _mm256_storeu_si256((__m256i *)(o1 + 16), P4); _mm256_storeu_si256((__m256i *)(o2 + 16), Q4); + _mm256_storeu_si256((__m256i *)(o1 + 20), P5); _mm256_storeu_si256((__m256i *)(o2 + 20), Q5); + _mm256_storeu_si256((__m256i *)(o1 + 24), P6); _mm256_storeu_si256((__m256i *)(o2 + 24), Q6); + _mm256_storeu_si256((__m256i *)(o1 + 28), P7); _mm256_storeu_si256((__m256i *)(o2 + 28), Q7); + _mm256_storeu_si256((__m256i *)(o1 + 32), P8); _mm256_storeu_si256((__m256i *)(o2 + 32), Q8); + _mm256_storeu_si256((__m256i *)(o1 + 36), P9); _mm256_storeu_si256((__m256i *)(o2 + 36), Q9); +} + +/* 20 digits, 5 registers a number */ +IFMA_TARGET static void ifma_amm20_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1, + const uint64_t *m1, uint64_t k1, + uint64_t *r2, const uint64_t *a2, const uint64_t *b2, + const uint64_t *m2, uint64_t k2) +{ + const __m256i zero = _mm256_setzero_si256(); +#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k))) + __m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero; + __m256i A0 = L(a1, 0), A1 = L(a1, 1), A2 = L(a1, 2), A3 = L(a1, 3), A4 = L(a1, 4); + __m256i B0 = L(a2, 0), B1 = L(a2, 1), B2 = L(a2, 2), B3 = L(a2, 3), B4 = L(a2, 4); + uint64_t x1 = 0, x2 = 0; + int i; + + for (i = 0; i < 20; i++) { + uint64_t bi = b1[i], ci = b2[i], y, z; + /* lane 0 in full: the products' high halves, which are lane 1's, too */ + unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1; + unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2; + __m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci); + __m256i yv, zv; + + y = ((uint64_t) s1 * k1) & IFMA_MASK52; + z = ((uint64_t) s2 * k2) & IFMA_MASK52; + s1 += (unsigned __int128) m1[0] * y; + s2 += (unsigned __int128) m2[0] * z; + yv = _mm256_set1_epi64x((long long) y); + zv = _mm256_set1_epi64x((long long) z); + X0 = _mm256_madd52lo_epu64(X0, A0, bv); Y0 = _mm256_madd52lo_epu64(Y0, B0, cv); + X1 = _mm256_madd52lo_epu64(X1, A1, bv); Y1 = _mm256_madd52lo_epu64(Y1, B1, cv); + X2 = _mm256_madd52lo_epu64(X2, A2, bv); Y2 = _mm256_madd52lo_epu64(Y2, B2, cv); + X3 = _mm256_madd52lo_epu64(X3, A3, bv); Y3 = _mm256_madd52lo_epu64(Y3, B3, cv); + X4 = _mm256_madd52lo_epu64(X4, A4, bv); Y4 = _mm256_madd52lo_epu64(Y4, B4, cv); + X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv); + /* down a lane: lane 0 is a multiple of 2^52, and s carries it */ + X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1); + X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1); + X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1); + X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1); + X4 = _mm256_alignr_epi64(zero, X4, 1); Y4 = _mm256_alignr_epi64(zero, Y4, 1); + x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52); + x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52); + X0 = _mm256_madd52hi_epu64(X0, A0, bv); Y0 = _mm256_madd52hi_epu64(Y0, B0, cv); + X1 = _mm256_madd52hi_epu64(X1, A1, bv); Y1 = _mm256_madd52hi_epu64(Y1, B1, cv); + X2 = _mm256_madd52hi_epu64(X2, A2, bv); Y2 = _mm256_madd52hi_epu64(Y2, B2, cv); + X3 = _mm256_madd52hi_epu64(X3, A3, bv); Y3 = _mm256_madd52hi_epu64(Y3, B3, cv); + X4 = _mm256_madd52hi_epu64(X4, A4, bv); Y4 = _mm256_madd52hi_epu64(Y4, B4, cv); + X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv); + } +#undef L + /* lane 0, with the carry it was never given, is what the scalar holds */ + X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1); + Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2); + ifma_norm5(r1, X0, X1, X2, X3, X4); + ifma_norm5(r2, Y0, Y1, Y2, Y3, Y4); +} + +/* 30 digits, 8 registers a number */ +IFMA_TARGET static void ifma_amm30_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1, + const uint64_t *m1, uint64_t k1, + uint64_t *r2, const uint64_t *a2, const uint64_t *b2, + const uint64_t *m2, uint64_t k2) +{ + const __m256i zero = _mm256_setzero_si256(); +#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k))) + __m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, X5 = zero, X6 = zero, X7 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero, Y5 = zero, Y6 = zero, Y7 = zero; + uint64_t x1 = 0, x2 = 0; + int i; + + for (i = 0; i < 30; i++) { + uint64_t bi = b1[i], ci = b2[i], y, z; + /* lane 0 in full: the products' high halves, which are lane 1's, too */ + unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1; + unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2; + __m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci); + __m256i yv, zv; + + y = ((uint64_t) s1 * k1) & IFMA_MASK52; + z = ((uint64_t) s2 * k2) & IFMA_MASK52; + s1 += (unsigned __int128) m1[0] * y; + s2 += (unsigned __int128) m2[0] * z; + yv = _mm256_set1_epi64x((long long) y); + zv = _mm256_set1_epi64x((long long) z); + X0 = _mm256_madd52lo_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52lo_epu64(Y0, L(a2, 0), cv); + X1 = _mm256_madd52lo_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52lo_epu64(Y1, L(a2, 1), cv); + X2 = _mm256_madd52lo_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52lo_epu64(Y2, L(a2, 2), cv); + X3 = _mm256_madd52lo_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52lo_epu64(Y3, L(a2, 3), cv); + X4 = _mm256_madd52lo_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52lo_epu64(Y4, L(a2, 4), cv); + X5 = _mm256_madd52lo_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52lo_epu64(Y5, L(a2, 5), cv); + X6 = _mm256_madd52lo_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52lo_epu64(Y6, L(a2, 6), cv); + X7 = _mm256_madd52lo_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52lo_epu64(Y7, L(a2, 7), cv); + X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv); + X5 = _mm256_madd52lo_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52lo_epu64(Y5, L(m2, 5), zv); + X6 = _mm256_madd52lo_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52lo_epu64(Y6, L(m2, 6), zv); + X7 = _mm256_madd52lo_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52lo_epu64(Y7, L(m2, 7), zv); + /* down a lane: lane 0 is a multiple of 2^52, and s carries it */ + X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1); + X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1); + X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1); + X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1); + X4 = _mm256_alignr_epi64(X5, X4, 1); Y4 = _mm256_alignr_epi64(Y5, Y4, 1); + X5 = _mm256_alignr_epi64(X6, X5, 1); Y5 = _mm256_alignr_epi64(Y6, Y5, 1); + X6 = _mm256_alignr_epi64(X7, X6, 1); Y6 = _mm256_alignr_epi64(Y7, Y6, 1); + X7 = _mm256_alignr_epi64(zero, X7, 1); Y7 = _mm256_alignr_epi64(zero, Y7, 1); + x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52); + x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52); + X0 = _mm256_madd52hi_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52hi_epu64(Y0, L(a2, 0), cv); + X1 = _mm256_madd52hi_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52hi_epu64(Y1, L(a2, 1), cv); + X2 = _mm256_madd52hi_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52hi_epu64(Y2, L(a2, 2), cv); + X3 = _mm256_madd52hi_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52hi_epu64(Y3, L(a2, 3), cv); + X4 = _mm256_madd52hi_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52hi_epu64(Y4, L(a2, 4), cv); + X5 = _mm256_madd52hi_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52hi_epu64(Y5, L(a2, 5), cv); + X6 = _mm256_madd52hi_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52hi_epu64(Y6, L(a2, 6), cv); + X7 = _mm256_madd52hi_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52hi_epu64(Y7, L(a2, 7), cv); + X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv); + X5 = _mm256_madd52hi_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52hi_epu64(Y5, L(m2, 5), zv); + X6 = _mm256_madd52hi_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52hi_epu64(Y6, L(m2, 6), zv); + X7 = _mm256_madd52hi_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52hi_epu64(Y7, L(m2, 7), zv); + } +#undef L + /* lane 0, with the carry it was never given, is what the scalar holds */ + X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1); + Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2); + ifma_norm8(r1, X0, X1, X2, X3, X4, X5, X6, X7); + ifma_norm8(r2, Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7); +} + +/* 40 digits, 10 registers a number */ +IFMA_TARGET static void ifma_amm40_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1, + const uint64_t *m1, uint64_t k1, + uint64_t *r2, const uint64_t *a2, const uint64_t *b2, + const uint64_t *m2, uint64_t k2) +{ + const __m256i zero = _mm256_setzero_si256(); +#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k))) + __m256i X0 = zero, X1 = zero, X2 = zero, X3 = zero, X4 = zero, X5 = zero, X6 = zero, X7 = zero, X8 = zero, X9 = zero, Y0 = zero, Y1 = zero, Y2 = zero, Y3 = zero, Y4 = zero, Y5 = zero, Y6 = zero, Y7 = zero, Y8 = zero, Y9 = zero; + uint64_t x1 = 0, x2 = 0; + int i; + + for (i = 0; i < 40; i++) { + uint64_t bi = b1[i], ci = b2[i], y, z; + /* lane 0 in full: the products' high halves, which are lane 1's, too */ + unsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1; + unsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2; + __m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci); + __m256i yv, zv; + + y = ((uint64_t) s1 * k1) & IFMA_MASK52; + z = ((uint64_t) s2 * k2) & IFMA_MASK52; + s1 += (unsigned __int128) m1[0] * y; + s2 += (unsigned __int128) m2[0] * z; + yv = _mm256_set1_epi64x((long long) y); + zv = _mm256_set1_epi64x((long long) z); + X0 = _mm256_madd52lo_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52lo_epu64(Y0, L(a2, 0), cv); + X1 = _mm256_madd52lo_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52lo_epu64(Y1, L(a2, 1), cv); + X2 = _mm256_madd52lo_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52lo_epu64(Y2, L(a2, 2), cv); + X3 = _mm256_madd52lo_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52lo_epu64(Y3, L(a2, 3), cv); + X4 = _mm256_madd52lo_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52lo_epu64(Y4, L(a2, 4), cv); + X5 = _mm256_madd52lo_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52lo_epu64(Y5, L(a2, 5), cv); + X6 = _mm256_madd52lo_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52lo_epu64(Y6, L(a2, 6), cv); + X7 = _mm256_madd52lo_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52lo_epu64(Y7, L(a2, 7), cv); + X8 = _mm256_madd52lo_epu64(X8, L(a1, 8), bv); Y8 = _mm256_madd52lo_epu64(Y8, L(a2, 8), cv); + X9 = _mm256_madd52lo_epu64(X9, L(a1, 9), bv); Y9 = _mm256_madd52lo_epu64(Y9, L(a2, 9), cv); + X0 = _mm256_madd52lo_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52lo_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52lo_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52lo_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52lo_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52lo_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52lo_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52lo_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52lo_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52lo_epu64(Y4, L(m2, 4), zv); + X5 = _mm256_madd52lo_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52lo_epu64(Y5, L(m2, 5), zv); + X6 = _mm256_madd52lo_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52lo_epu64(Y6, L(m2, 6), zv); + X7 = _mm256_madd52lo_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52lo_epu64(Y7, L(m2, 7), zv); + X8 = _mm256_madd52lo_epu64(X8, L(m1, 8), yv); Y8 = _mm256_madd52lo_epu64(Y8, L(m2, 8), zv); + X9 = _mm256_madd52lo_epu64(X9, L(m1, 9), yv); Y9 = _mm256_madd52lo_epu64(Y9, L(m2, 9), zv); + /* down a lane: lane 0 is a multiple of 2^52, and s carries it */ + X0 = _mm256_alignr_epi64(X1, X0, 1); Y0 = _mm256_alignr_epi64(Y1, Y0, 1); + X1 = _mm256_alignr_epi64(X2, X1, 1); Y1 = _mm256_alignr_epi64(Y2, Y1, 1); + X2 = _mm256_alignr_epi64(X3, X2, 1); Y2 = _mm256_alignr_epi64(Y3, Y2, 1); + X3 = _mm256_alignr_epi64(X4, X3, 1); Y3 = _mm256_alignr_epi64(Y4, Y3, 1); + X4 = _mm256_alignr_epi64(X5, X4, 1); Y4 = _mm256_alignr_epi64(Y5, Y4, 1); + X5 = _mm256_alignr_epi64(X6, X5, 1); Y5 = _mm256_alignr_epi64(Y6, Y5, 1); + X6 = _mm256_alignr_epi64(X7, X6, 1); Y6 = _mm256_alignr_epi64(Y7, Y6, 1); + X7 = _mm256_alignr_epi64(X8, X7, 1); Y7 = _mm256_alignr_epi64(Y8, Y7, 1); + X8 = _mm256_alignr_epi64(X9, X8, 1); Y8 = _mm256_alignr_epi64(Y9, Y8, 1); + X9 = _mm256_alignr_epi64(zero, X9, 1); Y9 = _mm256_alignr_epi64(zero, Y9, 1); + x1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52); + x2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52); + X0 = _mm256_madd52hi_epu64(X0, L(a1, 0), bv); Y0 = _mm256_madd52hi_epu64(Y0, L(a2, 0), cv); + X1 = _mm256_madd52hi_epu64(X1, L(a1, 1), bv); Y1 = _mm256_madd52hi_epu64(Y1, L(a2, 1), cv); + X2 = _mm256_madd52hi_epu64(X2, L(a1, 2), bv); Y2 = _mm256_madd52hi_epu64(Y2, L(a2, 2), cv); + X3 = _mm256_madd52hi_epu64(X3, L(a1, 3), bv); Y3 = _mm256_madd52hi_epu64(Y3, L(a2, 3), cv); + X4 = _mm256_madd52hi_epu64(X4, L(a1, 4), bv); Y4 = _mm256_madd52hi_epu64(Y4, L(a2, 4), cv); + X5 = _mm256_madd52hi_epu64(X5, L(a1, 5), bv); Y5 = _mm256_madd52hi_epu64(Y5, L(a2, 5), cv); + X6 = _mm256_madd52hi_epu64(X6, L(a1, 6), bv); Y6 = _mm256_madd52hi_epu64(Y6, L(a2, 6), cv); + X7 = _mm256_madd52hi_epu64(X7, L(a1, 7), bv); Y7 = _mm256_madd52hi_epu64(Y7, L(a2, 7), cv); + X8 = _mm256_madd52hi_epu64(X8, L(a1, 8), bv); Y8 = _mm256_madd52hi_epu64(Y8, L(a2, 8), cv); + X9 = _mm256_madd52hi_epu64(X9, L(a1, 9), bv); Y9 = _mm256_madd52hi_epu64(Y9, L(a2, 9), cv); + X0 = _mm256_madd52hi_epu64(X0, L(m1, 0), yv); Y0 = _mm256_madd52hi_epu64(Y0, L(m2, 0), zv); + X1 = _mm256_madd52hi_epu64(X1, L(m1, 1), yv); Y1 = _mm256_madd52hi_epu64(Y1, L(m2, 1), zv); + X2 = _mm256_madd52hi_epu64(X2, L(m1, 2), yv); Y2 = _mm256_madd52hi_epu64(Y2, L(m2, 2), zv); + X3 = _mm256_madd52hi_epu64(X3, L(m1, 3), yv); Y3 = _mm256_madd52hi_epu64(Y3, L(m2, 3), zv); + X4 = _mm256_madd52hi_epu64(X4, L(m1, 4), yv); Y4 = _mm256_madd52hi_epu64(Y4, L(m2, 4), zv); + X5 = _mm256_madd52hi_epu64(X5, L(m1, 5), yv); Y5 = _mm256_madd52hi_epu64(Y5, L(m2, 5), zv); + X6 = _mm256_madd52hi_epu64(X6, L(m1, 6), yv); Y6 = _mm256_madd52hi_epu64(Y6, L(m2, 6), zv); + X7 = _mm256_madd52hi_epu64(X7, L(m1, 7), yv); Y7 = _mm256_madd52hi_epu64(Y7, L(m2, 7), zv); + X8 = _mm256_madd52hi_epu64(X8, L(m1, 8), yv); Y8 = _mm256_madd52hi_epu64(Y8, L(m2, 8), zv); + X9 = _mm256_madd52hi_epu64(X9, L(m1, 9), yv); Y9 = _mm256_madd52hi_epu64(Y9, L(m2, 9), zv); + } +#undef L + /* lane 0, with the carry it was never given, is what the scalar holds */ + X0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1); + Y0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2); + ifma_norm10(r1, X0, X1, X2, X3, X4, X5, X6, X7, X8, X9); + ifma_norm10(r2, Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7, Y8, Y9); +} + +#endif diff --git a/cbits/crypton_powm_ifma.py b/cbits/crypton_powm_ifma.py new file mode 100644 --- /dev/null +++ b/cbits/crypton_powm_ifma.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Generate cbits/crypton_powm_ifma.h: two almost-Montgomery multiplications +at once in radix 2^52 with AVX-512 IFMA on 256-bit registers, at 20, 30 and +40 digits. Usage: python3 cbits/crypton_powm_ifma.py > cbits/crypton_powm_ifma.h + +Gueron and Krasnov's method, a digit of b at a time. Lane 0 is followed in +full in a scalar, so it is read before the high halves go in and they stay +off the chain the next digit waits on. +""" + +SIZES = (20, 30, 40) + +HEADER = """\ +/* Generated by cbits/crypton_powm_ifma.py; do not edit. + * + * Almost-Montgomery multiplication in radix 2^52, two at once: r = a b + * 2^-52D mod m, under 2m for a and b under 2m. Digits are 52 bits in 64-bit + * words, each number padded with zero words to a whole number of 256-bit + * registers. m is read from memory; at 20 digits a is held in registers, + * above that it is read from memory too. */ +#ifndef CRYPTON_POWM_IFMA_H +#define CRYPTON_POWM_IFMA_H + +#include +#include + +#define IFMA_MASK52 ((UINT64_C(1) << 52) - 1) +#define IFMA_TARGET __attribute__((target("avx2,avx512f,avx512vl,avx512ifma"))) +""" + +FOOTER = """\ +#endif +""" + + +def regs(d): + return (d + 3) // 4 + + +def gen_norm(R): + o = [] + w = o.append + xs = [f"X{j}" for j in range(R)] + w(f"/* {4 * R} lanes to 52-bit digits: what is above 52 bits moves up a lane,") + w(f" * and the one-bit carries that can then remain are resolved together */") + w(f"IFMA_TARGET static inline void ifma_norm{R}(uint64_t *r, " + + ", ".join(f"__m256i {x}" for x in xs) + ")") + w("{") + w("\tconst __m256i mk = _mm256_set1_epi64x((long long) IFMA_MASK52), one = _mm256_set1_epi64x(1);") + w("\tconst __m256i zero = _mm256_setzero_si256();") + w("\tuint64_t g = 0, p = 0, c;") + for j in range(R): + w(f"\t__m256i C{j} = _mm256_srli_epi64(X{j}, 52);") + for j in range(R): + prev = f"C{j - 1}" if j else "zero" + w(f"\tX{j} = _mm256_add_epi64(_mm256_and_si256(X{j}, mk), _mm256_alignr_epi64(C{j}, {prev}, 3));") + for j in range(R): + w(f"\tg |= (uint64_t) _mm256_cmpgt_epu64_mask(X{j}, mk) << {4 * j};") + w(f"\tp |= (uint64_t) _mm256_cmpeq_epu64_mask(X{j}, mk) << {4 * j};") + w("\tc = ((g << 1) + p) ^ p;") + for j in range(R): + w(f"\tX{j} = _mm256_and_si256(_mm256_mask_add_epi64(X{j}, (__mmask8)(c >> {4 * j} & 15), X{j}, one), mk);") + w(f"\t_mm256_storeu_si256((__m256i *)(r + {4 * j}), X{j});") + w("}") + w("") + return "\n".join(o) + + +def gen_amm(d): + R = regs(d) + in_regs = R <= 5 + o = [] + w = o.append + w(f"/* {d} digits, {R} registers a number */") + w(f"IFMA_TARGET static void ifma_amm{d}_x2(uint64_t *r1, const uint64_t *a1, const uint64_t *b1,") + w(f" const uint64_t *m1, uint64_t k1,") + w(f" uint64_t *r2, const uint64_t *a2, const uint64_t *b2,") + w(f" const uint64_t *m2, uint64_t k2)") + w("{") + w("\tconst __m256i zero = _mm256_setzero_si256();") + w("#define L(p, k) _mm256_loadu_si256((const __m256i *)((p) + 4 * (k)))") + X = [f"X{j}" for j in range(R)] + Y = [f"Y{j}" for j in range(R)] + w("\t__m256i " + ", ".join(f"{v} = zero" for v in X + Y) + ";") + if in_regs: + w("\t__m256i " + ", ".join(f"A{j} = L(a1, {j})" for j in range(R)) + ";") + w("\t__m256i " + ", ".join(f"B{j} = L(a2, {j})" for j in range(R)) + ";") + A = [f"A{j}" for j in range(R)] + B = [f"B{j}" for j in range(R)] + else: + A = [f"L(a1, {j})" for j in range(R)] + B = [f"L(a2, {j})" for j in range(R)] + M = [f"L(m1, {j})" for j in range(R)] + N = [f"L(m2, {j})" for j in range(R)] + w("\tuint64_t x1 = 0, x2 = 0;") + w("\tint i;") + w("") + w(f"\tfor (i = 0; i < {d}; i++) {{") + w("\t\tuint64_t bi = b1[i], ci = b2[i], y, z;") + w("\t\t/* lane 0 in full: the products' high halves, which are lane 1's, too */") + w("\t\tunsigned __int128 s1 = (unsigned __int128) a1[0] * bi + x1;") + w("\t\tunsigned __int128 s2 = (unsigned __int128) a2[0] * ci + x2;") + w("\t\t__m256i bv = _mm256_set1_epi64x((long long) bi), cv = _mm256_set1_epi64x((long long) ci);") + w("\t\t__m256i yv, zv;") + w("") + w("\t\ty = ((uint64_t) s1 * k1) & IFMA_MASK52;") + w("\t\tz = ((uint64_t) s2 * k2) & IFMA_MASK52;") + w("\t\ts1 += (unsigned __int128) m1[0] * y;") + w("\t\ts2 += (unsigned __int128) m2[0] * z;") + w("\t\tyv = _mm256_set1_epi64x((long long) y);") + w("\t\tzv = _mm256_set1_epi64x((long long) z);") + for j in range(R): + w(f"\t\tX{j} = _mm256_madd52lo_epu64(X{j}, {A[j]}, bv); Y{j} = _mm256_madd52lo_epu64(Y{j}, {B[j]}, cv);") + for j in range(R): + w(f"\t\tX{j} = _mm256_madd52lo_epu64(X{j}, {M[j]}, yv); Y{j} = _mm256_madd52lo_epu64(Y{j}, {N[j]}, zv);") + w("\t\t/* down a lane: lane 0 is a multiple of 2^52, and s carries it */") + for j in range(R): + nx = f"X{j + 1}" if j + 1 < R else "zero" + ny = f"Y{j + 1}" if j + 1 < R else "zero" + w(f"\t\tX{j} = _mm256_alignr_epi64({nx}, X{j}, 1); Y{j} = _mm256_alignr_epi64({ny}, Y{j}, 1);") + w("\t\tx1 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(X0)) + (uint64_t)(s1 >> 52);") + w("\t\tx2 = (uint64_t) _mm_cvtsi128_si64(_mm256_castsi256_si128(Y0)) + (uint64_t)(s2 >> 52);") + for j in range(R): + w(f"\t\tX{j} = _mm256_madd52hi_epu64(X{j}, {A[j]}, bv); Y{j} = _mm256_madd52hi_epu64(Y{j}, {B[j]}, cv);") + for j in range(R): + w(f"\t\tX{j} = _mm256_madd52hi_epu64(X{j}, {M[j]}, yv); Y{j} = _mm256_madd52hi_epu64(Y{j}, {N[j]}, zv);") + w("\t}") + w("#undef L") + w("\t/* lane 0, with the carry it was never given, is what the scalar holds */") + w("\tX0 = _mm256_mask_set1_epi64(X0, 1, (long long) x1);") + w("\tY0 = _mm256_mask_set1_epi64(Y0, 1, (long long) x2);") + w(f"\tifma_norm{R}(r1, " + ", ".join(X) + ");") + w(f"\tifma_norm{R}(r2, " + ", ".join(Y) + ");") + w("}") + w("") + return "\n".join(o) + + +def gen_sel(R): + o = [] + w = o.append + w(f"/* both tables' entries k1 and k2 of 32, every entry read and a mask keeping") + w(f" * one; entries are {4 * R} words apart */") + w(f"IFMA_TARGET static void ifma_sel{R}(uint64_t *o1, const uint64_t *t1, uint64_t k1,") + w(f" uint64_t *o2, const uint64_t *t2, uint64_t k2)") + w("{") + w("\tconst __m256i one = _mm256_set1_epi64x(1);") + w("\t__m256i v1 = _mm256_set1_epi64x((long long) k1), v2 = _mm256_set1_epi64x((long long) k2);") + w("\t__m256i kv = _mm256_setzero_si256();") + w("\t__m256i " + ", ".join(f"P{j} = kv, Q{j} = kv" for j in range(R)) + ";") + w("\tint k;") + w("\tfor (k = 0; k < 32; k++) {") + w("\t\t__mmask8 s1 = _mm256_cmpeq_epi64_mask(v1, kv), s2 = _mm256_cmpeq_epi64_mask(v2, kv);") + w(f"\t\tconst uint64_t *e1 = t1 + {4 * R} * k, *e2 = t2 + {4 * R} * k;") + for j in range(R): + w(f"\t\tP{j} = _mm256_mask_loadu_epi64(P{j}, s1, e1 + {4 * j}); Q{j} = _mm256_mask_loadu_epi64(Q{j}, s2, e2 + {4 * j});") + w("\t\tkv = _mm256_add_epi64(kv, one);") + w("\t}") + for j in range(R): + w(f"\t_mm256_storeu_si256((__m256i *)(o1 + {4 * j}), P{j}); _mm256_storeu_si256((__m256i *)(o2 + {4 * j}), Q{j});") + w("}") + w("") + return "\n".join(o) + + +def main(): + print(HEADER) + rs = sorted({regs(d) for d in SIZES}) + for R in rs: + print(gen_norm(R)) + print(gen_sel(R)) + for d in SIZES: + print(gen_amm(d)) + print(FOOTER, end="") + + +if __name__ == "__main__": + main() diff --git a/crypton.cabal b/crypton.cabal --- a/crypton.cabal +++ b/crypton.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: crypton -version: 2.2.3 +version: 2.2.4 -- crypton's own code is BSD-3-Clause. The parts of -- cbits/aes/gcm_fused_x86.c that follow picotls's fusion are MIT, and the -- vendored s2n-bignum assembly in cbits/s2n is taken under ISC, and the @@ -69,7 +69,8 @@ cbits/blake2/ref/*.h cbits/blake2/sse/*.h cbits/crypton_hash_prefix.c - cbits/crypton_mont_sqr.py + cbits/crypton_mont.py + cbits/crypton_powm_ifma.py cbits/mlkem/*.S cbits/mlkem/*.c cbits/mlkem/*.h @@ -329,6 +330,7 @@ Crypto.PubKey.Rabin.RW Crypto.PubKey.Rabin.Types Crypto.PubKey.RSA + Crypto.PubKey.RSA.Blinded Crypto.PubKey.RSA.OAEP Crypto.PubKey.RSA.PKCS15 Crypto.PubKey.RSA.Prim @@ -386,6 +388,7 @@ cbits/crypton_pbkdf2.c cbits/crypton_poly1305.c cbits/crypton_powm.c + cbits/crypton_powm_ifma.c cbits/crypton_rc4.c cbits/crypton_ripemd.c cbits/crypton_salsa.c @@ -405,6 +408,7 @@ cbits/p256/p256_ec.c other-modules: + Crypto.PubKey.RSA.Blinded.Internal Crypto.Cipher.AES.Primitive Crypto.Cipher.Blowfish.Primitive Crypto.Cipher.Camellia.Primitive diff --git a/tests/NumberSpec.hs b/tests/NumberSpec.hs --- a/tests/NumberSpec.hs +++ b/tests/NumberSpec.hs @@ -202,6 +202,13 @@ forAll (wideOf bytes) $ \e -> forAll (wideOf 128) $ \b -> expSafe b e modulus1024 === expFast b e modulus1024 + -- the fast one keeps R^2 for the last few moduli; asking under more of + -- them than it keeps, round and round, would show one used for another + it "keeps each modulus' constant to that modulus" $ do + let ms = [modulus2048 - 2 * 0x10001 * fromIntegral k | k <- [0 .. 11 :: Int]] + bs = [modulus2048 `div` (3 + fromIntegral k) | k <- [0 .. 11 :: Int]] + asked = concat (replicate 3 (zip bs ms)) + [(b, m) | (b, m) <- asked, expFast b 65537 m /= expSafe b 65537 m] `shouldBe` [] where safely (b, e, m) = expSafe b e m fastly (b, e, m) = expFast b e m diff --git a/tests/PubKey/RSASpec.hs b/tests/PubKey/RSASpec.hs --- a/tests/PubKey/RSASpec.hs +++ b/tests/PubKey/RSASpec.hs @@ -7,11 +7,16 @@ module PubKey.RSASpec (spec) where +import Control.Concurrent (forkIO) +import Control.Concurrent.MVar import Crypto.Hash import Crypto.Number.ModArithmetic (inverse) import Crypto.Number.Serialize (i2osp, i2ospOf_, os2ip) import qualified Crypto.PubKey.RSA as RSA +import qualified Crypto.PubKey.RSA.Blinded as RSA +import qualified Crypto.PubKey.RSA.OAEP as OAEP import qualified Crypto.PubKey.RSA.PKCS15 as RSA +import qualified Crypto.PubKey.RSA.PSS as PSS import Crypto.PubKey.RSA.Prim (ep) import qualified Crypto.PubKey.RSA.Prim as Prim import qualified Data.ByteString as B @@ -427,6 +432,110 @@ key = vectorToPrivate vector pub = vectorToPublic vector +-- | A 'RSA.BlindedKey' carries one blinding pair, each operation returning +-- the key with it squared and a new one drawn every 32 uses; the @Safer@ +-- functions keep one per key for the last few keys. None of it may change +-- an answer, so every answer is held against the unblinded one, over more +-- uses than one pair lasts. +blindedKeyTests :: Spec +blindedKeyTests = describe "BlindedKey" $ do + it "signs what signDigest Nothing signs, across refreshes" $ + threaded 100 $ \bk i -> do + let dg = hashWith SHA256 (B.replicate i 0x61) + (got, bk') <- RSA.signDigestBlinded bk dg + got `shouldBe` RSA.signDigest Nothing key dg + return bk' + + it "signs a DigestInfo as signDigestInfo Nothing does" $ + threaded 40 $ \bk i -> do + let di = B.replicate (20 + i `mod` 10) (fromIntegral i) + (got, bk') <- RSA.signDigestInfoBlinded bk di + got `shouldBe` RSA.signDigestInfo Nothing key di + return bk' + + it "decrypts what it was given" $ + threaded 40 $ \bk i -> do + let m = B.replicate i 0x5a + c <- either (error . show) id <$> RSA.encrypt pub m + (got, bk') <- RSA.decryptBlinded bk c + got `shouldBe` (Right m :: Either RSA.Error ByteString) + return bk' + + it "refuses what decrypt refuses" $ do + let c = B.replicate (RSA.public_size pub + 1) 1 + (got, _) <- RSA.decryptBlinded (RSA.blindedKey key) c + got `shouldBe` (Left RSA.MessageSizeIncorrect :: Either RSA.Error ByteString) + + it "makes PSS signatures that verify" $ + threaded 40 $ \bk i -> do + let m = B.replicate i 0x33 + params = PSS.defaultPSSParams SHA256 + (s, bk') <- PSS.signBlinded params bk m + either (error . show) (PSS.verify params pub m) s `shouldBe` True + return bk' + + it "decrypts OAEP" $ + threaded 40 $ \bk i -> do + let m = B.replicate i 0x77 + params = OAEP.defaultOAEPParams SHA256 + c <- either (error . show) id <$> OAEP.encrypt params pub m + (got, bk') <- OAEP.decryptBlinded params bk c + got `shouldBe` Right m + return bk' + + it "works for a key without its CRT values" $ do + let bare = key {RSA.private_p = 0, RSA.private_q = 0} + go _ 41 = return () + go bk i = do + let dg = hashWith SHA256 (B.replicate i 0x62) + (got, bk') <- RSA.signDigestBlinded bk dg + got `shouldBe` RSA.signDigest Nothing key dg + go bk' (i + 1) + go (RSA.blindedKey bare) (1 :: Int) + + -- The randomness comes from the monad, so a pure generator will do. + prop "runs in a pure generator" $ \testDRG -> + let go bk i acc + | i > (40 :: Int) = return (reverse acc) + | otherwise = do + (s, bk') <- RSA.signDigestBlinded bk (hashWith SHA256 (B.replicate i 0x64)) + go bk' (i + 1) (s : acc) + sigs = withTestDRG testDRG (go (RSA.blindedKey key) 1 []) + in sigs === [RSA.signDigest Nothing key (hashWith SHA256 (B.replicate i 0x64)) | i <- [1 .. 40]] + + it "gives every thread a right answer through the Safer functions" $ do + dones <- forM [1 .. 8 :: Int] $ \t -> do + done <- newEmptyMVar + _ <- forkIO $ do + oks <- forM [1 .. 30 :: Int] $ \i -> do + let dg = hashWith SHA256 (B.pack [fromIntegral t, fromIntegral i]) + got <- RSA.signSaferDigest key dg + return $! got == RSA.signDigest Nothing key dg + putMVar done (and oks) + return done + results <- mapM takeMVar dones + results `shouldBe` replicate 8 True + + -- More keys than the Safer functions remember, interleaved, so that the + -- table both hits and evicts. + it "keeps the Safer functions right across more keys than it remembers" $ do + let keys = key : [snd (withTestDRG (TestDRG (s', 2, 3, 4, 5)) (RSA.generate 64 65537)) | s' <- [1 .. 5]] + forM_ [1 .. 12 :: Int] $ \i -> + forM_ keys $ \k -> do + let dg = hashWith SHA256 (B.replicate i 0x63) + got <- RSA.signSaferDigest k dg + got `shouldBe` RSA.signDigest Nothing k dg + where + vector = firstVector vectorsSHA1 + key = vectorToPrivate vector + pub = vectorToPublic vector + -- run count operations, each handed the key the one before returned + threaded count op = go (RSA.blindedKey key) (1 :: Int) + where + go bk i + | i > count = return () + | otherwise = op bk i >>= \bk' -> go bk' (i + 1) + spec :: Spec spec = do keyGenerationTests @@ -443,5 +552,6 @@ zipWith doMalleabilityTest [katZero ..] $ filter vectorHasSignature vectorsSHA1 digestOperationTests + blindedKeyTests unpadTests ciphertextRangeTests