diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ - SPDX-License-Identifier: MPL-2.0 --> +## v0.0.4 + +* Bundled blst version bumped to v0.3.13 + ## v0.0.3 * Minor tweaks for GHC 9.8 compatibility. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ HsBLST is low-level Haskell bindings and a high-level interface to [BLST](https://github.com/supranational/blst) -- a multilingual BLS12-381 signature library. -This package includes sources from v0.3.11 version of BLST, licensed under +This package includes sources from v0.3.13 version of BLST, licensed under [Apache License 2.0](https://github.com/supranational/blst/blob/master/LICENSE) ## Build Instructions [↑](#hsblst) @@ -28,4 +28,4 @@ HsBLST is maintained and funded with ❤️ by [Serokell](https://serokell.io/). The names and logo for Serokell are trademark of Serokell OÜ. -We love open source software! See [our other projects](https://serokell.io/community?utm_source=github) or [hire us](https://serokell.io/hire-us?utm_source=github) to design, develop and grow your idea! +We love open source software! See [our other projects](https://serokell.io/community?utm_source=github) or [hire us](https://serokell.io/?utm_source=github) to design, develop and grow your idea! diff --git a/c-source/README.md b/c-source/README.md --- a/c-source/README.md +++ b/c-source/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/supranational/blst.svg?branch=master)](https://travis-ci.com/github/supranational/blst) [![Actions status](https://github.com/supranational/blst/workflows/build/badge.svg)](https://github.com/supranational/blst/actions) +[![Build Status](https://api.travis-ci.com/supranational/blst.svg?branch=master)](https://travis-ci.com/github/supranational/blst) [![Actions status](https://github.com/supranational/blst/workflows/build/badge.svg)](https://github.com/supranational/blst/actions) [![CodeQL status](https://github.com/supranational/blst/workflows/CodeQL/badge.svg)](https://github.com/supranational/blst/actions/workflows/codeql-analysis.yml)
@@ -35,7 +35,7 @@ This library is compliant with the following IETF draft specifications: - [IETF BLS Signature V5](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature) -- [IETF Hash-to-Curve V16](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve) +- [IETF RFC 9380 Hashing to Elliptic Curves](https://www.rfc-editor.org/rfc/rfc9380.html) The serialization formatting is implemented according to [the ZCash definition](#serialization-format). @@ -199,6 +199,8 @@ * blst_aux.h - contains experimental functions not yet committed for long-term maintenance * blst.hpp - provides foundational class-oriented C++ interface to blst library * blst.swg - provides SWIG definitions for creating blst bindings for other languages, such as Java and Python + * **C#** - folder containing C# bindings and an example of how to use them + * **Emscripten** - folder containing an example of how to use Emscripten WebAssembly bindings from Javascript * **Go** - folder containing Go bindings for blst, including tests and benchmarks * **Java** - folder containing an example of how to use SWIG Java bindings for blst * **Node.js** - folder containing an example of how to use SWIG Javascript bindings for blst @@ -209,6 +211,7 @@ * **Src** - folder containing C code for lower level blst functions such as field operations, extension field operations, hash-to-field, and more * **Asm** - folder containing Perl scripts that are used to generate assembly code for different hardware platforms including x86 with ADX instructions, x86 without ADX instructions, and ARMv8, and [ABI](https://en.wikipedia.org/wiki/Application_binary_interface)[1] * **Build** - this folder containing a set of pre-generated assembly files for a variety of operating systems and maintenance scripts. + * **Cheri** - assembly code for use on [CHERI](https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/) platforms * **Coff** - assembly code for use on Window systems with GNU toolchain * **Elf** - assembly code for use on Unix systems * **Mach-o** - assembly code for use on Apple operating systems diff --git a/c-source/bindings/blst.h b/c-source/bindings/blst.h --- a/c-source/bindings/blst.h +++ b/c-source/bindings/blst.h @@ -25,10 +25,13 @@ extern "C" { #elif defined(__BLST_CGO__) typedef _Bool bool; /* it's assumed that cgo calls modern enough compiler */ -#elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 -# define bool _Bool -#else -# define bool int +#elif !defined(bool) +# if defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 +# define bool _Bool +# else +# define bool int +# endif +# define __blst_h_bool__ #endif #ifdef SWIG @@ -478,5 +481,8 @@ #ifdef __cplusplus } +#elif defined(__blst_h_bool__) +# undef __blst_h_bool__ +# undef bool #endif #endif diff --git a/c-source/build/assembly.S b/c-source/build/assembly.S --- a/c-source/build/assembly.S +++ b/c-source/build/assembly.S @@ -26,6 +26,19 @@ # include "elf/div3w-x86_64.s" # include "elf/ct_is_square_mod_384-x86_64.s" # elif defined(_WIN64) || defined(__CYGWIN__) +# if defined(__BLST_PORTABLE__) +# include "coff/sha256-portable-x86_64.s" +# define blst_sha256_block_data_order blst_sha256_block_ssse3 +# define LSEH_begin_blst_sha256_block_data_order LSEH_begin_blst_sha256_block_ssse3 +# define LSEH_body_blst_sha256_block_data_order LSEH_body_blst_sha256_block_ssse3 +# define LSEH_info_blst_sha256_block_data_order_prologue LSEH_info_blst_sha256_block_ssse3_prologue +# define LSEH_body_blst_sha256_block_data_order LSEH_body_blst_sha256_block_ssse3 +# define LSEH_epilogue_blst_sha256_block_data_order LSEH_epilogue_blst_sha256_block_ssse3 +# define LSEH_info_blst_sha256_block_data_order_body LSEH_info_blst_sha256_block_ssse3_body +# define LSEH_epilogue_blst_sha256_block_data_order LSEH_epilogue_blst_sha256_block_ssse3 +# define LSEH_end_blst_sha256_block_data_order LSEH_end_blst_sha256_block_ssse3 +# define LSEH_info_blst_sha256_block_data_order_epilogue LSEH_info_blst_sha256_block_ssse3_epilogue +# endif # include "coff/sha256-x86_64.s" # if defined(__ADX__) || defined(__BLST_PORTABLE__) # include "coff/ctx_inverse_mod_384-x86_64.s" @@ -71,7 +84,19 @@ # include "mach-o/ct_is_square_mod_384-x86_64.s" # endif #elif defined(__aarch64__) -# if defined(__ELF__) +# if defined(__CHERI_PURE_CAPABILITY__) +# include "cheri/sha256-armv8.S" +# include "cheri/ct_inverse_mod_384-armv8.S" +# include "cheri/add_mod_384-armv8.S" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# include "cheri/mul_mont_384-armv8.S" +# include "cheri/mul_mont_256-armv8.S" +# include "cheri/add_mod_256-armv8.S" +# include "cheri/ct_inverse_mod_256-armv8.S" +# include "cheri/div3w-armv8.S" +# include "cheri/ct_is_square_mod_384-armv8.S" +# elif defined(__ELF__) # include "elf/sha256-armv8.S" # include "elf/ct_inverse_mod_384-armv8.S" # include "elf/add_mod_384-armv8.S" diff --git a/c-source/build/cheri/add_mod_256-armv8.S b/c-source/build/cheri/add_mod_256-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/add_mod_256-armv8.S @@ -0,0 +1,379 @@ +.text + +.globl add_mod_256 +.hidden add_mod_256 +.type add_mod_256,%function +.align 5 +add_mod_256: + ldp x8,x9,[c1] + ldp x12,x13,[c2] + + ldp x10,x11,[c1,#16] + adds x8,x8,x12 + ldp x14,x15,[c2,#16] + adcs x9,x9,x13 + ldp x4,x5,[c3] + adcs x10,x10,x14 + ldp x6,x7,[c3,#16] + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[c0] + csel x11,x11,x2,lo + stp x10,x11,[c0,#16] + + ret +.size add_mod_256,.-add_mod_256 + +.globl mul_by_3_mod_256 +.hidden mul_by_3_mod_256 +.type mul_by_3_mod_256,%function +.align 5 +mul_by_3_mod_256: + ldp x12,x13,[c1] + ldp x14,x15,[c1,#16] + + adds x8,x12,x12 + ldp x4,x5,[c2] + adcs x9,x13,x13 + ldp x6,x7,[c2,#16] + adcs x10,x14,x14 + adcs x11,x15,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + adds x8,x8,x12 + adcs x9,x9,x13 + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[c0] + csel x11,x11,x2,lo + stp x10,x11,[c0,#16] + + ret +.size mul_by_3_mod_256,.-mul_by_3_mod_256 + +.globl lshift_mod_256 +.hidden lshift_mod_256 +.type lshift_mod_256,%function +.align 5 +lshift_mod_256: + ldp x8,x9,[c1] + ldp x10,x11,[c1,#16] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + +.Loop_lshift_mod_256: + adds x8,x8,x8 + sub x2,x2,#1 + adcs x9,x9,x9 + adcs x10,x10,x10 + adcs x11,x11,x11 + adc x3,xzr,xzr + + subs x12,x8,x4 + sbcs x13,x9,x5 + sbcs x14,x10,x6 + sbcs x15,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x12,lo + csel x9,x9,x13,lo + csel x10,x10,x14,lo + csel x11,x11,x15,lo + + cbnz x2,.Loop_lshift_mod_256 + + stp x8,x9,[c0] + stp x10,x11,[c0,#16] + + ret +.size lshift_mod_256,.-lshift_mod_256 + +.globl rshift_mod_256 +.hidden rshift_mod_256 +.type rshift_mod_256,%function +.align 5 +rshift_mod_256: + ldp x8,x9,[c1] + ldp x10,x11,[c1,#16] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + +.Loop_rshift: + adds x12,x8,x4 + sub x2,x2,#1 + adcs x13,x9,x5 + adcs x14,x10,x6 + adcs x15,x11,x7 + adc x3,xzr,xzr + tst x8,#1 + + csel x12,x12,x8,ne + csel x13,x13,x9,ne + csel x14,x14,x10,ne + csel x15,x15,x11,ne + csel x3,x3,xzr,ne + + extr x8,x13,x12,#1 + extr x9,x14,x13,#1 + extr x10,x15,x14,#1 + extr x11,x3,x15,#1 + + cbnz x2,.Loop_rshift + + stp x8,x9,[c0] + stp x10,x11,[c0,#16] + + ret +.size rshift_mod_256,.-rshift_mod_256 + +.globl cneg_mod_256 +.hidden cneg_mod_256 +.type cneg_mod_256,%function +.align 5 +cneg_mod_256: + ldp x8,x9,[c1] + ldp x4,x5,[c3] + + ldp x10,x11,[c1,#16] + subs x12,x4,x8 + ldp x6,x7,[c3,#16] + orr x4,x8,x9 + sbcs x13,x5,x9 + orr x5,x10,x11 + sbcs x14,x6,x10 + orr x3,x4,x5 + sbc x15,x7,x11 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x8,x8,x12,eq + csel x9,x9,x13,eq + csel x10,x10,x14,eq + stp x8,x9,[c0] + csel x11,x11,x15,eq + stp x10,x11,[c0,#16] + + ret +.size cneg_mod_256,.-cneg_mod_256 + +.globl sub_mod_256 +.hidden sub_mod_256 +.type sub_mod_256,%function +.align 5 +sub_mod_256: + ldp x8,x9,[c1] + ldp x12,x13,[c2] + + ldp x10,x11,[c1,#16] + subs x8,x8,x12 + ldp x14,x15,[c2,#16] + sbcs x9,x9,x13 + ldp x4,x5,[c3] + sbcs x10,x10,x14 + ldp x6,x7,[c3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + stp x8,x9,[c0] + adc x11,x11,x7 + stp x10,x11,[c0,#16] + + ret +.size sub_mod_256,.-sub_mod_256 + +.globl check_mod_256 +.hidden check_mod_256 +.type check_mod_256,%function +.align 5 +check_mod_256: + ldp x8,x9,[c0] + ldp x10,x11,[c0,#16] + ldp x4,x5,[c1] + ldp x6,x7,[c1,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + subs xzr,x8,x4 + sbcs xzr,x9,x5 + orr x8,x8,x9 + sbcs xzr,x10,x6 + orr x8,x8,x10 + sbcs xzr,x11,x7 + orr x8,x8,x11 + sbc x1,xzr,xzr + + cmp x8,#0 + mov x0,#1 + csel x0,x0,xzr,ne + and x0,x0,x1 + + ret +.size check_mod_256,.-check_mod_256 + +.globl add_n_check_mod_256 +.hidden add_n_check_mod_256 +.type add_n_check_mod_256,%function +.align 5 +add_n_check_mod_256: + ldp x8,x9,[c1] + ldp x12,x13,[c2] + ldp x10,x11,[c1,#16] + ldp x14,x15,[c2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + adds x8,x8,x12 + ldp x4,x5,[c3] + adcs x9,x9,x13 + ldp x6,x7,[c3,#16] + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[c0] + stp x10,x11,[c0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret +.size add_n_check_mod_256,.-add_n_check_mod_256 + +.globl sub_n_check_mod_256 +.hidden sub_n_check_mod_256 +.type sub_n_check_mod_256,%function +.align 5 +sub_n_check_mod_256: + ldp x8,x9,[c1] + ldp x12,x13,[c2] + ldp x10,x11,[c1,#16] + ldp x14,x15,[c2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + subs x8,x8,x12 + sbcs x9,x9,x13 + ldp x4,x5,[c3] + sbcs x10,x10,x14 + ldp x6,x7,[c3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + adc x11,x11,x7 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[c0] + stp x10,x11,[c0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret +.size sub_n_check_mod_256,.-sub_n_check_mod_256 diff --git a/c-source/build/cheri/add_mod_384-armv8.S b/c-source/build/cheri/add_mod_384-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/add_mod_384-armv8.S @@ -0,0 +1,1000 @@ +.text + +.globl add_mod_384 +.hidden add_mod_384 +.type add_mod_384,%function +.align 5 +add_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + + bl __add_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size add_mod_384,.-add_mod_384 + +.type __add_mod_384,%function +.align 5 +__add_mod_384: + ldp x10,x11,[c1] + ldp x16,x17,[c2] + ldp x12,x13,[c1,#16] + ldp x19,x20,[c2,#16] + ldp x14,x15,[c1,#32] + ldp x21,x22,[c2,#32] + +__add_mod_384_ab_are_loaded: + adds x10,x10,x16 + adcs x11,x11,x17 + adcs x12,x12,x19 + adcs x13,x13,x20 + adcs x14,x14,x21 + adcs x15,x15,x22 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret +.size __add_mod_384,.-__add_mod_384 + +.globl add_mod_384x +.hidden add_mod_384x +.type add_mod_384x,%function +.align 5 +add_mod_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + + bl __add_mod_384 + + stp x10,x11,[c0] + add c1,c1,#48 + stp x12,x13,[c0,#16] + add c2,c2,#48 + stp x14,x15,[c0,#32] + + bl __add_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0,#48] + stp x12,x13,[c0,#64] + stp x14,x15,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size add_mod_384x,.-add_mod_384x + +.globl rshift_mod_384 +.hidden rshift_mod_384 +.type rshift_mod_384,%function +.align 5 +rshift_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + +.Loop_rshift_mod_384: + sub x2,x2,#1 + bl __rshift_mod_384 + cbnz x2,.Loop_rshift_mod_384 + + ldr c30,[csp,#__SIZEOF_POINTER__] + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size rshift_mod_384,.-rshift_mod_384 + +.type __rshift_mod_384,%function +.align 5 +__rshift_mod_384: + sbfx x22,x10,#0,#1 + and x16,x22,x4 + and x17,x22,x5 + adds x10,x10,x16 + and x19,x22,x6 + adcs x11,x11,x17 + and x20,x22,x7 + adcs x12,x12,x19 + and x21,x22,x8 + adcs x13,x13,x20 + and x22,x22,x9 + adcs x14,x14,x21 + extr x10,x11,x10,#1 // a[0:5] >>= 1 + adcs x15,x15,x22 + extr x11,x12,x11,#1 + adc x22,xzr,xzr + extr x12,x13,x12,#1 + extr x13,x14,x13,#1 + extr x14,x15,x14,#1 + extr x15,x22,x15,#1 + ret +.size __rshift_mod_384,.-__rshift_mod_384 + +.globl div_by_2_mod_384 +.hidden div_by_2_mod_384 +.type div_by_2_mod_384,%function +.align 5 +div_by_2_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + + bl __rshift_mod_384 + + ldr c30,[csp,#__SIZEOF_POINTER__] + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size div_by_2_mod_384,.-div_by_2_mod_384 + +.globl lshift_mod_384 +.hidden lshift_mod_384 +.type lshift_mod_384,%function +.align 5 +lshift_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + +.Loop_lshift_mod_384: + sub x2,x2,#1 + bl __lshift_mod_384 + cbnz x2,.Loop_lshift_mod_384 + + ldr c30,[csp,#__SIZEOF_POINTER__] + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size lshift_mod_384,.-lshift_mod_384 + +.type __lshift_mod_384,%function +.align 5 +__lshift_mod_384: + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret +.size __lshift_mod_384,.-__lshift_mod_384 + +.globl mul_by_3_mod_384 +.hidden mul_by_3_mod_384 +.type mul_by_3_mod_384,%function +.align 5 +mul_by_3_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[c1] + ldp x19,x20,[c1,#16] + ldp x21,x22,[c1,#32] + + bl __add_mod_384_ab_are_loaded + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_by_3_mod_384,.-mul_by_3_mod_384 + +.globl mul_by_8_mod_384 +.hidden mul_by_8_mod_384 +.type mul_by_8_mod_384,%function +.align 5 +mul_by_8_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_by_8_mod_384,.-mul_by_8_mod_384 + +.globl mul_by_3_mod_384x +.hidden mul_by_3_mod_384x +.type mul_by_3_mod_384x,%function +.align 5 +mul_by_3_mod_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[c1] + ldp x19,x20,[c1,#16] + ldp x21,x22,[c1,#32] + + bl __add_mod_384_ab_are_loaded + + stp x10,x11,[c0] + ldp x10,x11,[c1,#48] + stp x12,x13,[c0,#16] + ldp x12,x13,[c1,#64] + stp x14,x15,[c0,#32] + ldp x14,x15,[c1,#80] + + bl __lshift_mod_384 + + ldp x16,x17,[c1,#48] + ldp x19,x20,[c1,#64] + ldp x21,x22,[c1,#80] + + bl __add_mod_384_ab_are_loaded + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0,#48] + stp x12,x13,[c0,#64] + stp x14,x15,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_by_3_mod_384x,.-mul_by_3_mod_384x + +.globl mul_by_8_mod_384x +.hidden mul_by_8_mod_384x +.type mul_by_8_mod_384x,%function +.align 5 +mul_by_8_mod_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + ldp x14,x15,[c1,#32] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + + stp x10,x11,[c0] + ldp x10,x11,[c1,#48] + stp x12,x13,[c0,#16] + ldp x12,x13,[c1,#64] + stp x14,x15,[c0,#32] + ldp x14,x15,[c1,#80] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0,#48] + stp x12,x13,[c0,#64] + stp x14,x15,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_by_8_mod_384x,.-mul_by_8_mod_384x + +.globl cneg_mod_384 +.hidden cneg_mod_384 +.type cneg_mod_384,%function +.align 5 +cneg_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldp x4,x5,[c3] + ldp x12,x13,[c1,#16] + ldp x6,x7,[c3,#16] + + subs x16,x4,x10 + ldp x14,x15,[c1,#32] + ldp x8,x9,[c3,#32] + orr x3,x10,x11 + sbcs x17,x5,x11 + orr x3,x3,x12 + sbcs x19,x6,x12 + orr x3,x3,x13 + sbcs x20,x7,x13 + orr x3,x3,x14 + sbcs x21,x8,x14 + orr x3,x3,x15 + sbc x22,x9,x15 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x10,x10,x16,eq + csel x11,x11,x17,eq + csel x12,x12,x19,eq + csel x13,x13,x20,eq + stp x10,x11,[c0] + csel x14,x14,x21,eq + stp x12,x13,[c0,#16] + csel x15,x15,x22,eq + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size cneg_mod_384,.-cneg_mod_384 + +.globl sub_mod_384 +.hidden sub_mod_384 +.type sub_mod_384,%function +.align 5 +sub_mod_384: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + + bl __sub_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + stp x14,x15,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sub_mod_384,.-sub_mod_384 + +.type __sub_mod_384,%function +.align 5 +__sub_mod_384: + ldp x10,x11,[c1] + ldp x16,x17,[c2] + ldp x12,x13,[c1,#16] + ldp x19,x20,[c2,#16] + ldp x14,x15,[c1,#32] + ldp x21,x22,[c2,#32] + + subs x10,x10,x16 + sbcs x11,x11,x17 + sbcs x12,x12,x19 + sbcs x13,x13,x20 + sbcs x14,x14,x21 + sbcs x15,x15,x22 + sbc x3,xzr,xzr + + and x16,x4,x3 + and x17,x5,x3 + adds x10,x10,x16 + and x19,x6,x3 + adcs x11,x11,x17 + and x20,x7,x3 + adcs x12,x12,x19 + and x21,x8,x3 + adcs x13,x13,x20 + and x22,x9,x3 + adcs x14,x14,x21 + adc x15,x15,x22 + + ret +.size __sub_mod_384,.-__sub_mod_384 + +.globl sub_mod_384x +.hidden sub_mod_384x +.type sub_mod_384x,%function +.align 5 +sub_mod_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x4,x5,[c3] + ldp x6,x7,[c3,#16] + ldp x8,x9,[c3,#32] + + bl __sub_mod_384 + + stp x10,x11,[c0] + add c1,c1,#48 + stp x12,x13,[c0,#16] + add c2,c2,#48 + stp x14,x15,[c0,#32] + + bl __sub_mod_384 + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0,#48] + stp x12,x13,[c0,#64] + stp x14,x15,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sub_mod_384x,.-sub_mod_384x + +.globl mul_by_1_plus_i_mod_384x +.hidden mul_by_1_plus_i_mod_384x +.type mul_by_1_plus_i_mod_384x,%function +.align 5 +mul_by_1_plus_i_mod_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x4,x5,[c2] + ldp x6,x7,[c2,#16] + ldp x8,x9,[c2,#32] + add c2,c1,#48 + + bl __sub_mod_384 // a->re - a->im + + ldp x16,x17,[c1] + ldp x19,x20,[c1,#16] + ldp x21,x22,[c1,#32] + stp x10,x11,[c0] + ldp x10,x11,[c1,#48] + stp x12,x13,[c0,#16] + ldp x12,x13,[c1,#64] + stp x14,x15,[c0,#32] + ldp x14,x15,[c1,#80] + + bl __add_mod_384_ab_are_loaded // a->re + a->im + ldr c30,[csp,#__SIZEOF_POINTER__] + + stp x10,x11,[c0,#48] + stp x12,x13,[c0,#64] + stp x14,x15,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x + +.globl sgn0_pty_mod_384 +.hidden sgn0_pty_mod_384 +.type sgn0_pty_mod_384,%function +.align 5 +sgn0_pty_mod_384: + ldp x10,x11,[c0] + ldp x12,x13,[c0,#16] + ldp x14,x15,[c0,#32] + + ldp x4,x5,[c1] + ldp x6,x7,[c1,#16] + ldp x8,x9,[c1,#32] + + and x0,x10,#1 + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x3,x3,xzr + + mvn x3,x3 + and x3,x3,#2 + orr x0,x0,x3 + + ret +.size sgn0_pty_mod_384,.-sgn0_pty_mod_384 + +.globl sgn0_pty_mod_384x +.hidden sgn0_pty_mod_384x +.type sgn0_pty_mod_384x,%function +.align 5 +sgn0_pty_mod_384x: + ldp x10,x11,[c0] + ldp x12,x13,[c0,#16] + ldp x14,x15,[c0,#32] + + ldp x4,x5,[c1] + ldp x6,x7,[c1,#16] + ldp x8,x9,[c1,#32] + + and x2,x10,#1 + orr x3,x10,x11 + adds x10,x10,x10 + orr x3,x3,x12 + adcs x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + ldp x10,x11,[c0,#48] + ldp x12,x13,[c0,#64] + ldp x14,x15,[c0,#80] + + mvn x16,x16 + and x16,x16,#2 + orr x2,x2,x16 + + and x0,x10,#1 + orr x1,x10,x11 + adds x10,x10,x10 + orr x1,x1,x12 + adcs x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + mvn x16,x16 + and x16,x16,#2 + orr x0,x0,x16 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ret +.size sgn0_pty_mod_384x,.-sgn0_pty_mod_384x +.globl vec_select_32 +.hidden vec_select_32 +.type vec_select_32,%function +.align 5 +vec_select_32: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0] + ret +.size vec_select_32,.-vec_select_32 +.globl vec_select_48 +.hidden vec_select_48 +.type vec_select_48,%function +.align 5 +vec_select_48: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0] + ret +.size vec_select_48,.-vec_select_48 +.globl vec_select_96 +.hidden vec_select_96 +.type vec_select_96,%function +.align 5 +vec_select_96: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0] + ret +.size vec_select_96,.-vec_select_96 +.globl vec_select_192 +.hidden vec_select_192 +.type vec_select_192,%function +.align 5 +vec_select_192: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0] + ret +.size vec_select_192,.-vec_select_192 +.globl vec_select_144 +.hidden vec_select_144 +.type vec_select_144,%function +.align 5 +vec_select_144: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0] + ret +.size vec_select_144,.-vec_select_144 +.globl vec_select_288 +.hidden vec_select_288 +.type vec_select_288,%function +.align 5 +vec_select_288: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [c1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [c2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [c0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [c0] + ret +.size vec_select_288,.-vec_select_288 +.globl vec_prefetch +.hidden vec_prefetch +.type vec_prefetch,%function +.align 5 +vec_prefetch: + add x1, x1, x0 + sub x1, x1, #1 + mov x2, #64 + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [c0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + prfm pldl1keep, [c0] + ret +.size vec_prefetch,.-vec_prefetch +.globl vec_is_zero_16x +.hidden vec_is_zero_16x +.type vec_is_zero_16x,%function +.align 5 +vec_is_zero_16x: + ld1 {v0.2d}, [c0], #16 + lsr x1, x1, #4 + sub x1, x1, #1 + cbz x1, .Loop_is_zero_done + +.Loop_is_zero: + ld1 {v1.2d}, [c0], #16 + orr v0.16b, v0.16b, v1.16b + sub x1, x1, #1 + cbnz x1, .Loop_is_zero + +.Loop_is_zero_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + umov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret +.size vec_is_zero_16x,.-vec_is_zero_16x +.globl vec_is_equal_16x +.hidden vec_is_equal_16x +.type vec_is_equal_16x,%function +.align 5 +vec_is_equal_16x: + ld1 {v0.2d}, [c0], #16 + ld1 {v1.2d}, [c1], #16 + lsr x2, x2, #4 + eor v0.16b, v0.16b, v1.16b + +.Loop_is_equal: + sub x2, x2, #1 + cbz x2, .Loop_is_equal_done + ld1 {v1.2d}, [c0], #16 + ld1 {v2.2d}, [c1], #16 + eor v1.16b, v1.16b, v2.16b + orr v0.16b, v0.16b, v1.16b + b .Loop_is_equal + nop + +.Loop_is_equal_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + umov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret +.size vec_is_equal_16x,.-vec_is_equal_16x diff --git a/c-source/build/cheri/ct_inverse_mod_256-armv8.S b/c-source/build/cheri/ct_inverse_mod_256-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/ct_inverse_mod_256-armv8.S @@ -0,0 +1,880 @@ +.text + +.globl ct_inverse_mod_256 +.hidden ct_inverse_mod_256 +.type ct_inverse_mod_256, %function +.align 5 +ct_inverse_mod_256: + .inst 0xd503233f + stp c29, c30, [csp,#-10*__SIZEOF_POINTER__]! + add c29, csp, #0 + stp c19, c20, [csp,#2*__SIZEOF_POINTER__] + stp c21, c22, [csp,#4*__SIZEOF_POINTER__] + stp c23, c24, [csp,#6*__SIZEOF_POINTER__] + stp c25, c26, [csp,#8*__SIZEOF_POINTER__] + sub csp, csp, #1040 + + ldp x4, x5, [c1,#8*0] + ldp x6, x7, [c1,#8*2] + +#ifdef __CHERI_PURE_CAPABILITY__ + add c1,csp,#16+511 + alignd c1,c1,#9 +#else + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... +#endif + str c0, [csp] // offload out_ptr + + ldp x8, x9, [c2,#8*0] + ldp x10, x11, [c2,#8*2] + + stp x4, x5, [c1,#8*0] // copy input to |a| + stp x6, x7, [c1,#8*2] + stp x8, x9, [c1,#8*4] // copy modulus to |b| + stp x10, x11, [c1,#8*6] + + ////////////////////////////////////////// first iteration + bl .Lab_approximation_31_256_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + str x12,[c0,#8*8] // initialize |u| with |f0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + str x12, [c0,#8*9] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + ldr x8, [c1,#8*8] // |u| + ldr x9, [c1,#8*13] // |v| + madd x4, x16, x8, xzr // |u|*|f0| + madd x4, x17, x9, x4 // |v|*|g0| + str x4, [c0,#8*4] + asr x5, x4, #63 // sign extension + stp x5, x5, [c0,#8*5] + stp x5, x5, [c0,#8*7] + + madd x4, x12, x8, xzr // |u|*|f1| + madd x4, x13, x9, x4 // |v|*|g1| + str x4, [c0,#8*9] + asr x5, x4, #63 // sign extension + stp x5, x5, [c0,#8*10] + stp x5, x5, [c0,#8*12] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [c0,#8*4] + stp x22, x22, [c0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add c0,c0,#8*4 + bl __smul_256_n_shift_by_31 + + add c0,c0,#8*4 + bl __smul_256x63 + adc x22, x22, x23 + str x22, [c0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add c0,c0,#8*5 + bl __smul_256x63 + bl __smul_512x63_tail + ////////////////////////////////////////// two[!] last iterations + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #47 // 31 + 512 % 31 + //bl __ab_approximation_62_256 // |a| and |b| are exact, + ldr x7, [c1,#8*0] // just load + ldr x11, [c1,#8*4] + bl __inner_loop_62_256 + + mov x16, x14 + mov x17, x15 + ldr c0, [csp] // original out_ptr + bl __smul_256x63 + bl __smul_512x63_tail + ldr c30, [c29,#__SIZEOF_POINTER__] + + smulh x20, x7, x17 // figure out top-most limb + ldp x8, x9, [c3,#8*0] + adc x23, x23, x25 + ldp x10, x11, [c3,#8*2] + + add x20, x20, x23 // x20 is 1, 0 or -1 + asr x19, x20, #63 // sign as mask + + and x23, x8, x19 // add mod<<256 conditionally + and x24, x9, x19 + adds x4, x4, x23 + and x25, x10, x19 + adcs x5, x5, x24 + and x26, x11, x19 + adcs x6, x6, x25 + adcs x7, x22, x26 + adc x20, x20, xzr // x20 is 1, 0 or -1 + + neg x19, x20 + orr x20, x20, x19 // excess bit or sign as mask + asr x19, x19, #63 // excess bit as mask + + and x8, x8, x20 // mask |mod| + and x9, x9, x20 + and x10, x10, x20 + and x11, x11, x20 + + eor x8, x8, x19 // conditionally negate |mod| + eor x9, x9, x19 + adds x8, x8, x19, lsr#63 + eor x10, x10, x19 + adcs x9, x9, xzr + eor x11, x11, x19 + adcs x10, x10, xzr + adc x11, x11, xzr + + adds x4, x4, x8 // final adjustment for |mod|<<256 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [c0,#8*4] + adc x7, x7, x11 + stp x6, x7, [c0,#8*6] + + add csp, csp, #1040 + ldp c19, c20, [c29,#2*__SIZEOF_POINTER__] + ldp c21, c22, [c29,#4*__SIZEOF_POINTER__] + ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] + ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] + ldr c29, [csp],#10*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size ct_inverse_mod_256,.-ct_inverse_mod_256 + +//////////////////////////////////////////////////////////////////////// +.type __smul_256x63, %function +.align 5 +__smul_256x63: + ldp x4, x5, [c1,#8*0+64] // load |u| (or |v|) + asr x14, x16, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x6, x7, [c1,#8*2+64] + eor x16, x16, x14 // conditionally negate |f_| (or |g_|) + ldr x22, [c1,#8*4+64] + + eor x4, x4, x14 // conditionally negate |u| (or |v|) + sub x16, x16, x14 + eor x5, x5, x14 + adds x4, x4, x14, lsr#63 + eor x6, x6, x14 + adcs x5, x5, xzr + eor x7, x7, x14 + adcs x6, x6, xzr + eor x22, x22, x14 + umulh x19, x4, x16 + adcs x7, x7, xzr + umulh x20, x5, x16 + adcs x22, x22, xzr + umulh x21, x6, x16 + mul x4, x4, x16 + cmp x16, #0 + mul x5, x5, x16 + csel x22, x22, xzr, ne + mul x6, x6, x16 + adds x5, x5, x19 + mul x24, x7, x16 + adcs x6, x6, x20 + adcs x24, x24, x21 + adc x26, xzr, xzr + ldp x8, x9, [c1,#8*0+104] // load |u| (or |v|) + asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x10, x11, [c1,#8*2+104] + eor x17, x17, x14 // conditionally negate |f_| (or |g_|) + ldr x23, [c1,#8*4+104] + + eor x8, x8, x14 // conditionally negate |u| (or |v|) + sub x17, x17, x14 + eor x9, x9, x14 + adds x8, x8, x14, lsr#63 + eor x10, x10, x14 + adcs x9, x9, xzr + eor x11, x11, x14 + adcs x10, x10, xzr + eor x23, x23, x14 + umulh x19, x8, x17 + adcs x11, x11, xzr + umulh x20, x9, x17 + adcs x23, x23, xzr + umulh x21, x10, x17 + adc x15, xzr, xzr // used in __smul_512x63_tail + mul x8, x8, x17 + cmp x17, #0 + mul x9, x9, x17 + csel x23, x23, xzr, ne + mul x10, x10, x17 + adds x9, x9, x19 + mul x25, x11, x17 + adcs x10, x10, x20 + adcs x25, x25, x21 + adc x26, x26, xzr + + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [c0,#8*0] + adcs x24, x24, x25 + stp x6, x24, [c0,#8*2] + + ret +.size __smul_256x63,.-__smul_256x63 + +.type __smul_512x63_tail, %function +.align 5 +__smul_512x63_tail: + umulh x24, x7, x16 + ldp x5, x6, [c1,#8*18] // load rest of |v| + adc x26, x26, xzr + ldr x7, [c1,#8*20] + and x22, x22, x16 + + umulh x11, x11, x17 // resume |v|*|g1| chain + + sub x24, x24, x22 // tie up |u|*|f1| chain + asr x25, x24, #63 + + eor x5, x5, x14 // conditionally negate rest of |v| + eor x6, x6, x14 + adds x5, x5, x15 + eor x7, x7, x14 + adcs x6, x6, xzr + umulh x19, x23, x17 + adc x7, x7, xzr + umulh x20, x5, x17 + add x11, x11, x26 + umulh x21, x6, x17 + + mul x4, x23, x17 + mul x5, x5, x17 + adds x4, x4, x11 + mul x6, x6, x17 + adcs x5, x5, x19 + mul x22, x7, x17 + adcs x6, x6, x20 + adcs x22, x22, x21 + adc x23, xzr, xzr // used in the final step + + adds x4, x4, x24 + adcs x5, x5, x25 + adcs x6, x6, x25 + stp x4, x5, [c0,#8*4] + adcs x22, x22, x25 // carry is used in the final step + stp x6, x22, [c0,#8*6] + + ret +.size __smul_512x63_tail,.-__smul_512x63_tail + +.type __smul_256_n_shift_by_31, %function +.align 5 +__smul_256_n_shift_by_31: + ldp x4, x5, [c1,#8*0+0] // load |a| (or |b|) + asr x24, x12, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x6, x7, [c1,#8*2+0] + eor x25, x12, x24 // conditionally negate |f0| (or |g0|) + + eor x4, x4, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x5, x5, x24 + adds x4, x4, x24, lsr#63 + eor x6, x6, x24 + adcs x5, x5, xzr + eor x7, x7, x24 + umulh x19, x4, x25 + adcs x6, x6, xzr + umulh x20, x5, x25 + adc x7, x7, xzr + umulh x21, x6, x25 + and x24, x24, x25 + umulh x22, x7, x25 + neg x24, x24 + + mul x4, x4, x25 + mul x5, x5, x25 + mul x6, x6, x25 + adds x5, x5, x19 + mul x7, x7, x25 + adcs x6, x6, x20 + adcs x7, x7, x21 + adc x22, x22, x24 + ldp x8, x9, [c1,#8*0+32] // load |a| (or |b|) + asr x24, x13, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x10, x11, [c1,#8*2+32] + eor x25, x13, x24 // conditionally negate |f0| (or |g0|) + + eor x8, x8, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x9, x9, x24 + adds x8, x8, x24, lsr#63 + eor x10, x10, x24 + adcs x9, x9, xzr + eor x11, x11, x24 + umulh x19, x8, x25 + adcs x10, x10, xzr + umulh x20, x9, x25 + adc x11, x11, xzr + umulh x21, x10, x25 + and x24, x24, x25 + umulh x23, x11, x25 + neg x24, x24 + + mul x8, x8, x25 + mul x9, x9, x25 + mul x10, x10, x25 + adds x9, x9, x19 + mul x11, x11, x25 + adcs x10, x10, x20 + adcs x11, x11, x21 + adc x23, x23, x24 + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + adcs x7, x7, x11 + adc x8, x22, x23 + + extr x4, x5, x4, #31 + extr x5, x6, x5, #31 + extr x6, x7, x6, #31 + asr x23, x8, #63 // result's sign as mask + extr x7, x8, x7, #31 + + eor x4, x4, x23 // ensure the result is positive + eor x5, x5, x23 + adds x4, x4, x23, lsr#63 + eor x6, x6, x23 + adcs x5, x5, xzr + eor x7, x7, x23 + adcs x6, x6, xzr + stp x4, x5, [c0,#8*0] + adc x7, x7, xzr + stp x6, x7, [c0,#8*2] + + eor x12, x12, x23 // adjust |f/g| accordingly + eor x13, x13, x23 + sub x12, x12, x23 + sub x13, x13, x23 + + ret +.size __smul_256_n_shift_by_31,.-__smul_256_n_shift_by_31 +.type __ab_approximation_31_256, %function +.align 4 +__ab_approximation_31_256: + ldp x6, x7, [c1,#8*2] + ldp x10, x11, [c1,#8*6] + ldp x4, x5, [c1,#8*0] + ldp x8, x9, [c1,#8*4] + +.Lab_approximation_31_256_loaded: + orr x19, x7, x11 // check top-most limbs, ... + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x5, ne + orr x19, x7, x11 // and ones before top-most, ... + csel x10, x10, x9, ne + + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x4, ne + orr x19, x7, x11 // and one more, ... + csel x10, x10, x8, ne + + clz x19, x19 + cmp x19, #64 + csel x19, x19, xzr, ne + csel x7, x7, x6, ne + csel x11, x11, x10, ne + neg x20, x19 + + lslv x7, x7, x19 // align high limbs to the left + lslv x11, x11, x19 + lsrv x6, x6, x20 + lsrv x10, x10, x20 + and x6, x6, x20, asr#6 + and x10, x10, x20, asr#6 + orr x7, x7, x6 + orr x11, x11, x10 + + bfxil x7, x4, #0, #31 + bfxil x11, x8, #0, #31 + + b __inner_loop_31_256 + ret +.size __ab_approximation_31_256,.-__ab_approximation_31_256 + +.type __inner_loop_31_256, %function +.align 4 +__inner_loop_31_256: + mov x2, #31 + mov x13, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x15, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x23,#0x7FFFFFFF7FFFFFFF + +.Loop_31_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x15 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x15, x15, x13, hs // exchange |fg0| and |fg1| + csel x13, x13, x19, hs + lsr x7, x7, #1 + and x19, x15, x22 + and x20, x23, x22 + sub x13, x13, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x15, x15, x15 // |f1|<<=1 + add x13, x13, x20 + sub x15, x15, x23 + cbnz x2, .Loop_31_256 + + mov x23, #0x7FFFFFFF + ubfx x12, x13, #0, #32 + ubfx x13, x13, #32, #32 + ubfx x14, x15, #0, #32 + ubfx x15, x15, #32, #32 + sub x12, x12, x23 // remove bias + sub x13, x13, x23 + sub x14, x14, x23 + sub x15, x15, x23 + + ret +.size __inner_loop_31_256,.-__inner_loop_31_256 + +.type __inner_loop_62_256, %function +.align 4 +__inner_loop_62_256: + mov x12, #1 // |f0|=1 + mov x13, #0 // |g0|=0 + mov x14, #0 // |f1|=0 + mov x15, #1 // |g1|=1 + +.Loop_62_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x12 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + mov x20, x13 + csel x12, x12, x14, hs // exchange |f0| and |f1| + csel x14, x14, x19, hs + csel x13, x13, x15, hs // exchange |g0| and |g1| + csel x15, x15, x20, hs + lsr x7, x7, #1 + and x19, x14, x22 + and x20, x15, x22 + add x14, x14, x14 // |f1|<<=1 + add x15, x15, x15 // |g1|<<=1 + sub x12, x12, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x13, x13, x20 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62_256 + + ret +.size __inner_loop_62_256,.-__inner_loop_62_256 diff --git a/c-source/build/cheri/ct_inverse_mod_384-armv8.S b/c-source/build/cheri/ct_inverse_mod_384-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/ct_inverse_mod_384-armv8.S @@ -0,0 +1,795 @@ +.text + +.globl ct_inverse_mod_383 +.hidden ct_inverse_mod_383 +.type ct_inverse_mod_383, %function +.align 5 +ct_inverse_mod_383: + .inst 0xd503233f + stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! + add c29, csp, #0 + stp c19, c20, [csp,#2*__SIZEOF_POINTER__] + stp c21, c22, [csp,#4*__SIZEOF_POINTER__] + stp c23, c24, [csp,#6*__SIZEOF_POINTER__] + stp c25, c26, [csp,#8*__SIZEOF_POINTER__] + stp c27, c28, [csp,#10*__SIZEOF_POINTER__] + sub csp, csp, #1056 + + ldp x22, x4, [c1,#8*0] + ldp x5, x6, [c1,#8*2] + ldp x7, x8, [c1,#8*4] + +#ifdef __CHERI_PURE_CAPABILITY__ + add c1,csp,#32+511 + alignd c1,c1,#9 +#else + add x1, sp, #32+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... +#endif + stp c0, c3, [csp] // offload out_ptr, nx_ptr + + ldp x9, x10, [c2,#8*0] + ldp x11, x12, [c2,#8*2] + ldp x13, x14, [c2,#8*4] + + stp x22, x4, [c1,#8*0] // copy input to |a| + stp x5, x6, [c1,#8*2] + stp x7, x8, [c1,#8*4] + stp x9, x10, [c1,#8*6] // copy modulus to |b| + stp x11, x12, [c1,#8*8] + stp x13, x14, [c1,#8*10] + + ////////////////////////////////////////// first iteration + mov x2, #62 + bl .Lab_approximation_62_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + str x15,[c0,#8*12] // initialize |u| with |f0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + str x15, [c0,#8*12] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + ldr x7, [c1,#8*12] // |u| + ldr x8, [c1,#8*18] // |v| + mul x3, x20, x7 // |u|*|f0| + smulh x4, x20, x7 + mul x5, x21, x8 // |v|*|g0| + smulh x6, x21, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [c0,#8*6] + asr x5, x4, #63 // sign extension + stp x5, x5, [c0,#8*8] + stp x5, x5, [c0,#8*10] + + mul x3, x15, x7 // |u|*|f1| + smulh x4, x15, x7 + mul x5, x16, x8 // |v|*|g1| + smulh x6, x16, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [c0,#8*12] + asr x5, x4, #63 // sign extension + stp x5, x5, [c0,#8*14] + stp x5, x5, [c0,#8*16] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + asr x27, x27, #63 // sign extension + stp x27, x27, [c0,#8*6] + stp x27, x27, [c0,#8*8] + stp x27, x27, [c0,#8*10] + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add c0,c0,#8*6 + bl __smul_383_n_shift_by_62 + + add c0,c0,#8*6 + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + ////////////////////////////////////////// iteration before last + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldp x3, x8, [c1,#8*0] // just load + ldp x9, x14, [c1,#8*6] + bl __inner_loop_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + str x3, [c0,#8*0] + str x9, [c0,#8*6] + + mov x20, x15 // exact |f0| + mov x21, x16 // exact |g0| + mov x15, x17 + mov x16, x19 + add c0,c0,#8*12 + bl __smul_383x63 + + mov x20, x15 // exact |f1| + mov x21, x16 // exact |g1| + add c0,c0,#8*6 + bl __smul_383x63 + bl __smul_767x63_tail + + ////////////////////////////////////////// last iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + mov x2, #22 // 766 % 62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldr x3, [c1,#8*0] // just load + eor x8, x8, x8 + ldr x9, [c1,#8*6] + eor x14, x14, x14 + bl __inner_loop_62 + + mov x20, x17 + mov x21, x19 + ldp c0, c15, [csp] // original out_ptr and n_ptr + bl __smul_383x63 + bl __smul_767x63_tail + ldr c30, [c29,#__SIZEOF_POINTER__] + + asr x22, x8, #63 // sign as mask + ldp x9, x10, [c15,#8*0] + ldp x11, x12, [c15,#8*2] + ldp x13, x14, [c15,#8*4] + + and x9, x9, x22 // add mod<<384 conditionally + and x10, x10, x22 + adds x3, x3, x9 + and x11, x11, x22 + adcs x4, x4, x10 + and x12, x12, x22 + adcs x5, x5, x11 + and x13, x13, x22 + adcs x6, x6, x12 + and x14, x14, x22 + stp x3, x4, [c0,#8*6] + adcs x7, x7, x13 + stp x5, x6, [c0,#8*8] + adc x8, x8, x14 + stp x7, x8, [c0,#8*10] + + add csp, csp, #1056 + ldp c19, c20, [c29,#2*__SIZEOF_POINTER__] + ldp c21, c22, [c29,#4*__SIZEOF_POINTER__] + ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] + ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] + ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] + ldr c29, [csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size ct_inverse_mod_383,.-ct_inverse_mod_383 + +//////////////////////////////////////////////////////////////////////// +// see corresponding commentary in ctx_inverse_mod_384-x86_64... +.type __smul_383x63, %function +.align 5 +__smul_383x63: + ldp x3, x4, [c1,#8*0+96] // load |u| (or |v|) + asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x5, x6, [c1,#8*2+96] + eor x20, x20, x17 // conditionally negate |f_| (or |g_|) + ldp x7, x8, [c1,#8*4+96] + + eor x3, x3, x17 // conditionally negate |u| (or |v|) + sub x20, x20, x17 + eor x4, x4, x17 + adds x3, x3, x17, lsr#63 + eor x5, x5, x17 + adcs x4, x4, xzr + eor x6, x6, x17 + adcs x5, x5, xzr + eor x7, x7, x17 + adcs x6, x6, xzr + umulh x22, x3, x20 + eor x8, x8, x17 + umulh x23, x4, x20 + adcs x7, x7, xzr + umulh x24, x5, x20 + adcs x8, x8, xzr + umulh x25, x6, x20 + umulh x26, x7, x20 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x22 + mul x6, x6, x20 + adcs x5, x5, x23 + mul x7, x7, x20 + adcs x6, x6, x24 + mul x27,x8, x20 + adcs x7, x7, x25 + adcs x27,x27,x26 + adc x2, xzr, xzr + ldp x9, x10, [c1,#8*0+144] // load |u| (or |v|) + asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x11, x12, [c1,#8*2+144] + eor x21, x21, x17 // conditionally negate |f_| (or |g_|) + ldp x13, x14, [c1,#8*4+144] + + eor x9, x9, x17 // conditionally negate |u| (or |v|) + sub x21, x21, x17 + eor x10, x10, x17 + adds x9, x9, x17, lsr#63 + eor x11, x11, x17 + adcs x10, x10, xzr + eor x12, x12, x17 + adcs x11, x11, xzr + eor x13, x13, x17 + adcs x12, x12, xzr + umulh x22, x9, x21 + eor x14, x14, x17 + umulh x23, x10, x21 + adcs x13, x13, xzr + umulh x24, x11, x21 + adcs x14, x14, xzr + umulh x25, x12, x21 + adc x19, xzr, xzr // used in __smul_767x63_tail + umulh x26, x13, x21 + mul x9, x9, x21 + mul x10, x10, x21 + mul x11, x11, x21 + adds x10, x10, x22 + mul x12, x12, x21 + adcs x11, x11, x23 + mul x13, x13, x21 + adcs x12, x12, x24 + mul x28,x14, x21 + adcs x13, x13, x25 + adcs x28,x28,x26 + adc x2, x2, xzr + + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + stp x3, x4, [c0,#8*0] + adcs x7, x7, x13 + stp x5, x6, [c0,#8*2] + adcs x27, x27, x28 + stp x7, x27, [c0,#8*4] + adc x28, x2, xzr // used in __smul_767x63_tail + + ret +.size __smul_383x63,.-__smul_383x63 + +.type __smul_767x63_tail, %function +.align 5 +__smul_767x63_tail: + smulh x27, x8, x20 + ldp x3, x4, [c1,#8*24] // load rest of |v| + umulh x14,x14, x21 + ldp x5, x6, [c1,#8*26] + ldp x7, x8, [c1,#8*28] + + eor x3, x3, x17 // conditionally negate rest of |v| + eor x4, x4, x17 + eor x5, x5, x17 + adds x3, x3, x19 + eor x6, x6, x17 + adcs x4, x4, xzr + eor x7, x7, x17 + adcs x5, x5, xzr + eor x8, x8, x17 + adcs x6, x6, xzr + umulh x22, x3, x21 + adcs x7, x7, xzr + umulh x23, x4, x21 + adc x8, x8, xzr + + umulh x24, x5, x21 + add x14, x14, x28 + umulh x25, x6, x21 + asr x28, x27, #63 + umulh x26, x7, x21 + mul x3, x3, x21 + mul x4, x4, x21 + mul x5, x5, x21 + adds x3, x3, x14 + mul x6, x6, x21 + adcs x4, x4, x22 + mul x7, x7, x21 + adcs x5, x5, x23 + mul x8, x8, x21 + adcs x6, x6, x24 + adcs x7, x7, x25 + adc x8, x8, x26 + + adds x3, x3, x27 + adcs x4, x4, x28 + adcs x5, x5, x28 + adcs x6, x6, x28 + stp x3, x4, [c0,#8*6] + adcs x7, x7, x28 + stp x5, x6, [c0,#8*8] + adc x8, x8, x28 + stp x7, x8, [c0,#8*10] + + ret +.size __smul_767x63_tail,.-__smul_767x63_tail + +.type __smul_383_n_shift_by_62, %function +.align 5 +__smul_383_n_shift_by_62: + ldp x3, x4, [c1,#8*0+0] // load |a| (or |b|) + asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x5, x6, [c1,#8*2+0] + eor x2, x15, x28 // conditionally negate |f0| (or |g0|) + ldp x7, x8, [c1,#8*4+0] + + eor x3, x3, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + umulh x22, x3, x2 + adcs x6, x6, xzr + umulh x23, x4, x2 + eor x8, x8, x28 + umulh x24, x5, x2 + adcs x7, x7, xzr + umulh x25, x6, x2 + adc x8, x8, xzr + + umulh x26, x7, x2 + smulh x27, x8, x2 + mul x3, x3, x2 + mul x4, x4, x2 + mul x5, x5, x2 + adds x4, x4, x22 + mul x6, x6, x2 + adcs x5, x5, x23 + mul x7, x7, x2 + adcs x6, x6, x24 + mul x8, x8, x2 + adcs x7, x7, x25 + adcs x8, x8 ,x26 + adc x27, x27, xzr + ldp x9, x10, [c1,#8*0+48] // load |a| (or |b|) + asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x11, x12, [c1,#8*2+48] + eor x2, x16, x28 // conditionally negate |f0| (or |g0|) + ldp x13, x14, [c1,#8*4+48] + + eor x9, x9, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x10, x10, x28 + adds x9, x9, x28, lsr#63 + eor x11, x11, x28 + adcs x10, x10, xzr + eor x12, x12, x28 + adcs x11, x11, xzr + eor x13, x13, x28 + umulh x22, x9, x2 + adcs x12, x12, xzr + umulh x23, x10, x2 + eor x14, x14, x28 + umulh x24, x11, x2 + adcs x13, x13, xzr + umulh x25, x12, x2 + adc x14, x14, xzr + + umulh x26, x13, x2 + smulh x28, x14, x2 + mul x9, x9, x2 + mul x10, x10, x2 + mul x11, x11, x2 + adds x10, x10, x22 + mul x12, x12, x2 + adcs x11, x11, x23 + mul x13, x13, x2 + adcs x12, x12, x24 + mul x14, x14, x2 + adcs x13, x13, x25 + adcs x14, x14 ,x26 + adc x28, x28, xzr + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x27, x28 + + extr x3, x4, x3, #62 + extr x4, x5, x4, #62 + extr x5, x6, x5, #62 + asr x28, x9, #63 + extr x6, x7, x6, #62 + extr x7, x8, x7, #62 + extr x8, x9, x8, #62 + + eor x3, x3, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + adcs x6, x6, xzr + eor x8, x8, x28 + stp x3, x4, [c0,#8*0] + adcs x7, x7, xzr + stp x5, x6, [c0,#8*2] + adc x8, x8, xzr + stp x7, x8, [c0,#8*4] + + eor x15, x15, x28 + eor x16, x16, x28 + sub x15, x15, x28 + sub x16, x16, x28 + + ret +.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.type __ab_approximation_62, %function +.align 4 +__ab_approximation_62: + ldp x7, x8, [c1,#8*4] + ldp x13, x14, [c1,#8*10] + ldp x5, x6, [c1,#8*2] + ldp x11, x12, [c1,#8*8] + +.Lab_approximation_62_loaded: + orr x22, x8, x14 // check top-most limbs, ... + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x22, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + ldp x3, x4, [c1,#8*0] + ldp x9, x10, [c1,#8*6] + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x22, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x22, x8, x14 + csel x13, x13, x10, ne + + clz x22, x22 + cmp x22, #64 + csel x22, x22, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x23, x22 + + lslv x8, x8, x22 // align high limbs to the left + lslv x14, x14, x22 + lsrv x7, x7, x23 + lsrv x13, x13, x23 + and x7, x7, x23, asr#6 + and x13, x13, x23, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + b __inner_loop_62 + ret +.size __ab_approximation_62,.-__ab_approximation_62 +.type __inner_loop_62, %function +.align 4 +__inner_loop_62: + mov x15, #1 // |f0|=1 + mov x16, #0 // |g0|=0 + mov x17, #0 // |f1|=0 + mov x19, #1 // |g1|=1 + +.Loop_62: + sbfx x28, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + subs x24, x9, x3 // |b_|-|a_| + and x22, x9, x28 + sbc x25, x14, x8 + and x23, x14, x28 + subs x26, x3, x22 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x22, x15 + sbcs x27, x8, x23 + mov x23, x16 + csel x9, x9, x3, hs // |b_| = |a_| + csel x14, x14, x8, hs + csel x3, x26, x24, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x8, x27, x25, hs + csel x15, x15, x17, hs // exchange |f0| and |f1| + csel x17, x17, x22, hs + csel x16, x16, x19, hs // exchange |g0| and |g1| + csel x19, x19, x23, hs + extr x3, x8, x3, #1 + lsr x8, x8, #1 + and x22, x17, x28 + and x23, x19, x28 + add x17, x17, x17 // |f1|<<=1 + add x19, x19, x19 // |g1|<<=1 + sub x15, x15, x22 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x16, x16, x23 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62 + + ret +.size __inner_loop_62,.-__inner_loop_62 diff --git a/c-source/build/cheri/ct_is_square_mod_384-armv8.S b/c-source/build/cheri/ct_is_square_mod_384-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/ct_is_square_mod_384-armv8.S @@ -0,0 +1,334 @@ +.text + +.globl ct_is_square_mod_384 +.hidden ct_is_square_mod_384 +.type ct_is_square_mod_384, %function +.align 5 +ct_is_square_mod_384: + .inst 0xd503233f + stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! + add c29, csp, #0 + stp c19, c20, [csp,#2*__SIZEOF_POINTER__] + stp c21, c22, [csp,#4*__SIZEOF_POINTER__] + stp c23, c24, [csp,#6*__SIZEOF_POINTER__] + stp c25, c26, [csp,#8*__SIZEOF_POINTER__] + stp c27, c28, [csp,#10*__SIZEOF_POINTER__] + sub csp, csp, #512 + + ldp x3, x4, [c0,#8*0] // load input + ldp x5, x6, [c0,#8*2] + ldp x7, x8, [c0,#8*4] + + add x0, sp, #255 // find closest 256-byte-aligned spot + and x0, x0, #-256 // in the frame... +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + + ldp x9, x10, [c1,#8*0] // load modulus + ldp x11, x12, [c1,#8*2] + ldp x13, x14, [c1,#8*4] + + stp x3, x4, [c0,#8*6] // copy input to |a| + stp x5, x6, [c0,#8*8] + stp x7, x8, [c0,#8*10] + stp x9, x10, [c0,#8*0] // copy modulus to |b| + stp x11, x12, [c0,#8*2] + stp x13, x14, [c0,#8*4] + + eor x2, x2, x2 // init the .Legendre symbol + mov x15, #24 // 24 is 768/30-1 + b .Loop_is_square + +.align 4 +.Loop_is_square: + bl __ab_approximation_30 + sub x15, x15, #1 + + eor x1, x0, #128 // pointer to dst |b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif + bl __smul_384_n_shift_by_30 + + mov x19, x16 // |f0| + mov x20, x17 // |g0| + add c1,c1,#8*6 + bl __smul_384_n_shift_by_30 + + ldp x9, x10, [c1,#-8*6] + eor x0, x0, #128 // flip-flop src |a|b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif + and x27, x27, x9 // if |a| was negative, + add x2, x2, x27, lsr#1 // adjust |L| + + cbnz x15, .Loop_is_square + + ////////////////////////////////////////// last iteration + //bl __ab_approximation_30 // |a| and |b| are exact, + //ldr x8, [x0,#8*6] // and loaded + //ldr x14, [x0,#8*0] + mov x15, #48 // 48 is 768%30 + 30 + bl __inner_loop_48 + ldr c30, [c29,#__SIZEOF_POINTER__] + + and x0, x2, #1 + eor x0, x0, #1 + + add csp, csp, #512 + ldp c19, c20, [c29,#2*__SIZEOF_POINTER__] + ldp c21, c22, [c29,#4*__SIZEOF_POINTER__] + ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] + ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] + ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] + ldr c29, [csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size ct_is_square_mod_384,.-ct_is_square_mod_384 + +.type __smul_384_n_shift_by_30, %function +.align 5 +__smul_384_n_shift_by_30: + ldp x3, x4, [c0,#8*0+0] // load |b| (or |a|) + asr x27, x20, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x5, x6, [c0,#8*2+0] + eor x20, x20, x27 // conditionally negate |g1| (or |f1|) + ldp x7, x8, [c0,#8*4+0] + + eor x3, x3, x27 // conditionally negate |b| (or |a|) + sub x20, x20, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + umulh x21, x3, x20 + adcs x6, x6, xzr + umulh x22, x4, x20 + eor x8, x8, x27 + umulh x23, x5, x20 + adcs x7, x7, xzr + umulh x24, x6, x20 + adc x8, x8, xzr + + umulh x25, x7, x20 + and x28, x20, x27 + umulh x26, x8, x20 + neg x28, x28 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x21 + mul x6, x6, x20 + adcs x5, x5, x22 + mul x7, x7, x20 + adcs x6, x6, x23 + mul x8, x8, x20 + adcs x7, x7, x24 + adcs x8, x8 ,x25 + adc x26, x26, x28 + ldp x9, x10, [c0,#8*0+48] // load |b| (or |a|) + asr x27, x19, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x11, x12, [c0,#8*2+48] + eor x19, x19, x27 // conditionally negate |g1| (or |f1|) + ldp x13, x14, [c0,#8*4+48] + + eor x9, x9, x27 // conditionally negate |b| (or |a|) + sub x19, x19, x27 + eor x10, x10, x27 + adds x9, x9, x27, lsr#63 + eor x11, x11, x27 + adcs x10, x10, xzr + eor x12, x12, x27 + adcs x11, x11, xzr + eor x13, x13, x27 + umulh x21, x9, x19 + adcs x12, x12, xzr + umulh x22, x10, x19 + eor x14, x14, x27 + umulh x23, x11, x19 + adcs x13, x13, xzr + umulh x24, x12, x19 + adc x14, x14, xzr + + umulh x25, x13, x19 + and x28, x19, x27 + umulh x27, x14, x19 + neg x28, x28 + mul x9, x9, x19 + mul x10, x10, x19 + mul x11, x11, x19 + adds x10, x10, x21 + mul x12, x12, x19 + adcs x11, x11, x22 + mul x13, x13, x19 + adcs x12, x12, x23 + mul x14, x14, x19 + adcs x13, x13, x24 + adcs x14, x14 ,x25 + adc x27, x27, x28 + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x26, x27 + + extr x3, x4, x3, #30 + extr x4, x5, x4, #30 + extr x5, x6, x5, #30 + asr x27, x9, #63 + extr x6, x7, x6, #30 + extr x7, x8, x7, #30 + extr x8, x9, x8, #30 + + eor x3, x3, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + adcs x6, x6, xzr + eor x8, x8, x27 + stp x3, x4, [c1,#8*0] + adcs x7, x7, xzr + stp x5, x6, [c1,#8*2] + adc x8, x8, xzr + stp x7, x8, [c1,#8*4] + + ret +.size __smul_384_n_shift_by_30,.-__smul_384_n_shift_by_30 +.type __ab_approximation_30, %function +.align 4 +__ab_approximation_30: + ldp x13, x14, [c0,#8*4] // |a| is still in registers + ldp x11, x12, [c0,#8*2] + + orr x21, x8, x14 // check top-most limbs, ... + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x21, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x21, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x21, x8, x14 // and one more, ... + csel x13, x13, x10, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x3, ne + orr x21, x8, x14 + csel x13, x13, x9, ne + + clz x21, x21 + cmp x21, #64 + csel x21, x21, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x22, x21 + + lslv x8, x8, x21 // align high limbs to the left + lslv x14, x14, x21 + lsrv x7, x7, x22 + lsrv x13, x13, x22 + and x7, x7, x22, asr#6 + and x13, x13, x22, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + bfxil x8, x3, #0, #32 + bfxil x14, x9, #0, #32 + + b __inner_loop_30 + ret +.size __ab_approximation_30,.-__ab_approximation_30 + +.type __inner_loop_30, %function +.align 4 +__inner_loop_30: + mov x28, #30 + mov x17, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x20, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x27,#0x7FFFFFFF7FFFFFFF + +.Loop_30: + sbfx x24, x8, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x8, x14 + sub x28, x28, #1 + and x21, x14, x24 + + sub x22, x14, x8 // |b_|-|a_| + subs x23, x8, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 // L + (a_ & b_) >> 1 + mov x21, x20 + csel x14, x14, x8, hs // |b_| = |a_| + csel x8, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x20, x20, x17, hs // exchange |fg0| and |fg1| + csel x17, x17, x21, hs + csel x2, x2, x25, hs + lsr x8, x8, #1 + and x21, x20, x24 + and x22, x27, x24 + add x23, x14, #2 + sub x17, x17, x21 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x20, x20, x20 // |f1|<<=1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + add x17, x17, x22 + sub x20, x20, x27 + + cbnz x28, .Loop_30 + + mov x27, #0x7FFFFFFF + ubfx x16, x17, #0, #32 + ubfx x17, x17, #32, #32 + ubfx x19, x20, #0, #32 + ubfx x20, x20, #32, #32 + sub x16, x16, x27 // remove the bias + sub x17, x17, x27 + sub x19, x19, x27 + sub x20, x20, x27 + + ret +.size __inner_loop_30,.-__inner_loop_30 +.type __inner_loop_48, %function +.align 4 +__inner_loop_48: +.Loop_48: + sbfx x24, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x3, x9 + sub x15, x15, #1 + and x21, x9, x24 + sub x22, x9, x3 // |b_|-|a_| + subs x23, x3, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 + csel x9, x9, x3, hs // |b_| = |a_| + csel x3, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x2, x2, x25, hs + add x23, x9, #2 + lsr x3, x3, #1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + + cbnz x15, .Loop_48 + + ret +.size __inner_loop_48,.-__inner_loop_48 diff --git a/c-source/build/cheri/div3w-armv8.S b/c-source/build/cheri/div3w-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/div3w-armv8.S @@ -0,0 +1,91 @@ +.text + +.globl div_3_limbs +.hidden div_3_limbs +.type div_3_limbs,%function +.align 5 +div_3_limbs: + ldp x4,x5,[c0] // load R + eor x0,x0,x0 // Q = 0 + mov x3,#64 // loop counter + nop + +.Loop: + subs x6,x4,x1 // R - D + add x0,x0,x0 // Q <<= 1 + sbcs x7,x5,x2 + add x0,x0,#1 // Q + speculative bit + csel x4,x4,x6,lo // select between R and R - D + extr x1,x2,x1,#1 // D >>= 1 + csel x5,x5,x7,lo + lsr x2,x2,#1 + sbc x0,x0,xzr // subtract speculative bit + sub x3,x3,#1 + cbnz x3,.Loop + + asr x3,x0,#63 // top bit -> mask + add x0,x0,x0 // Q <<= 1 + subs x6,x4,x1 // R - D + add x0,x0,#1 // Q + speculative bit + sbcs x7,x5,x2 + sbc x0,x0,xzr // subtract speculative bit + + orr x0,x0,x3 // all ones if overflow + + ret +.size div_3_limbs,.-div_3_limbs +.globl quot_rem_128 +.hidden quot_rem_128 +.type quot_rem_128,%function +.align 5 +quot_rem_128: + ldp x3,x4,[c1] + + mul x5,x3,x2 // divisor[0:1} * quotient + umulh x6,x3,x2 + mul x11, x4,x2 + umulh x7,x4,x2 + + ldp x8,x9,[c0] // load 3 limbs of the dividend + ldr x10,[c0,#16] + + adds x6,x6,x11 + adc x7,x7,xzr + + subs x8,x8,x5 // dividend - divisor * quotient + sbcs x9,x9,x6 + sbcs x10,x10,x7 + sbc x5,xzr,xzr // borrow -> mask + + add x2,x2,x5 // if borrowed, adjust the quotient ... + and x3,x3,x5 + and x4,x4,x5 + adds x8,x8,x3 // ... and add divisor + adc x9,x9,x4 + + stp x8,x9,[c0] // save 2 limbs of the remainder + str x2,[c0,#16] // and one limb of the quotient + + mov x0,x2 // return adjusted quotient + + ret +.size quot_rem_128,.-quot_rem_128 + +.globl quot_rem_64 +.hidden quot_rem_64 +.type quot_rem_64,%function +.align 5 +quot_rem_64: + ldr x3,[c1] + ldr x8,[c0] // load 1 limb of the dividend + + mul x5,x3,x2 // divisor * quotient + + sub x8,x8,x5 // dividend - divisor * quotient + + stp x8,x2,[c0] // save remainder and quotient + + mov x0,x2 // return quotient + + ret +.size quot_rem_64,.-quot_rem_64 diff --git a/c-source/build/cheri/mul_mont_256-armv8.S b/c-source/build/cheri/mul_mont_256-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/mul_mont_256-armv8.S @@ -0,0 +1,464 @@ +.text + +.globl mul_mont_sparse_256 +.hidden mul_mont_sparse_256 +.type mul_mont_sparse_256,%function +.align 5 +mul_mont_sparse_256: + stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + + ldp x10,x11,[c1] + ldr x9, [c2] + ldp x12,x13,[c1,#16] + + mul x19,x10,x9 + ldp x5,x6,[c3] + mul x20,x11,x9 + ldp x7,x8,[c3,#16] + mul x21,x12,x9 + mul x22,x13,x9 + + umulh x14,x10,x9 + umulh x15,x11,x9 + mul x3,x4,x19 + umulh x16,x12,x9 + umulh x17,x13,x9 + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,xzr, x17 + mul x17,x8,x3 + ldr x9,[c2,8*1] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[c2,8*2] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[c2,8*3] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + adcs x20,x21,x15 + adcs x21,x22,x16 + adcs x22,x23,x17 + adc x23,xzr,xzr + + subs x14,x19,x5 + sbcs x15,x20,x6 + sbcs x16,x21,x7 + sbcs x17,x22,x8 + sbcs xzr, x23,xzr + + csel x19,x19,x14,lo + csel x20,x20,x15,lo + csel x21,x21,x16,lo + csel x22,x22,x17,lo + + stp x19,x20,[c0] + stp x21,x22,[c0,#16] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldr c29,[csp],#8*__SIZEOF_POINTER__ + ret +.size mul_mont_sparse_256,.-mul_mont_sparse_256 +.globl sqr_mont_sparse_256 +.hidden sqr_mont_sparse_256 +.type sqr_mont_sparse_256,%function +.align 5 +sqr_mont_sparse_256: + .inst 0xd503233f + stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + + ldp x5,x6,[c1] + ldp x7,x8,[c1,#16] + mov x4,x3 + + //////////////////////////////////////////////////////////////// + // | | | | | |a1*a0| | + // | | | | |a2*a0| | | + // | |a3*a2|a3*a0| | | | + // | | | |a2*a1| | | | + // | | |a3*a1| | | | | + // *| | | | | | | | 2| + // +|a3*a3|a2*a2|a1*a1|a0*a0| + // |--+--+--+--+--+--+--+--| + // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is x10 + // + // "can't overflow" below mark carrying into high part of + // multiplication result, which can't overflow, because it + // can never be all ones. + + mul x11,x6,x5 // a[1]*a[0] + umulh x15,x6,x5 + mul x12,x7,x5 // a[2]*a[0] + umulh x16,x7,x5 + mul x13,x8,x5 // a[3]*a[0] + umulh x19,x8,x5 + + adds x12,x12,x15 // accumulate high parts of multiplication + mul x14,x7,x6 // a[2]*a[1] + umulh x15,x7,x6 + adcs x13,x13,x16 + mul x16,x8,x6 // a[3]*a[1] + umulh x17,x8,x6 + adc x19,x19,xzr // can't overflow + + mul x20,x8,x7 // a[3]*a[2] + umulh x21,x8,x7 + + adds x15,x15,x16 // accumulate high parts of multiplication + mul x10,x5,x5 // a[0]*a[0] + adc x16,x17,xzr // can't overflow + + adds x13,x13,x14 // accumulate low parts of multiplication + umulh x5,x5,x5 + adcs x19,x19,x15 + mul x15,x6,x6 // a[1]*a[1] + adcs x20,x20,x16 + umulh x6,x6,x6 + adc x21,x21,xzr // can't overflow + + adds x11,x11,x11 // acc[1-6]*=2 + mul x16,x7,x7 // a[2]*a[2] + adcs x12,x12,x12 + umulh x7,x7,x7 + adcs x13,x13,x13 + mul x17,x8,x8 // a[3]*a[3] + adcs x19,x19,x19 + umulh x8,x8,x8 + adcs x20,x20,x20 + adcs x21,x21,x21 + adc x22,xzr,xzr + + adds x11,x11,x5 // +a[i]*a[i] + adcs x12,x12,x15 + adcs x13,x13,x6 + adcs x19,x19,x16 + adcs x20,x20,x7 + adcs x21,x21,x17 + adc x22,x22,x8 + + bl __mul_by_1_mont_256 + ldr c30,[c29,#__SIZEOF_POINTER__] + + adds x10,x10,x19 // accumulate upper half + adcs x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adc x19,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x19,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldr c29,[csp],#6*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_mont_sparse_256,.-sqr_mont_sparse_256 +.globl from_mont_256 +.hidden from_mont_256 +.type from_mont_256,%function +.align 5 +from_mont_256: + .inst 0xd503233f + stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! + add c29,csp,#0 + + mov x4,x3 + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + + bl __mul_by_1_mont_256 + ldr c30,[c29,#__SIZEOF_POINTER__] + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + + ldr c29,[csp],#2*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size from_mont_256,.-from_mont_256 + +.globl redc_mont_256 +.hidden redc_mont_256 +.type redc_mont_256,%function +.align 5 +redc_mont_256: + .inst 0xd503233f + stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! + add c29,csp,#0 + + mov x4,x3 + ldp x10,x11,[c1] + ldp x12,x13,[c1,#16] + + bl __mul_by_1_mont_256 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp x14,x15,[c1,#32] + ldp x16,x17,[c1,#48] + + adds x10,x10,x14 + adcs x11,x11,x15 + adcs x12,x12,x16 + adcs x13,x13,x17 + adc x9,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x9,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[c0] + stp x12,x13,[c0,#16] + + ldr c29,[csp],#2*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size redc_mont_256,.-redc_mont_256 + +.type __mul_by_1_mont_256,%function +.align 5 +__mul_by_1_mont_256: + mul x3,x4,x10 + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + adc x13,x9,x17 + + ret +.size __mul_by_1_mont_256,.-__mul_by_1_mont_256 diff --git a/c-source/build/cheri/mul_mont_384-armv8.S b/c-source/build/cheri/mul_mont_384-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/mul_mont_384-armv8.S @@ -0,0 +1,2372 @@ +.text + +.globl add_mod_384x384 +.type add_mod_384x384,%function +.align 5 +add_mod_384x384: + .inst 0xd503233f + stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + bl __add_mod_384x384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldr c29,[csp],#8*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size add_mod_384x384,.-add_mod_384x384 + +.type __add_mod_384x384,%function +.align 5 +__add_mod_384x384: + ldp x11, x12, [c1] + ldp x19,x20,[c2] + ldp x13, x14, [c1,#16] + adds x11,x11,x19 + ldp x21,x22,[c2,#16] + adcs x12,x12,x20 + ldp x15, x16, [c1,#32] + adcs x13,x13,x21 + ldp x23,x24,[c2,#32] + adcs x14,x14,x22 + stp x11, x12, [c0] + adcs x15,x15,x23 + ldp x11, x12, [c1,#48] + adcs x16,x16,x24 + + ldp x19,x20,[c2,#48] + stp x13, x14, [c0,#16] + ldp x13, x14, [c1,#64] + ldp x21,x22,[c2,#64] + + adcs x11,x11,x19 + stp x15, x16, [c0,#32] + adcs x12,x12,x20 + ldp x15, x16, [c1,#80] + adcs x13,x13,x21 + ldp x23,x24,[c2,#80] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + stp x11,x12,[c0,#48] + csel x15,x15,x23,lo + stp x13,x14,[c0,#64] + csel x16,x16,x24,lo + stp x15,x16,[c0,#80] + + ret +.size __add_mod_384x384,.-__add_mod_384x384 + +.globl sub_mod_384x384 +.type sub_mod_384x384,%function +.align 5 +sub_mod_384x384: + .inst 0xd503233f + stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + bl __sub_mod_384x384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldr c29,[csp],#8*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sub_mod_384x384,.-sub_mod_384x384 + +.type __sub_mod_384x384,%function +.align 5 +__sub_mod_384x384: + ldp x11, x12, [c1] + ldp x19,x20,[c2] + ldp x13, x14, [c1,#16] + subs x11,x11,x19 + ldp x21,x22,[c2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [c1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[c2,#32] + sbcs x14,x14,x22 + stp x11, x12, [c0] + sbcs x15,x15,x23 + ldp x11, x12, [c1,#48] + sbcs x16,x16,x24 + + ldp x19,x20,[c2,#48] + stp x13, x14, [c0,#16] + ldp x13, x14, [c1,#64] + ldp x21,x22,[c2,#64] + + sbcs x11,x11,x19 + stp x15, x16, [c0,#32] + sbcs x12,x12,x20 + ldp x15, x16, [c1,#80] + sbcs x13,x13,x21 + ldp x23,x24,[c2,#80] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[c0,#48] + adc x16,x16,x24 + stp x13,x14,[c0,#64] + stp x15,x16,[c0,#80] + + ret +.size __sub_mod_384x384,.-__sub_mod_384x384 + +.type __add_mod_384,%function +.align 5 +__add_mod_384: + ldp x11, x12, [c1] + ldp x19,x20,[c2] + ldp x13, x14, [c1,#16] + adds x11,x11,x19 + ldp x21,x22,[c2,#16] + adcs x12,x12,x20 + ldp x15, x16, [c1,#32] + adcs x13,x13,x21 + ldp x23,x24,[c2,#32] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + stp x11,x12,[c0] + csel x16,x16,x24,lo + stp x13,x14,[c0,#16] + stp x15,x16,[c0,#32] + + ret +.size __add_mod_384,.-__add_mod_384 + +.type __sub_mod_384,%function +.align 5 +__sub_mod_384: + ldp x11, x12, [c1] + ldp x19,x20,[c2] + ldp x13, x14, [c1,#16] + subs x11,x11,x19 + ldp x21,x22,[c2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [c1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[c2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[c0] + adc x16,x16,x24 + stp x13,x14,[c0,#16] + stp x15,x16,[c0,#32] + + ret +.size __sub_mod_384,.-__sub_mod_384 + +.globl mul_mont_384x +.hidden mul_mont_384x +.type mul_mont_384x,%function +.align 5 +mul_mont_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + sub csp,csp,#288 // space for 3 768-bit vectors + + mov c26,c0 // save r_ptr + mov c27,c1 // save b_ptr + mov c28,c2 // save b_ptr + + add c0,csp,#0 + bl __mul_384 + + add c1,c1,#48 + add c2,c2,#48 + add c0,csp,#96 + bl __mul_384 + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + sub c2,c1,#48 + add c0,csp,#240 + bl __add_mod_384 + + add c1,c28,#0 + add c2,c28,#48 + add c0,csp,#192 + bl __add_mod_384 + + add c1,c0,#0 + add c2,c0,#48 + bl __mul_384 // mul_384(t2, a->re+a->im, b->re+b->im) + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + mov c1,c0 + add c2,csp,#0 + bl __sub_mod_384x384 + + add c2,csp,#96 + bl __sub_mod_384x384 // t2 = t2-t0-t1 + + add c1,csp,#0 + add c2,csp,#96 + add c0,csp,#0 + bl __sub_mod_384x384 // t0 = t0-t1 + + add c1,csp,#0 + add c0,c26,#0 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + + add c1,csp,#192 + add c0,c0,#48 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + add csp,csp,#288 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_mont_384x,.-mul_mont_384x + +.globl sqr_mont_384x +.hidden sqr_mont_384x +.type sqr_mont_384x,%function +.align 5 +sqr_mont_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + stp c3,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there + sub csp,csp,#96 // space for 2 384-bit vectors + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + ldp x9,x10,[c2,#32] + + add c2,c1,#48 + add c0,csp,#0 + bl __add_mod_384 // t0 = a->re + a->im + + add c0,csp,#48 + bl __sub_mod_384 // t1 = a->re - a->im + + ldp x11,x12,[c1] + ldr x17, [c2] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + bl __mul_mont_384 // mul_mont_384(ret->im, a->re, a->im) + + adds x11,x11,x11 // add with itself + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x19,x11,x19,lo + csel x20,x12,x20,lo + csel x21,x13,x21,lo + ldp x11,x12,[csp] + csel x22,x14,x22,lo + ldr x17, [csp,#48] + csel x23,x15,x23,lo + ldp x13,x14,[csp,#16] + csel x24,x16,x24,lo + ldp x15,x16,[csp,#32] + + stp x19,x20,[c2,#48] + stp x21,x22,[c2,#64] + stp x23,x24,[c2,#80] + + add c2,csp,#48 + bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) + ldr c30,[c29,#__SIZEOF_POINTER__] + + stp x11,x12,[c2] + stp x13,x14,[c2,#16] + stp x15,x16,[c2,#32] + + add csp,csp,#96 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_mont_384x,.-sqr_mont_384x + +.globl mul_mont_384 +.hidden mul_mont_384 +.type mul_mont_384,%function +.align 5 +mul_mont_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + stp c4,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there + + ldp x11,x12,[c1] + ldr x17, [c2] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + bl __mul_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + stp x11,x12,[c2] + stp x13,x14,[c2,#16] + stp x15,x16,[c2,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_mont_384,.-mul_mont_384 + +.type __mul_mont_384,%function +.align 5 +__mul_mont_384: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + mov x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[c2,8*1] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[c29,#12*__SIZEOF_POINTER__] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[c2,8*2] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[c29,#12*__SIZEOF_POINTER__] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[c2,8*3] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[c29,#12*__SIZEOF_POINTER__] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[c2,8*4] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[c29,#12*__SIZEOF_POINTER__] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[c2,8*5] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[c29,#12*__SIZEOF_POINTER__] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + ldp c4,c2,[c29,#12*__SIZEOF_POINTER__] // pull r_ptr + adc x17,x17,xzr + + adds x19,x20,x26 + adcs x20,x21,x27 + adcs x21,x22,x28 + adcs x22,x23,x0 + adcs x23,x24,x1 + adcs x24,x25,x3 + adc x25,x17,xzr + + subs x26,x19,x5 + sbcs x27,x20,x6 + sbcs x28,x21,x7 + sbcs x0,x22,x8 + sbcs x1,x23,x9 + sbcs x3,x24,x10 + sbcs xzr, x25,xzr + + csel x11,x19,x26,lo + csel x12,x20,x27,lo + csel x13,x21,x28,lo + csel x14,x22,x0,lo + csel x15,x23,x1,lo + csel x16,x24,x3,lo + ret +.size __mul_mont_384,.-__mul_mont_384 + +.globl sqr_mont_384 +.hidden sqr_mont_384 +.type sqr_mont_384,%function +.align 5 +sqr_mont_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + sub csp,csp,#96 // space for 768-bit vector + mov c4,c3 // adjust for missing b_ptr + + mov c3,c0 // save r_ptr + mov c0,csp + + ldp x11,x12,[c1] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + bl __sqr_384 + + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + ldp x9,x10,[c2,#32] + + mov c1,csp + mov c0,c3 // restore r_ptr + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + add csp,csp,#96 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_mont_384,.-sqr_mont_384 + +.globl sqr_n_mul_mont_383 +.hidden sqr_n_mul_mont_383 +.type sqr_n_mul_mont_383,%function +.align 5 +sqr_n_mul_mont_383: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + stp c4,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there + sub csp,csp,#96 // space for 768-bit vector + mov c17,c5 // save b_ptr + + ldp x11,x12,[c1] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + mov c0,csp +.Loop_sqr_383: + bl __sqr_384 + sub x2,x2,#1 // counter + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + mov c1,csp + bl __mul_by_1_mont_384 + + ldp x19,x20,[c1,#48] + ldp x21,x22,[c1,#64] + ldp x23,x24,[c1,#80] + + adds x11,x11,x19 // just accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + cbnz x2,.Loop_sqr_383 + + mov c2,c17 + ldr x17,[c17] + bl __mul_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + stp x11,x12,[c2] + stp x13,x14,[c2,#16] + stp x15,x16,[c2,#32] + + add csp,csp,#96 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 +.type __sqr_384,%function +.align 5 +__sqr_384: + mul x19,x12,x11 + mul x20,x13,x11 + mul x21,x14,x11 + mul x22,x15,x11 + mul x23,x16,x11 + + umulh x6,x12,x11 + umulh x7,x13,x11 + umulh x8,x14,x11 + umulh x9,x15,x11 + adds x20,x20,x6 + umulh x10,x16,x11 + adcs x21,x21,x7 + mul x7,x13,x12 + adcs x22,x22,x8 + mul x8,x14,x12 + adcs x23,x23,x9 + mul x9,x15,x12 + adc x24,xzr, x10 + mul x10,x16,x12 + + adds x21,x21,x7 + umulh x7,x13,x12 + adcs x22,x22,x8 + umulh x8,x14,x12 + adcs x23,x23,x9 + umulh x9,x15,x12 + adcs x24,x24,x10 + umulh x10,x16,x12 + adc x25,xzr,xzr + + mul x5,x11,x11 + adds x22,x22,x7 + umulh x11, x11,x11 + adcs x23,x23,x8 + mul x8,x14,x13 + adcs x24,x24,x9 + mul x9,x15,x13 + adc x25,x25,x10 + mul x10,x16,x13 + + adds x23,x23,x8 + umulh x8,x14,x13 + adcs x24,x24,x9 + umulh x9,x15,x13 + adcs x25,x25,x10 + umulh x10,x16,x13 + adc x26,xzr,xzr + + mul x6,x12,x12 + adds x24,x24,x8 + umulh x12, x12,x12 + adcs x25,x25,x9 + mul x9,x15,x14 + adc x26,x26,x10 + mul x10,x16,x14 + + adds x25,x25,x9 + umulh x9,x15,x14 + adcs x26,x26,x10 + umulh x10,x16,x14 + adc x27,xzr,xzr + mul x7,x13,x13 + adds x26,x26,x9 + umulh x13, x13,x13 + adc x27,x27,x10 + mul x8,x14,x14 + + mul x10,x16,x15 + umulh x14, x14,x14 + adds x27,x27,x10 + umulh x10,x16,x15 + mul x9,x15,x15 + adc x28,x10,xzr + + adds x19,x19,x19 + adcs x20,x20,x20 + adcs x21,x21,x21 + adcs x22,x22,x22 + adcs x23,x23,x23 + adcs x24,x24,x24 + adcs x25,x25,x25 + adcs x26,x26,x26 + umulh x15, x15,x15 + adcs x27,x27,x27 + mul x10,x16,x16 + adcs x28,x28,x28 + umulh x16, x16,x16 + adc x1,xzr,xzr + + adds x19,x19,x11 + adcs x20,x20,x6 + adcs x21,x21,x12 + adcs x22,x22,x7 + adcs x23,x23,x13 + adcs x24,x24,x8 + adcs x25,x25,x14 + stp x5,x19,[c0] + adcs x26,x26,x9 + stp x20,x21,[c0,#16] + adcs x27,x27,x15 + stp x22,x23,[c0,#32] + adcs x28,x28,x10 + stp x24,x25,[c0,#48] + adc x16,x16,x1 + stp x26,x27,[c0,#64] + stp x28,x16,[c0,#80] + + ret +.size __sqr_384,.-__sqr_384 +.globl sqr_384 +.hidden sqr_384 +.type sqr_384,%function +.align 5 +sqr_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + + ldp x11,x12,[c1] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + bl __sqr_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_384,.-sqr_384 + +.globl redc_mont_384 +.hidden redc_mont_384 +.type redc_mont_384,%function +.align 5 +redc_mont_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + ldp x9,x10,[c2,#32] + + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size redc_mont_384,.-redc_mont_384 + +.globl from_mont_384 +.hidden from_mont_384 +.type from_mont_384,%function +.align 5 +from_mont_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + ldp x9,x10,[c2,#32] + + bl __mul_by_1_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[c0] + stp x13,x14,[c0,#16] + stp x15,x16,[c0,#32] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size from_mont_384,.-from_mont_384 + +.type __mul_by_1_mont_384,%function +.align 5 +__mul_by_1_mont_384: + ldp x11,x12,[c1] + ldp x13,x14,[c1,#16] + mul x26,x4,x11 + ldp x15,x16,[c1,#32] + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + ret +.size __mul_by_1_mont_384,.-__mul_by_1_mont_384 + +.type __redc_tail_mont_384,%function +.align 5 +__redc_tail_mont_384: + ldp x19,x20,[c1,#48] + ldp x21,x22,[c1,#64] + ldp x23,x24,[c1,#80] + + adds x11,x11,x19 // accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[c0] + stp x13,x14,[c0,#16] + stp x15,x16,[c0,#32] + + ret +.size __redc_tail_mont_384,.-__redc_tail_mont_384 + +.globl mul_384 +.hidden mul_384 +.type mul_384,%function +.align 5 +mul_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + + bl __mul_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_384,.-mul_384 + +.type __mul_384,%function +.align 5 +__mul_384: + ldp x11,x12,[c1] + ldr x17, [c2] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + + umulh x5,x11,x17 + umulh x6,x12,x17 + umulh x7,x13,x17 + umulh x8,x14,x17 + umulh x9,x15,x17 + umulh x10,x16,x17 + ldr x17,[c2,8*1] + + str x19,[c0] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,xzr, x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[c2,#8*(1+1)] + adc x25,xzr,xzr + + str x19,[c0,8*1] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[c2,#8*(2+1)] + adc x25,xzr,xzr + + str x19,[c0,8*2] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[c2,#8*(3+1)] + adc x25,xzr,xzr + + str x19,[c0,8*3] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[c2,#8*(4+1)] + adc x25,xzr,xzr + + str x19,[c0,8*4] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + adc x25,xzr,xzr + + str x19,[c0,8*5] + adds x19,x20,x5 + adcs x20,x21,x6 + adcs x21,x22,x7 + adcs x22,x23,x8 + adcs x23,x24,x9 + adc x24,x25,x10 + + stp x19,x20,[c0,#48] + stp x21,x22,[c0,#64] + stp x23,x24,[c0,#80] + + ret +.size __mul_384,.-__mul_384 + +.globl mul_382x +.hidden mul_382x +.type mul_382x,%function +.align 5 +mul_382x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + sub csp,csp,#96 // space for two 384-bit vectors + + ldp x11,x12,[c1] + mov c26,c0 // save r_ptr + ldp x19,x20,[c1,#48] + mov c27,c1 // save a_ptr + ldp x13,x14,[c1,#16] + mov c28,c2 // save b_ptr + ldp x21,x22,[c1,#64] + ldp x15,x16,[c1,#32] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x23,x24,[c1,#80] + adcs x6,x12,x20 + ldp x11,x12,[c2] + adcs x7,x13,x21 + ldp x19,x20,[c2,#48] + adcs x8,x14,x22 + ldp x13,x14,[c2,#16] + adcs x9,x15,x23 + ldp x21,x22,[c2,#64] + adc x10,x16,x24 + ldp x15,x16,[c2,#32] + + stp x5,x6,[csp] + adds x5,x11,x19 // t1 = b->re + b->im + ldp x23,x24,[c2,#80] + adcs x6,x12,x20 + stp x7,x8,[csp,#16] + adcs x7,x13,x21 + adcs x8,x14,x22 + stp x9,x10,[csp,#32] + adcs x9,x15,x23 + stp x5,x6,[csp,#48] + adc x10,x16,x24 + stp x7,x8,[csp,#64] + stp x9,x10,[csp,#80] + + bl __mul_384 // mul_384(ret->re, a->re, b->re) + + add c1,csp,#0 + add c2,csp,#48 + add c0,c26,#96 + bl __mul_384 + + add c1,c27,#48 + add c2,c28,#48 + add c0,csp,#0 + bl __mul_384 + + ldp x5,x6,[c3] + ldp x7,x8,[c3,#16] + ldp x9,x10,[c3,#32] + + add c1,c26,#96 + add c2,csp,#0 + add c0,c26,#96 + bl __sub_mod_384x384 + + add c2,c26,#0 + bl __sub_mod_384x384 + + add c1,c26,#0 + add c2,csp,#0 + add c0,c26,#0 + bl __sub_mod_384x384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + add csp,csp,#96 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size mul_382x,.-mul_382x + +.globl sqr_382x +.hidden sqr_382x +.type sqr_382x,%function +.align 5 +sqr_382x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + + ldp x11,x12,[c1] + ldp x19,x20,[c1,#48] + ldp x13,x14,[c1,#16] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x21,x22,[c1,#64] + adcs x6,x12,x20 + ldp x15,x16,[c1,#32] + adcs x7,x13,x21 + ldp x23,x24,[c1,#80] + adcs x8,x14,x22 + stp x5,x6,[c0] + adcs x9,x15,x23 + ldp x5,x6,[c2] + adc x10,x16,x24 + stp x7,x8,[c0,#16] + + subs x11,x11,x19 // t1 = a->re - a->im + ldp x7,x8,[c2,#16] + sbcs x12,x12,x20 + stp x9,x10,[c0,#32] + sbcs x13,x13,x21 + ldp x9,x10,[c2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + adds x11,x11,x19 + and x21,x7,x25 + adcs x12,x12,x20 + and x22,x8,x25 + adcs x13,x13,x21 + and x23,x9,x25 + adcs x14,x14,x22 + and x24,x10,x25 + adcs x15,x15,x23 + stp x11,x12,[c0,#48] + adc x16,x16,x24 + stp x13,x14,[c0,#64] + stp x15,x16,[c0,#80] + + mov c4,c1 // save a_ptr + add c1,c0,#0 + add c2,c0,#48 + bl __mul_384 + + add c1,c4,#0 + add c2,c4,#48 + add c0,c0,#96 + bl __mul_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldp x11,x12,[c0] + ldp x13,x14,[c0,#16] + adds x11,x11,x11 // add with itself + ldp x15,x16,[c0,#32] + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adcs x19,x19,x19 + adcs x20,x20,x20 + stp x11,x12,[c0] + adcs x21,x21,x21 + stp x13,x14,[c0,#16] + adcs x22,x22,x22 + stp x15,x16,[c0,#32] + adcs x23,x23,x23 + stp x19,x20,[c0,#48] + adc x24,x24,x24 + stp x21,x22,[c0,#64] + stp x23,x24,[c0,#80] + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_382x,.-sqr_382x + +.globl sqr_mont_382x +.hidden sqr_mont_382x +.type sqr_mont_382x,%function +.align 5 +sqr_mont_382x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + stp c3,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there + sub csp,csp,#112 // space for two 384-bit vectors + word + mov x4,x3 // adjust for missing b_ptr + + ldp x11,x12,[c1] + ldp x13,x14,[c1,#16] + ldp x15,x16,[c1,#32] + + ldp x17,x20,[c1,#48] + ldp x21,x22,[c1,#64] + ldp x23,x24,[c1,#80] + + adds x5,x11,x17 // t0 = a->re + a->im + adcs x6,x12,x20 + adcs x7,x13,x21 + adcs x8,x14,x22 + adcs x9,x15,x23 + adc x10,x16,x24 + + subs x19,x11,x17 // t1 = a->re - a->im + sbcs x20,x12,x20 + sbcs x21,x13,x21 + sbcs x22,x14,x22 + sbcs x23,x15,x23 + sbcs x24,x16,x24 + sbc x25,xzr,xzr // borrow flag as mask + + stp x5,x6,[csp] + stp x7,x8,[csp,#16] + stp x9,x10,[csp,#32] + stp x19,x20,[csp,#48] + stp x21,x22,[csp,#64] + stp x23,x24,[csp,#80] + str x25,[csp,#96] + + ldp x5,x6,[c2] + ldp x7,x8,[c2,#16] + ldp x9,x10,[c2,#32] + + add c2,c1,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) + + adds x19,x11,x11 // add with itself + adcs x20,x12,x12 + adcs x21,x13,x13 + adcs x22,x14,x14 + adcs x23,x15,x15 + adc x24,x16,x16 + + stp x19,x20,[c2,#48] + stp x21,x22,[c2,#64] + stp x23,x24,[c2,#80] + + ldp x11,x12,[csp] + ldr x17,[csp,#48] + ldp x13,x14,[csp,#16] + ldp x15,x16,[csp,#32] + + add c2,csp,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) + ldr c30,[c29,#__SIZEOF_POINTER__] + + ldr x25,[csp,#96] // account for sign from a->re - a->im + ldp x19,x20,[csp] + ldp x21,x22,[csp,#16] + ldp x23,x24,[csp,#32] + + and x19,x19,x25 + and x20,x20,x25 + and x21,x21,x25 + and x22,x22,x25 + and x23,x23,x25 + and x24,x24,x25 + + subs x11,x11,x19 + sbcs x12,x12,x20 + sbcs x13,x13,x21 + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + and x21,x7,x25 + and x22,x8,x25 + and x23,x9,x25 + and x24,x10,x25 + + adds x11,x11,x19 + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + stp x11,x12,[c2] + stp x13,x14,[c2,#16] + stp x15,x16,[c2,#32] + + add csp,csp,#112 + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sqr_mont_382x,.-sqr_mont_382x + +.type __mul_mont_383_nonred,%function +.align 5 +__mul_mont_383_nonred: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + ldr x17,[c2,8*1] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[c29,#12*__SIZEOF_POINTER__] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[c2,8*2] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[c29,#12*__SIZEOF_POINTER__] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[c2,8*3] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[c29,#12*__SIZEOF_POINTER__] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[c2,8*4] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[c29,#12*__SIZEOF_POINTER__] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[c2,8*5] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[c29,#12*__SIZEOF_POINTER__] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + ldp c4,c2,[c29,#12*__SIZEOF_POINTER__] // pull r_ptr + + adds x11,x20,x26 + adcs x12,x21,x27 + adcs x13,x22,x28 + adcs x14,x23,x0 + adcs x15,x24,x1 + adcs x16,x25,x3 + + ret +.size __mul_mont_383_nonred,.-__mul_mont_383_nonred + +.globl sgn0_pty_mont_384 +.hidden sgn0_pty_mont_384 +.type sgn0_pty_mont_384,%function +.align 5 +sgn0_pty_mont_384: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + + mov x4,x2 + ldp x5,x6,[c1] + ldp x7,x8,[c1,#16] + ldp x9,x10,[c1,#32] + mov c1,c0 + + bl __mul_by_1_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + and x0,x11,#1 + adds x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sgn0_pty_mont_384,.-sgn0_pty_mont_384 + +.globl sgn0_pty_mont_384x +.hidden sgn0_pty_mont_384x +.type sgn0_pty_mont_384x,%function +.align 5 +sgn0_pty_mont_384x: + .inst 0xd503233f + stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! + add c29,csp,#0 + stp c19,c20,[csp,#2*__SIZEOF_POINTER__] + stp c21,c22,[csp,#4*__SIZEOF_POINTER__] + stp c23,c24,[csp,#6*__SIZEOF_POINTER__] + stp c25,c26,[csp,#8*__SIZEOF_POINTER__] + stp c27,c28,[csp,#10*__SIZEOF_POINTER__] + + mov x4,x2 + ldp x5,x6,[c1] + ldp x7,x8,[c1,#16] + ldp x9,x10,[c1,#32] + mov c1,c0 + + bl __mul_by_1_mont_384 + add c1,c1,#48 + + and x2,x11,#1 + orr x3,x11,x12 + adds x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + orr x3,x3,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x2,x2,x17 + + bl __mul_by_1_mont_384 + ldr c30,[c29,#__SIZEOF_POINTER__] + + and x0,x11,#1 + orr x1,x11,x12 + adds x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + orr x1,x1,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] + ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] + ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] + ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] + ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] + ldr c29,[csp],#16*__SIZEOF_POINTER__ + .inst 0xd50323bf + ret +.size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x diff --git a/c-source/build/cheri/sha256-armv8.S b/c-source/build/cheri/sha256-armv8.S new file mode 100644 --- /dev/null +++ b/c-source/build/cheri/sha256-armv8.S @@ -0,0 +1,1083 @@ +// +// Copyright Supranational LLC +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// ==================================================================== +// Written by Andy Polyakov, @dot-asm, initially for the OpenSSL +// project. +// ==================================================================== +// +// sha256_block procedure for ARMv8. +// +// This module is stripped of scalar code paths, with rationale that all +// known processors are NEON-capable. +// +// See original module at CRYPTOGAMS for further details. + +.comm __blst_platform_cap,4 +.text + +.align 6 +.type .LK256,%object +.LK256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0 //terminator +.size .LK256,.-.LK256 +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.align 2 +.align 2 +.globl blst_sha256_block_armv8 +.type blst_sha256_block_armv8,%function +.align 6 +blst_sha256_block_armv8: +.Lv8_entry: + stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! + add c29,csp,#0 + + ld1 {v0.4s,v1.4s},[c0] + adr c3,.LK256 + +.Loop_hw: + ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[c1],#64 + sub x2,x2,#1 + ld1 {v16.4s},[c3],#16 + rev32 v4.16b,v4.16b + rev32 v5.16b,v5.16b + rev32 v6.16b,v6.16b + rev32 v7.16b,v7.16b + orr v18.16b,v0.16b,v0.16b // offload + orr v19.16b,v1.16b,v1.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[c3],#16 + add v16.4s,v16.4s,v4.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + ld1 {v16.4s},[c3],#16 + add v17.4s,v17.4s,v5.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + ld1 {v17.4s},[c3] + add v16.4s,v16.4s,v6.4s + sub x3,x3,#64*4-16 // rewind + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + add v17.4s,v17.4s,v7.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + add v0.4s,v0.4s,v18.4s + add v1.4s,v1.4s,v19.4s + + cbnz x2,.Loop_hw + + st1 {v0.4s,v1.4s},[c0] + + ldr c29,[csp],#2*__SIZEOF_POINTER__ + ret +.size blst_sha256_block_armv8,.-blst_sha256_block_armv8 +.globl blst_sha256_block_data_order +.type blst_sha256_block_data_order,%function +.align 4 +blst_sha256_block_data_order: + adrp c16,__blst_platform_cap + ldr w16,[c16,#:lo12:__blst_platform_cap] + tst w16,#1 + b.ne .Lv8_entry + + stp c29, c30, [csp, #-2*__SIZEOF_POINTER__]! + mov c29, csp + sub csp,csp,#16*4 + + adr c16,.LK256 + add x2,x1,x2,lsl#6 // len to point at the end of inp + + ld1 {v0.16b},[c1], #16 + ld1 {v1.16b},[c1], #16 + ld1 {v2.16b},[c1], #16 + ld1 {v3.16b},[c1], #16 + ld1 {v4.4s},[c16], #16 + ld1 {v5.4s},[c16], #16 + ld1 {v6.4s},[c16], #16 + ld1 {v7.4s},[c16], #16 + rev32 v0.16b,v0.16b // yes, even on + rev32 v1.16b,v1.16b // big-endian + rev32 v2.16b,v2.16b + rev32 v3.16b,v3.16b + mov c17,csp + add v4.4s,v4.4s,v0.4s + add v5.4s,v5.4s,v1.4s + add v6.4s,v6.4s,v2.4s + st1 {v4.4s,v5.4s},[c17], #32 + add v7.4s,v7.4s,v3.4s + st1 {v6.4s,v7.4s},[c17] + sub c17,c17,#32 + + ldp w3,w4,[c0] + ldp w5,w6,[c0,#8] + ldp w7,w8,[c0,#16] + ldp w9,w10,[c0,#24] + ldr w12,[csp,#0] + mov w13,wzr + eor w14,w4,w5 + mov w15,wzr + b .L_00_48 + +.align 4 +.L_00_48: + ext v4.16b,v0.16b,v1.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v2.16b,v3.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v3.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v0.4s,v0.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[csp,#4] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v0.4s,v0.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[csp,#8] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v0.4s,v0.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v0.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v0.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v0.4s,#15 + add w8,w8,w12 + ushr v17.4s,v0.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v0.4s,#13 + ldr w12,[csp,#12] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[c16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v0.4s,v0.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v0.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[csp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[c17], #16 + ext v4.16b,v1.16b,v2.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v3.16b,v0.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v0.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v1.4s,v1.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[csp,#20] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v1.4s,v1.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[csp,#24] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v1.4s,v1.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v1.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v1.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v1.4s,#15 + add w4,w4,w12 + ushr v17.4s,v1.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v1.4s,#13 + ldr w12,[csp,#28] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[c16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v1.4s,v1.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v1.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[csp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[c17], #16 + ext v4.16b,v2.16b,v3.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v0.16b,v1.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v1.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v2.4s,v2.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[csp,#36] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v2.4s,v2.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[csp,#40] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v2.4s,v2.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v2.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v2.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v2.4s,#15 + add w8,w8,w12 + ushr v17.4s,v2.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v2.4s,#13 + ldr w12,[csp,#44] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[c16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v2.4s,v2.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v2.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[csp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[c17], #16 + ext v4.16b,v3.16b,v0.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v1.16b,v2.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v2.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v3.4s,v3.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[csp,#52] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v3.4s,v3.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[csp,#56] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v3.4s,v3.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v3.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v3.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v3.4s,#15 + add w4,w4,w12 + ushr v17.4s,v3.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v3.4s,#13 + ldr w12,[csp,#60] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[c16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v3.4s,v3.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v3.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[c16] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[c17], #16 + cmp w12,#0 // check for K256 terminator + ldr w12,[csp,#0] + sub c17,c17,#64 + bne .L_00_48 + + sub c16,c16,#256 + cmp x1,x2 + mov x17, #-64 + csel x17, x17, xzr, eq + add c1,c1,x17 + mov c17,csp + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v0.16b},[c1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[c16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v0.16b,v0.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v0.4s + add w10,w10,w11 + ldr w12,[csp,#4] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[csp,#8] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[csp,#12] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[csp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[c17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v1.16b},[c1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[c16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v1.16b,v1.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v1.4s + add w6,w6,w11 + ldr w12,[csp,#20] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[csp,#24] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[csp,#28] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[csp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[c17], #16 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v2.16b},[c1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[c16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v2.16b,v2.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v2.4s + add w10,w10,w11 + ldr w12,[csp,#36] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[csp,#40] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[csp,#44] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[csp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[c17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v3.16b},[c1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[c16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v3.16b,v3.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v3.4s + add w6,w6,w11 + ldr w12,[csp,#52] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[csp,#56] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[csp,#60] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[c17], #16 + add w3,w3,w15 // h+=Sigma0(a) from the past + ldp w11,w12,[c0,#0] + add w3,w3,w13 // h+=Maj(a,b,c) from the past + ldp w13,w14,[c0,#8] + add w3,w3,w11 // accumulate + add w4,w4,w12 + ldp w11,w12,[c0,#16] + add w5,w5,w13 + add w6,w6,w14 + ldp w13,w14,[c0,#24] + add w7,w7,w11 + add w8,w8,w12 + ldr w12,[csp,#0] + stp w3,w4,[c0,#0] + add w9,w9,w13 + mov w13,wzr + stp w5,w6,[c0,#8] + add w10,w10,w14 + stp w7,w8,[c0,#16] + eor w14,w4,w5 + stp w9,w10,[c0,#24] + mov w15,wzr + mov c17,csp + b.ne .L_00_48 + + ldr c29,[c29] + add csp,csp,#16*4+2*__SIZEOF_POINTER__ + ret +.size blst_sha256_block_data_order,.-blst_sha256_block_data_order +.globl blst_sha256_emit +.hidden blst_sha256_emit +.type blst_sha256_emit,%function +.align 4 +blst_sha256_emit: + ldp x4,x5,[c1] + ldp x6,x7,[c1,#16] +#ifndef __AARCH64EB__ + rev x4,x4 + rev x5,x5 + rev x6,x6 + rev x7,x7 +#endif + str w4,[c0,#4] + lsr x4,x4,#32 + str w5,[c0,#12] + lsr x5,x5,#32 + str w6,[c0,#20] + lsr x6,x6,#32 + str w7,[c0,#28] + lsr x7,x7,#32 + str w4,[c0,#0] + str w5,[c0,#8] + str w6,[c0,#16] + str w7,[c0,#24] + ret +.size blst_sha256_emit,.-blst_sha256_emit + +.globl blst_sha256_bcopy +.hidden blst_sha256_bcopy +.type blst_sha256_bcopy,%function +.align 4 +blst_sha256_bcopy: +.Loop_bcopy: + ldrb w3,[c1],#1 + sub x2,x2,#1 + strb w3,[c0],#1 + cbnz x2,.Loop_bcopy + ret +.size blst_sha256_bcopy,.-blst_sha256_bcopy + +.globl blst_sha256_hcopy +.hidden blst_sha256_hcopy +.type blst_sha256_hcopy,%function +.align 4 +blst_sha256_hcopy: + ldp x4,x5,[c1] + ldp x6,x7,[c1,#16] + stp x4,x5,[c0] + stp x6,x7,[c0,#16] + ret +.size blst_sha256_hcopy,.-blst_sha256_hcopy diff --git a/c-source/build/coff/add_mod_256-x86_64.s b/c-source/build/coff/add_mod_256-x86_64.s --- a/c-source/build/coff/add_mod_256-x86_64.s +++ b/c-source/build/coff/add_mod_256-x86_64.s @@ -25,6 +25,9 @@ .LSEH_body_add_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -66,7 +69,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_add_mod_256: @@ -96,6 +107,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -117,7 +131,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_3_mod_256: @@ -147,7 +169,15 @@ cmovcq %rbx,%r10 cmovcq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif @@ -176,6 +206,9 @@ .LSEH_body_lshift_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -203,7 +236,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_lshift_mod_256: @@ -233,6 +274,9 @@ .LSEH_body_rshift_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rbp movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -292,7 +336,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_rshift_mod_256: @@ -322,6 +374,9 @@ .LSEH_body_cneg_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r12 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -370,7 +425,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_cneg_mod_256: @@ -400,6 +463,9 @@ .LSEH_body_sub_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -439,7 +505,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sub_mod_256: @@ -458,6 +532,9 @@ movq %rcx,%rdi movq %rdx,%rsi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%rax movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -482,7 +559,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_check_mod_256: @@ -512,6 +597,9 @@ .LSEH_body_add_n_check_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -558,7 +646,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_add_n_check_mod_256: @@ -588,6 +684,9 @@ .LSEH_body_sub_n_check_mod_256: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -633,7 +732,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sub_n_check_mod_256: .section .pdata diff --git a/c-source/build/coff/add_mod_384-armv8.S b/c-source/build/coff/add_mod_384-armv8.S --- a/c-source/build/coff/add_mod_384-armv8.S +++ b/c-source/build/coff/add_mod_384-armv8.S @@ -8,25 +8,25 @@ .p2align 5 add_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -78,10 +78,10 @@ .p2align 5 add_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -96,15 +96,15 @@ stp x14,x15,[x0,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -117,10 +117,10 @@ .p2align 5 rshift_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -135,14 +135,14 @@ bl __rshift_mod_384 cbnz x2,.Loop_rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -183,10 +183,10 @@ .p2align 5 div_by_2_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -198,14 +198,14 @@ bl __rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -218,10 +218,10 @@ .p2align 5 lshift_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -236,14 +236,14 @@ bl __lshift_mod_384 cbnz x2,.Loop_lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -287,10 +287,10 @@ .p2align 5 mul_by_3_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -307,15 +307,15 @@ ldp x21,x22,[x1,#32] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -328,10 +328,10 @@ .p2align 5 mul_by_8_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -344,15 +344,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -365,10 +365,10 @@ .p2align 5 mul_by_3_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -400,15 +400,15 @@ ldp x21,x22,[x1,#80] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -421,10 +421,10 @@ .p2align 5 mul_by_8_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -448,15 +448,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -469,10 +469,10 @@ .p2align 5 cneg_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x4,x5,[x3] @@ -507,9 +507,9 @@ csel x15,x15,x22,eq stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -522,25 +522,25 @@ .p2align 5 sub_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -589,10 +589,10 @@ .p2align 5 sub_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -607,15 +607,15 @@ stp x14,x15,[x0,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -628,10 +628,10 @@ .p2align 5 mul_by_1_plus_i_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x2] ldp x6,x7,[x2,#16] @@ -651,15 +651,15 @@ ldp x14,x15,[x1,#80] bl __add_mod_384_ab_are_loaded // a->re + a->im - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1017,7 +1017,7 @@ .Loop_is_zero_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq @@ -1048,7 +1048,7 @@ .Loop_is_equal_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq diff --git a/c-source/build/coff/add_mod_384-x86_64.s b/c-source/build/coff/add_mod_384-x86_64.s --- a/c-source/build/coff/add_mod_384-x86_64.s +++ b/c-source/build/coff/add_mod_384-x86_64.s @@ -53,7 +53,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_add_mod_384: @@ -62,6 +70,9 @@ __add_mod_384: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -105,7 +116,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl add_mod_384x @@ -171,7 +190,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_add_mod_384x: @@ -209,6 +236,9 @@ .LSEH_body_rshift_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -246,7 +276,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_rshift_mod_384: @@ -302,7 +340,15 @@ orq %rbp,%r12 orq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r14 + lfence + jmpq *%r14 + ud2 +#else .byte 0xf3,0xc3 +#endif .globl div_by_2_mod_384 @@ -337,6 +383,9 @@ .LSEH_body_div_by_2_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq %rdx,%rcx movq 8(%rsi),%r9 @@ -372,7 +421,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_div_by_2_mod_384: @@ -410,6 +467,9 @@ .LSEH_body_lshift_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -476,7 +536,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_lshift_mod_384: @@ -514,7 +582,15 @@ cmovcq %rbp,%r12 cmovcq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif @@ -550,6 +626,9 @@ .LSEH_body_mul_by_3_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -561,6 +640,9 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -581,7 +663,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_3_mod_384: @@ -617,6 +707,9 @@ .LSEH_body_mul_by_8_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -654,7 +747,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_8_mod_384: @@ -691,6 +792,9 @@ .LSEH_body_mul_by_3_mod_384x: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -702,11 +806,17 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq (%rsp),%rsi leaq 48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%r8 movq 56(%rsi),%r9 movq 64(%rsi),%r10 @@ -718,6 +828,9 @@ movq $48,%rdx addq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -738,7 +851,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_3_mod_384x: @@ -774,6 +895,9 @@ .LSEH_body_mul_by_8_mod_384x: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -794,6 +918,9 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -830,7 +957,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_8_mod_384x: @@ -868,6 +1003,9 @@ .LSEH_body_cneg_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -937,7 +1075,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_cneg_mod_384: @@ -995,7 +1141,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sub_mod_384: @@ -1004,6 +1158,9 @@ __sub_mod_384: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1045,7 +1202,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sub_mod_384x @@ -1111,7 +1276,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sub_mod_384x: .globl mul_by_1_plus_i_mod_384x @@ -1146,6 +1319,9 @@ .LSEH_body_mul_by_1_plus_i_mod_384x: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1261,7 +1437,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_by_1_plus_i_mod_384x: .globl sgn0_pty_mod_384 @@ -1280,6 +1464,9 @@ movq %rdx,%rsi .LSEH_body_sgn0_pty_mod_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -1314,7 +1501,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0_pty_mod_384: @@ -1341,6 +1536,9 @@ .LSEH_body_sgn0_pty_mod_384x: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rdi),%r8 movq 56(%rdi),%r9 movq 64(%rdi),%r10 @@ -1436,7 +1634,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0_pty_mod_384x: .globl vec_select_32 @@ -1449,6 +1655,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 16(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1466,7 +1675,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,16-16(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_select_48 @@ -1478,6 +1695,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 24(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1501,7 +1721,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,32-24(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_select_96 @@ -1513,6 +1741,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 48(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1554,7 +1785,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,80-48(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_select_192 @@ -1566,6 +1805,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 96(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1643,7 +1885,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,176-96(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_select_144 @@ -1655,6 +1905,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 72(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1714,7 +1967,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,128-72(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_select_288 @@ -1726,6 +1987,9 @@ movd %r9d,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdx),%xmm0 leaq 144(%rdx),%rdx pcmpeqd %xmm4,%xmm5 @@ -1839,7 +2103,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,272-144(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_prefetch @@ -1851,6 +2123,9 @@ leaq -1(%rcx,%rdx,1),%rdx movq $64,%rax xorq %r8,%r8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif prefetchnta (%rcx) leaq (%rcx,%rax,1),%rcx cmpq %rdx,%rcx @@ -1881,7 +2156,15 @@ cmpq %rdx,%rcx cmovaq %rdx,%rcx prefetchnta (%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_is_zero_16x @@ -1891,6 +2174,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%edx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rcx),%xmm0 leaq 16(%rcx),%rcx @@ -1910,7 +2196,15 @@ testq %rax,%rax cmovnzl %edx,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl vec_is_equal_16x @@ -1920,6 +2214,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%r8d +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rcx),%xmm0 movdqu (%rdx),%xmm1 subq %rcx,%rdx @@ -1944,7 +2241,15 @@ testq %rax,%rax cmovnzl %r8d,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/add_mod_384x384-x86_64.s b/c-source/build/coff/add_mod_384x384-x86_64.s --- a/c-source/build/coff/add_mod_384x384-x86_64.s +++ b/c-source/build/coff/add_mod_384x384-x86_64.s @@ -1,10 +1,41 @@ .text -.def __add_mod_384x384; .scl 3; .type 32; .endef +.globl add_mod_384x384 + +.def add_mod_384x384; .scl 2; .type 32; .endef .p2align 5 -__add_mod_384x384: +add_mod_384x384: .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_mod_384x384: + + pushq %rbp + + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_add_mod_384x384: + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -65,14 +96,72 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_add_mod_384x384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif +.LSEH_end_add_mod_384x384: -.def __sub_mod_384x384; .scl 3; .type 32; .endef +.globl sub_mod_384x384 + +.def sub_mod_384x384; .scl 2; .type 32; .endef .p2align 5 -__sub_mod_384x384: +sub_mod_384x384: .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_mod_384x384: + + pushq %rbp + + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sub_mod_384x384: + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -132,44 +221,6 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) - .byte 0xf3,0xc3 - - -.globl add_mod_384x384 - -.def add_mod_384x384; .scl 2; .type 32; .endef -.p2align 5 -add_mod_384x384: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_add_mod_384x384: - - - pushq %rbp - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $8,%rsp - -.LSEH_body_add_mod_384x384: - - - call __add_mod_384x384 - movq 8(%rsp),%r15 movq 16(%rsp),%r14 @@ -184,68 +235,19 @@ leaq 56(%rsp),%rsp -.LSEH_epilogue_add_mod_384x384: - mov 8(%rsp),%rdi - mov 16(%rsp),%rsi - - .byte 0xf3,0xc3 - -.LSEH_end_add_mod_384x384: - -.globl sub_mod_384x384 - -.def sub_mod_384x384; .scl 2; .type 32; .endef -.p2align 5 -sub_mod_384x384: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_sub_mod_384x384: - - - pushq %rbp - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $8,%rsp - -.LSEH_body_sub_mod_384x384: - - - call __sub_mod_384x384 - - movq 8(%rsp),%r15 - - movq 16(%rsp),%r14 - - movq 24(%rsp),%r13 - - movq 32(%rsp),%r12 - - movq 40(%rsp),%rbx - - movq 48(%rsp),%rbp - - leaq 56(%rsp),%rsp - .LSEH_epilogue_sub_mod_384x384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sub_mod_384x384: .section .pdata diff --git a/c-source/build/coff/ct_inverse_mod_256-armv8.S b/c-source/build/coff/ct_inverse_mod_256-armv8.S --- a/c-source/build/coff/ct_inverse_mod_256-armv8.S +++ b/c-source/build/coff/ct_inverse_mod_256-armv8.S @@ -8,20 +8,25 @@ .p2align 5 ct_inverse_mod_256: .long 3573752639 - stp x29, x30, [sp,#-80]! + stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] sub sp, sp, #1040 ldp x4, x5, [x1,#8*0] ldp x6, x7, [x1,#8*2] +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#16+511 + alignd c1,c1,#9 +#else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - str x0, [sp] +#endif + str x0, [sp] // offload out_ptr ldp x8, x9, [x2,#8*0] ldp x10, x11, [x2,#8*2] @@ -35,27 +40,36 @@ bl .Lab_approximation_31_256_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to dst |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 str x12, [x0,#8*9] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| @@ -74,318 +88,399 @@ stp x5, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, ldr x7, [x1,#8*0] // just load @@ -397,7 +492,7 @@ ldr x0, [sp] // original out_ptr bl __smul_256x63 bl __smul_512x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] smulh x20, x7, x17 // figure out top-most limb ldp x8, x9, [x3,#8*0] @@ -443,11 +538,11 @@ stp x6, x7, [x0,#8*6] add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldr x29, [sp],#80 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldr x29, [sp],#10*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/coff/ct_inverse_mod_256-x86_64.s b/c-source/build/coff/ct_inverse_mod_256-x86_64.s --- a/c-source/build/coff/ct_inverse_mod_256-x86_64.s +++ b/c-source/build/coff/ct_inverse_mod_256-x86_64.s @@ -38,6 +38,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -569,6 +572,9 @@ movq %rdx,%r8 movq %rdx,%r9 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 movq %rdx,%r10 andq 8(%rsi),%r9 @@ -634,7 +640,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_ct_inverse_mod_256: .def __smulq_512x63; .scl 3; .type 32; .endef @@ -783,7 +797,15 @@ movq %r14,48(%rdi) movq %r15,56(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulq_256x63; .scl 3; .type 32; .endef @@ -892,7 +914,15 @@ movq %r11,24(%rdi) movq %rbp,32(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulq_256_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 @@ -1021,7 +1051,15 @@ addq %rax,%rdx addq %rax,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __ab_approximation_31_256; .scl 3; .type 32; .endef .p2align 5 @@ -1075,7 +1113,15 @@ jmp __inner_loop_31_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_31_256; .scl 3; .type 32; .endef .p2align 5 @@ -1123,7 +1169,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_62_256; .scl 3; .type 32; .endef @@ -1167,7 +1221,15 @@ subl $1,%r15d jnz .Loop_62_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/ct_inverse_mod_384-armv8.S b/c-source/build/coff/ct_inverse_mod_384-armv8.S --- a/c-source/build/coff/ct_inverse_mod_384-armv8.S +++ b/c-source/build/coff/ct_inverse_mod_384-armv8.S @@ -8,22 +8,27 @@ .p2align 5 ct_inverse_mod_383: .long 3573752639 - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] - sub sp, sp, #1040 + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] + sub sp, sp, #1056 ldp x22, x4, [x1,#8*0] ldp x5, x6, [x1,#8*2] ldp x7, x8, [x1,#8*4] - add x1, sp, #16+511 // find closest 512-byte-aligned spot +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#32+511 + alignd c1,c1,#9 +#else + add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - stp x0, x3, [sp] +#endif + stp x0, x3, [sp] // offload out_ptr, nx_ptr ldp x9, x10, [x2,#8*0] ldp x11, x12, [x2,#8*2] @@ -41,28 +46,37 @@ bl .Lab_approximation_62_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to dst |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 str x15, [x0,#8*12] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 ldr x7, [x1,#8*12] // |u| @@ -89,205 +103,262 @@ stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, ldp x3, x8, [x1,#8*0] // just load @@ -295,6 +366,9 @@ bl __inner_loop_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -302,17 +376,20 @@ mov x21, x16 // exact |g0| mov x15, x17 mov x16, x19 - add x0, x0, #8*12 // pointer to dst |u| + add x0,x0,#8*12 bl __smul_383x63 mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0, x0, #8*6 // pointer to dst |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #22 // 766 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load @@ -323,10 +400,10 @@ mov x20, x17 mov x21, x19 - ldp x0, x15, [sp] // original out_ptr and n_ptr + ldp x0, x15, [sp] // original out_ptr and n_ptr bl __smul_383x63 bl __smul_767x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] asr x22, x8, #63 // sign as mask ldp x9, x10, [x15,#8*0] @@ -349,13 +426,13 @@ adc x8, x8, x14 stp x7, x8, [x0,#8*10] - add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + add sp, sp, #1056 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/coff/ct_is_square_mod_384-armv8.S b/c-source/build/coff/ct_is_square_mod_384-armv8.S --- a/c-source/build/coff/ct_is_square_mod_384-armv8.S +++ b/c-source/build/coff/ct_is_square_mod_384-armv8.S @@ -8,13 +8,13 @@ .p2align 5 ct_is_square_mod_384: .long 3573752639 - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] sub sp, sp, #512 ldp x3, x4, [x0,#8*0] // load input @@ -23,6 +23,9 @@ add x0, sp, #255 // find closest 256-byte-aligned spot and x0, x0, #-256 // in the frame... +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif ldp x9, x10, [x1,#8*0] // load modulus ldp x11, x12, [x1,#8*2] @@ -45,15 +48,21 @@ sub x15, x15, #1 eor x1, x0, #128 // pointer to dst |b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __smul_384_n_shift_by_30 mov x19, x16 // |f0| mov x20, x17 // |g0| - add x1, x1, #8*6 // pointer to dst |a| + add x1,x1,#8*6 bl __smul_384_n_shift_by_30 ldp x9, x10, [x1,#-8*6] eor x0, x0, #128 // flip-flop src |a|b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif and x27, x27, x9 // if |a| was negative, add x2, x2, x27, lsr#1 // adjust |L| @@ -65,18 +74,18 @@ //ldr x14, [x0,#8*0] mov x15, #48 // 48 is 768%30 + 30 bl __inner_loop_48 - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] and x0, x2, #1 eor x0, x0, #1 add sp, sp, #512 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/coff/ct_is_square_mod_384-x86_64.s b/c-source/build/coff/ct_is_square_mod_384-x86_64.s --- a/c-source/build/coff/ct_is_square_mod_384-x86_64.s +++ b/c-source/build/coff/ct_is_square_mod_384-x86_64.s @@ -34,6 +34,9 @@ leaq 24+255(%rsp),%rax andq $-256,%rax +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -123,7 +126,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_ct_is_square_mod_384: @@ -298,7 +309,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __ab_approximation_30; .scl 3; .type 32; .endef .p2align 5 @@ -363,7 +382,15 @@ jmp __inner_loop_30 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_30; .scl 3; .type 32; .endef .p2align 5 @@ -424,7 +451,15 @@ subq %r15,%rdx subq %r15,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_48; .scl 3; .type 32; .endef @@ -463,7 +498,15 @@ subl $1,%edi jnz .Loop_48 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/ctq_inverse_mod_384-x86_64.s b/c-source/build/coff/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/coff/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/coff/ctq_inverse_mod_384-x86_64.s @@ -544,7 +544,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_ct_inverse_mod_383: .def __smulq_767x63; .scl 3; .type 32; .endef @@ -756,7 +764,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulq_383x63; .scl 3; .type 32; .endef .p2align 5 @@ -898,7 +914,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulq_383_n_shift_by_62; .scl 3; .type 32; .endef .p2align 5 @@ -1075,7 +1099,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __ab_approximation_62; .scl 3; .type 32; .endef .p2align 5 @@ -1131,7 +1163,15 @@ jmp __inner_loop_62 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_62; .scl 3; .type 32; .endef .p2align 3 @@ -1184,7 +1224,15 @@ jnz .Loop_62 movq 8(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/ctx_inverse_mod_384-x86_64.s b/c-source/build/coff/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/coff/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/coff/ctx_inverse_mod_384-x86_64.s @@ -39,6 +39,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -840,6 +843,9 @@ movq %rax,%r8 movq %rax,%r9 movq %rax,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 movq %rax,%r11 @@ -882,7 +888,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_ctx_inverse_mod_383: .def __smulx_767x63; .scl 3; .type 32; .endef @@ -1048,7 +1062,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulx_383x63; .scl 3; .type 32; .endef .p2align 5 @@ -1156,7 +1178,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulx_383_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 @@ -1302,7 +1332,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __smulx_191_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 @@ -1394,7 +1432,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __ab_approximation_31; .scl 3; .type 32; .endef .p2align 5 @@ -1465,7 +1511,15 @@ jmp __inner_loop_31 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __inner_loop_31; .scl 3; .type 32; .endef .p2align 5 @@ -1513,7 +1567,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .def __tail_loop_53; .scl 3; .type 32; .endef @@ -1555,7 +1617,15 @@ subl $1,%edi jnz .Loop_53 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/div3w-armv8.S b/c-source/build/coff/div3w-armv8.S --- a/c-source/build/coff/div3w-armv8.S +++ b/c-source/build/coff/div3w-armv8.S @@ -1,6 +1,7 @@ .text .globl div_3_limbs + .def div_3_limbs; .type 32; .endef @@ -36,6 +37,7 @@ ret .globl quot_rem_128 + .def quot_rem_128; .type 32; .endef @@ -74,6 +76,7 @@ .globl quot_rem_64 + .def quot_rem_64; .type 32; .endef diff --git a/c-source/build/coff/div3w-x86_64.s b/c-source/build/coff/div3w-x86_64.s --- a/c-source/build/coff/div3w-x86_64.s +++ b/c-source/build/coff/div3w-x86_64.s @@ -17,6 +17,9 @@ movq %r8,%rdx .LSEH_body_div_3_limbs: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq (%rdi),%r8 movq 8(%rdi),%r9 xorq %rax,%rax @@ -52,7 +55,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_div_3_limbs: .globl quot_rem_128 @@ -72,6 +83,9 @@ movq %r8,%rdx .LSEH_body_quot_rem_128: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax movq %rdx,%rcx @@ -110,7 +124,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_quot_rem_128: @@ -135,6 +157,9 @@ movq %r8,%rdx .LSEH_body_quot_rem_64: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax imulq 0(%rsi),%rdx @@ -149,7 +174,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_quot_rem_64: .section .pdata diff --git a/c-source/build/coff/mul_mont_256-armv8.S b/c-source/build/coff/mul_mont_256-armv8.S --- a/c-source/build/coff/mul_mont_256-armv8.S +++ b/c-source/build/coff/mul_mont_256-armv8.S @@ -7,11 +7,11 @@ .endef .p2align 5 mul_mont_sparse_256: - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldr x9, [x2] @@ -187,10 +187,10 @@ stp x19,x20,[x0] stp x21,x22,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ ret .globl sqr_mont_sparse_256 @@ -201,10 +201,10 @@ .p2align 5 sqr_mont_sparse_256: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x5,x6,[x1] ldp x7,x8,[x1,#16] @@ -276,7 +276,7 @@ adc x22,x22,x8 bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] adds x10,x10,x19 // accumulate upper half adcs x11,x11,x20 @@ -298,9 +298,9 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -312,7 +312,7 @@ .p2align 5 from_mont_256: .long 3573752639 - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -320,7 +320,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x14,x10,x5 sbcs x15,x11,x6 @@ -335,7 +335,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .long 3573752767 ret @@ -348,7 +348,7 @@ .p2align 5 redc_mont_256: .long 3573752639 - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -356,7 +356,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x14,x15,[x1,#32] ldp x16,x17,[x1,#48] @@ -381,7 +381,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/coff/mul_mont_384-armv8.S b/c-source/build/coff/mul_mont_384-armv8.S --- a/c-source/build/coff/mul_mont_384-armv8.S +++ b/c-source/build/coff/mul_mont_384-armv8.S @@ -7,23 +7,23 @@ .p2align 5 add_mod_384x384: .long 3573752639 - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __add_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .long 3573752767 ret @@ -92,23 +92,23 @@ .p2align 5 sub_mod_384x384: .long 3573752639 - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .long 3573752767 ret @@ -254,23 +254,23 @@ .p2align 5 mul_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#288 // space for 3 768-bit vectors mov x26,x0 // save r_ptr mov x27,x1 // save b_ptr mov x28,x2 // save b_ptr - sub x0,sp,#0 // mul_384(t0, a->re, b->re) + add x0,sp,#0 bl __mul_384 - add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x1,x1,#48 add x2,x2,#48 add x0,sp,#96 bl __mul_384 @@ -285,7 +285,7 @@ add x1,x28,#0 add x2,x28,#48 - add x0,sp,#192 // t2 + add x0,sp,#192 bl __add_mod_384 add x1,x0,#0 @@ -308,24 +308,24 @@ add x0,sp,#0 bl __sub_mod_384x384 // t0 = t0-t1 - add x1,sp,#0 // ret->re = redc(t0) + add x1,sp,#0 add x0,x26,#0 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - add x1,sp,#192 // ret->im = redc(t2) + add x1,sp,#192 add x0,x0,#48 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#288 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -338,14 +338,14 @@ .p2align 5 sqr_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 2 384-bit vectors mov x4,x3 // adjust for missing b_ptr @@ -400,19 +400,19 @@ add x2,sp,#48 bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -425,14 +425,14 @@ .p2align 5 mul_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there ldp x11,x12,[x1] ldr x17, [x2] @@ -444,18 +444,18 @@ ldp x9,x10,[x3,#32] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -522,7 +522,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -582,7 +582,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -642,7 +642,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -702,7 +702,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -762,7 +762,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -806,7 +806,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adcs x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adc x17,x17,xzr adds x19,x20,x26 @@ -842,13 +842,13 @@ .p2align 5 sqr_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for 768-bit vector mov x4,x3 // adjust for missing b_ptr @@ -869,15 +869,15 @@ mov x0,x3 // restore r_ptr bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -890,14 +890,14 @@ .p2align 5 sqr_n_mul_mont_383: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 768-bit vector mov x17,x5 // save b_ptr @@ -932,19 +932,19 @@ mov x2,x17 ldr x17,[x17] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1071,27 +1071,27 @@ .p2align 5 sqr_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x13,x14,[x1,#16] ldp x15,x16,[x1,#32] bl __sqr_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1104,13 +1104,13 @@ .p2align 5 redc_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1119,14 +1119,14 @@ bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1139,13 +1139,13 @@ .p2align 5 from_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1153,7 +1153,7 @@ ldp x9,x10,[x2,#32] bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x19,x11,x5 sbcs x20,x12,x6 @@ -1173,12 +1173,12 @@ stp x13,x14,[x0,#16] stp x15,x16,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1404,23 +1404,23 @@ .p2align 5 mul_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1612,13 +1612,13 @@ .p2align 5 mul_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for two 384-bit vectors ldp x11,x12,[x1] @@ -1658,12 +1658,12 @@ bl __mul_384 // mul_384(ret->re, a->re, b->re) - add x1,sp,#0 // mul_384(ret->im, t0, t1) + add x1,sp,#0 add x2,sp,#48 add x0,x26,#96 bl __mul_384 - add x1,x27,#48 // mul_384(tx, a->im, b->im) + add x1,x27,#48 add x2,x28,#48 add x0,sp,#0 bl __mul_384 @@ -1672,27 +1672,27 @@ ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] - add x1,x26,#96 // ret->im -= tx + add x1,x26,#96 add x2,sp,#0 add x0,x26,#96 bl __sub_mod_384x384 - add x2,x26,#0 // ret->im -= ret->re + add x2,x26,#0 bl __sub_mod_384x384 - add x1,x26,#0 // ret->re -= tx + add x1,x26,#0 add x2,sp,#0 add x0,x26,#0 bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1705,13 +1705,13 @@ .p2align 5 sqr_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x19,x20,[x1,#48] @@ -1757,15 +1757,15 @@ stp x15,x16,[x0,#80] mov x4,x1 // save a_ptr - add x1,x0,#0 // mul_384(ret->re, t0, t1) + add x1,x0,#0 add x2,x0,#48 bl __mul_384 - add x1,x4,#0 // mul_384(ret->im, a->re, a->im) + add x1,x4,#0 add x2,x4,#48 add x0,x0,#96 bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x11,x12,[x0] ldp x13,x14,[x0,#16] @@ -1789,12 +1789,12 @@ stp x21,x22,[x0,#64] stp x23,x24,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1807,14 +1807,14 @@ .p2align 5 sqr_mont_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#112 // space for two 384-bit vectors + word mov x4,x3 // adjust for missing b_ptr @@ -1874,7 +1874,7 @@ add x2,sp,#48 bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldr x25,[sp,#96] // account for sign from a->re - a->im ldp x19,x20,[sp] @@ -1915,12 +1915,12 @@ stp x15,x16,[x2,#32] add sp,sp,#112 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1972,7 +1972,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2029,7 +2029,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2086,7 +2086,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2143,7 +2143,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2200,7 +2200,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2255,7 +2255,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adc x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adds x11,x20,x26 adcs x12,x21,x27 @@ -2275,13 +2275,13 @@ .p2align 5 sgn0_pty_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2290,7 +2290,7 @@ mov x1,x0 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 adds x11,x11,x11 @@ -2313,12 +2313,12 @@ and x17,x17,#2 orr x0,x0,x17 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -2331,13 +2331,13 @@ .p2align 5 sgn0_pty_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2375,7 +2375,7 @@ orr x2,x2,x17 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 orr x1,x11,x12 @@ -2413,12 +2413,12 @@ and x1,x1,#2 orr x0,x1,x3 // pack sign and parity - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/coff/mulq_mont_256-x86_64.s b/c-source/build/coff/mulq_mont_256-x86_64.s --- a/c-source/build/coff/mulq_mont_256-x86_64.s +++ b/c-source/build/coff/mulq_mont_256-x86_64.s @@ -71,7 +71,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_mont_sparse_256: @@ -145,7 +153,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_mont_sparse_256: .def __mulq_mont_sparse_256; .scl 3; .type 32; .endef @@ -426,7 +442,15 @@ movq %r15,16(%rsi) movq %r9,24(%rsi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl from_mont_256 @@ -509,7 +533,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_from_mont_256: @@ -599,7 +631,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_redc_mont_256: .def __mulq_by_1_mont_256; .scl 3; .type 32; .endef @@ -737,7 +777,15 @@ addq %r9,%r15 adcq $0,%rdx movq %rdx,%r9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/mulq_mont_384-x86_64.s b/c-source/build/coff/mulq_mont_384-x86_64.s --- a/c-source/build/coff/mulq_mont_384-x86_64.s +++ b/c-source/build/coff/mulq_mont_384-x86_64.s @@ -71,7 +71,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __addq_mod_384; .scl 3; .type 32; .endef @@ -121,7 +129,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __subq_mod_384; .scl 3; .type 32; .endef @@ -171,7 +187,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl mul_mont_384x @@ -296,7 +320,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_mont_384x: .globl sqr_mont_384x @@ -428,7 +460,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_mont_384x: @@ -573,7 +613,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_382x: .globl sqr_382x @@ -714,7 +762,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_382x: .globl mul_384 @@ -760,7 +816,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_384: @@ -1048,7 +1112,15 @@ movq %r11,80(%rdi) movq %r12,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqr_384 @@ -1105,7 +1177,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_384: @@ -1298,7 +1378,15 @@ movq %rax,80(%rdi) movq %rdx,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqr_mont_384 @@ -1371,7 +1459,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_mont_384: @@ -1436,7 +1532,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_redc_mont_384: @@ -1531,7 +1635,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_from_mont_384: .def __mulq_by_1_mont_384; .scl 3; .type 32; .endef @@ -1829,7 +1941,15 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __redq_tail_mont_384; .scl 3; .type 32; .endef @@ -1876,7 +1996,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sgn0_pty_mont_384 @@ -1961,7 +2089,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0_pty_mont_384: @@ -2097,7 +2233,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0_pty_mont_384x: .globl mul_mont_384 @@ -2167,7 +2311,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mul_mont_384: .def __mulq_mont_384; .scl 3; .type 32; .endef @@ -2770,7 +2922,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqr_n_mul_mont_384 @@ -2867,7 +3027,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_n_mul_mont_384: @@ -2979,7 +3147,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_n_mul_mont_383: .def __mulq_mont_383_nonred; .scl 3; .type 32; .endef @@ -3539,7 +3715,15 @@ adcq $0,%rdx addq %r12,%r10 adcq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqr_mont_382x @@ -3728,7 +3912,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqr_mont_382x: .section .pdata diff --git a/c-source/build/coff/mulx_mont_256-x86_64.s b/c-source/build/coff/mulx_mont_256-x86_64.s --- a/c-source/build/coff/mulx_mont_256-x86_64.s +++ b/c-source/build/coff/mulx_mont_256-x86_64.s @@ -36,6 +36,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -65,7 +68,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mulx_mont_sparse_256: @@ -106,6 +117,9 @@ movq %rsi,%rbx movq %rcx,%r8 movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rbp @@ -134,7 +148,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_mont_sparse_256: .def __mulx_mont_sparse_256; .scl 3; .type 32; .endef @@ -332,7 +354,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl fromx_mont_256 @@ -411,7 +441,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_fromx_mont_256: @@ -498,7 +536,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_redcx_mont_256: .def __mulx_by_1_mont_256; .scl 3; .type 32; .endef @@ -506,6 +552,9 @@ __mulx_by_1_mont_256: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rax movq 8(%rsi),%r11 movq 16(%rsi),%r12 @@ -636,7 +685,15 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/coff/mulx_mont_384-x86_64.s b/c-source/build/coff/mulx_mont_384-x86_64.s --- a/c-source/build/coff/mulx_mont_384-x86_64.s +++ b/c-source/build/coff/mulx_mont_384-x86_64.s @@ -70,7 +70,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __addx_mod_384; .scl 3; .type 32; .endef @@ -78,6 +86,9 @@ __addx_mod_384: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -120,7 +131,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __subx_mod_384; .scl 3; .type 32; .endef @@ -128,6 +147,9 @@ __subx_mod_384: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -170,7 +192,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl mulx_mont_384x @@ -218,6 +248,9 @@ leaq 40(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -231,11 +264,17 @@ leaq (%rbx),%rsi leaq -48(%rbx),%rdx leaq 40+192+48(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 movq 24(%rsp),%rsi leaq 48(%rsi),%rdx leaq -48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 leaq (%rdi),%rbx @@ -246,6 +285,9 @@ leaq (%rdi),%rsi leaq 40(%rsp),%rdx movq 8(%rsp),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 leaq (%rdi),%rsi @@ -293,7 +335,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mulx_mont_384x: .globl sqrx_mont_384x @@ -351,6 +401,9 @@ movq 24(%rsp),%rsi leaq 48(%rsi),%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -433,7 +486,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_mont_384x: @@ -478,6 +539,9 @@ movq %rcx,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -530,6 +594,9 @@ movq 0(%rsp),%rsi movq 8(%rsp),%rbx leaq -96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -543,6 +610,9 @@ leaq 32(%rsp),%rdx movq 24(%rsp),%rcx movq %rsi,%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 @@ -575,7 +645,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mulx_382x: .globl sqrx_382x @@ -614,6 +692,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r14 movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -656,6 +737,9 @@ movq (%rsp),%rsi leaq 48(%rsi),%rbx leaq 96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rdi),%r8 @@ -713,7 +797,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_382x: .globl mulx_384 @@ -748,6 +840,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rsp),%r15 @@ -768,7 +863,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mulx_384: @@ -943,7 +1046,15 @@ movq %r12,80(%rdi) movq %r13,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqrx_384 @@ -977,6 +1088,9 @@ .LSEH_body_sqrx_384: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __sqrx_384 movq 8(%rsp),%r15 @@ -997,7 +1111,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_384: .def __sqrx_384; .scl 3; .type 32; .endef @@ -1136,7 +1258,15 @@ movq %rbx,80(%rdi) movq %rbp,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif @@ -1176,6 +1306,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 call __redx_tail_mont_384 @@ -1197,7 +1330,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_redcx_mont_384: @@ -1239,6 +1380,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 @@ -1289,7 +1433,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_fromx_mont_384: .def __mulx_by_1_mont_384; .scl 3; .type 32; .endef @@ -1478,7 +1630,15 @@ adcxq %rax,%r10 adoxq %r11,%rbp adcxq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .def __redx_tail_mont_384; .scl 3; .type 32; .endef @@ -1525,7 +1685,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sgn0x_pty_mont_384 @@ -1564,6 +1732,9 @@ movq %rsi,%rbx leaq 0(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 xorq %rax,%rax @@ -1607,7 +1778,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0x_pty_mont_384: @@ -1647,6 +1826,9 @@ movq %rsi,%rbx leaq 48(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 movq %r14,%r12 @@ -1740,7 +1922,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sgn0x_pty_mont_384x: .globl mulx_mont_384 @@ -1779,6 +1969,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -1812,7 +2005,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_mulx_mont_384: .def __mulx_mont_384; .scl 3; .type 32; .endef @@ -2213,7 +2414,15 @@ movq %rdi,32(%rbx) movq %rbp,40(%rbx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqrx_mont_384 @@ -2252,6 +2461,9 @@ movq %rcx,%r8 leaq -128(%rdx),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2285,7 +2497,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_mont_384: @@ -2326,6 +2546,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2378,7 +2601,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_n_mul_mont_384: @@ -2419,6 +2650,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2470,7 +2704,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_n_mul_mont_383: .def __mulx_mont_383_nonred; .scl 3; .type 32; .endef @@ -2832,7 +3074,15 @@ movq %r10,40(%rbx) movq %r10,%rbp + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .globl sqrx_mont_382x @@ -2875,6 +3125,9 @@ movq %rsi,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -3029,7 +3282,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_sqrx_mont_382x: .section .pdata diff --git a/c-source/build/coff/sha256-armv8.S b/c-source/build/coff/sha256-armv8.S --- a/c-source/build/coff/sha256-armv8.S +++ b/c-source/build/coff/sha256-armv8.S @@ -49,7 +49,7 @@ .p2align 6 blst_sha256_block_armv8: .Lv8_entry: - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 ld1 {v0.4s,v1.4s},[x0] @@ -180,7 +180,7 @@ st1 {v0.4s,v1.4s},[x0] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ ret .globl blst_sha256_block_data_order @@ -194,7 +194,7 @@ tst w16,#1 b.ne .Lv8_entry - stp x29, x30, [sp, #-16]! + stp x29, x30, [sp, #-2*__SIZEOF_POINTER__]! mov x29, sp sub sp,sp,#16*4 @@ -675,11 +675,11 @@ sub x17,x17,#64 bne .L_00_48 - sub x16,x16,#256 // rewind x16 + sub x16,x16,#256 cmp x1,x2 - mov x17, #64 + mov x17, #-64 csel x17, x17, xzr, eq - sub x1,x1,x17 // avoid SEGV + add x1,x1,x17 mov x17,sp add w10,w10,w12 add w3,w3,w15 @@ -1030,7 +1030,7 @@ b.ne .L_00_48 ldr x29,[x29] - add sp,sp,#16*4+16 + add sp,sp,#16*4+2*__SIZEOF_POINTER__ ret .globl blst_sha256_emit diff --git a/c-source/build/coff/sha256-portable-x86_64.s b/c-source/build/coff/sha256-portable-x86_64.s --- a/c-source/build/coff/sha256-portable-x86_64.s +++ b/c-source/build/coff/sha256-portable-x86_64.s @@ -1655,11 +1655,20 @@ mov 16(%r11),%rsi leaq (%r11),%rsp + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_blst_sha256_block_data_order: #ifndef __BLST_PORTABLE__ +.section .rdata .p2align 6 K256: @@ -1708,7 +1717,15 @@ shrq $32,%r11 movl %r10d,16(%rcx) movl %r11d,24(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl blst_sha256_bcopy @@ -1725,7 +1742,15 @@ movb %al,-1(%rcx,%rdx,1) decq %r8 jnz .Loop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl blst_sha256_hcopy @@ -1743,7 +1768,15 @@ movq %r9,8(%rcx) movq %r10,16(%rcx) movq %r11,24(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif #endif .section .pdata diff --git a/c-source/build/coff/sha256-x86_64.s b/c-source/build/coff/sha256-x86_64.s --- a/c-source/build/coff/sha256-x86_64.s +++ b/c-source/build/coff/sha256-x86_64.s @@ -1,6 +1,6 @@ .comm __blst_platform_cap,4 -.text +.section .rdata .p2align 6 K256: @@ -25,6 +25,7 @@ .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.text .globl blst_sha256_block_data_order_shaext .def blst_sha256_block_data_order_shaext; .scl 2; .type 32; .endef @@ -55,6 +56,9 @@ .LSEH_body_blst_sha256_block_data_order_shaext: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif leaq K256+128(%rip),%rcx movdqu (%rdi),%xmm1 movdqu 16(%rdi),%xmm2 @@ -269,7 +273,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_blst_sha256_block_data_order_shaext: .globl blst_sha256_block_data_order @@ -291,8 +303,10 @@ movq %rcx,%rdi movq %rdx,%rsi movq %r8,%rdx +#ifndef __SGX_LVI_HARDENING__ testl $2,__blst_platform_cap(%rip) jnz .Lblst_sha256_block_data_order$2 +#endif pushq %rbx pushq %r12 @@ -319,6 +333,9 @@ leaq -64(%rsp),%rsp +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movl 0(%rdi),%eax andq $-64,%rsp movl 4(%rdi),%ebx @@ -1363,6 +1380,9 @@ movl %r14d,%eax movq -56(%rbp),%rsi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif addl 0(%rdi),%eax addl 4(%rdi),%ebx addl 8(%rdi),%ecx @@ -1407,7 +1427,15 @@ mov 8(%rsp),%rdi mov 16(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .LSEH_end_blst_sha256_block_data_order: .globl blst_sha256_emit @@ -1417,6 +1445,9 @@ blst_sha256_emit: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%r8 movq 8(%rdx),%r9 movq 16(%rdx),%r10 @@ -1437,7 +1468,15 @@ shrq $32,%r11 movl %r10d,16(%rcx) movl %r11d,24(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl blst_sha256_bcopy @@ -1447,6 +1486,9 @@ blst_sha256_bcopy: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif subq %rdx,%rcx .Loop_bcopy: movzbl (%rdx),%eax @@ -1454,7 +1496,15 @@ movb %al,-1(%rcx,%rdx,1) decq %r8 jnz .Loop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .globl blst_sha256_hcopy @@ -1464,6 +1514,9 @@ blst_sha256_hcopy: .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%r8 movq 8(%rdx),%r9 movq 16(%rdx),%r10 @@ -1472,7 +1525,15 @@ movq %r9,8(%rcx) movq %r10,16(%rcx) movq %r11,24(%rcx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .section .pdata .p2align 2 diff --git a/c-source/build/elf/add_mod_256-x86_64.s b/c-source/build/elf/add_mod_256-x86_64.s --- a/c-source/build/elf/add_mod_256-x86_64.s +++ b/c-source/build/elf/add_mod_256-x86_64.s @@ -19,6 +19,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -57,7 +60,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size add_mod_256,.-add_mod_256 @@ -83,6 +94,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -101,7 +115,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_3_mod_256,.-mul_by_3_mod_256 @@ -132,7 +154,15 @@ cmovcq %rbx,%r10 cmovcq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __lshift_mod_256,.-__lshift_mod_256 @@ -157,6 +187,9 @@ .cfi_offset %r12,-32 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -181,7 +214,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size lshift_mod_256,.-lshift_mod_256 @@ -205,6 +246,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rbp movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -261,7 +305,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size rshift_mod_256,.-rshift_mod_256 @@ -286,6 +338,9 @@ .cfi_offset %r12,-32 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r12 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -331,7 +386,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size cneg_mod_256,.-cneg_mod_256 @@ -355,6 +418,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -391,7 +457,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sub_mod_256,.-sub_mod_256 @@ -405,6 +479,9 @@ .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%rax movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -426,7 +503,15 @@ cmovneq %rdx,%rax andq %rsi,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size check_mod_256,.-check_mod_256 @@ -450,6 +535,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -493,7 +581,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size add_n_check_mod_256,.-add_n_check_mod_256 @@ -517,6 +613,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -559,14 +658,24 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sub_n_check_mod_256,.-sub_n_check_mod_256 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/add_mod_384-armv8.S b/c-source/build/elf/add_mod_384-armv8.S --- a/c-source/build/elf/add_mod_384-armv8.S +++ b/c-source/build/elf/add_mod_384-armv8.S @@ -6,25 +6,25 @@ .align 5 add_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size add_mod_384,.-add_mod_384 @@ -72,10 +72,10 @@ .align 5 add_mod_384x: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -90,15 +90,15 @@ stp x14,x15,[x0,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size add_mod_384x,.-add_mod_384x @@ -109,10 +109,10 @@ .align 5 rshift_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -127,14 +127,14 @@ bl __rshift_mod_384 cbnz x2,.Loop_rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size rshift_mod_384,.-rshift_mod_384 @@ -171,10 +171,10 @@ .align 5 div_by_2_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -186,14 +186,14 @@ bl __rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -204,10 +204,10 @@ .align 5 lshift_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -222,14 +222,14 @@ bl __lshift_mod_384 cbnz x2,.Loop_lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size lshift_mod_384,.-lshift_mod_384 @@ -269,10 +269,10 @@ .align 5 mul_by_3_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -289,15 +289,15 @@ ldp x21,x22,[x1,#32] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -308,10 +308,10 @@ .align 5 mul_by_8_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -324,15 +324,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -343,10 +343,10 @@ .align 5 mul_by_3_mod_384x: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -378,15 +378,15 @@ ldp x21,x22,[x1,#80] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -397,10 +397,10 @@ .align 5 mul_by_8_mod_384x: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -424,15 +424,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -443,10 +443,10 @@ .align 5 cneg_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x4,x5,[x3] @@ -481,9 +481,9 @@ csel x15,x15,x22,eq stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size cneg_mod_384,.-cneg_mod_384 @@ -494,25 +494,25 @@ .align 5 sub_mod_384: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sub_mod_384,.-sub_mod_384 @@ -557,10 +557,10 @@ .align 5 sub_mod_384x: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -575,15 +575,15 @@ stp x14,x15,[x0,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sub_mod_384x,.-sub_mod_384x @@ -594,10 +594,10 @@ .align 5 mul_by_1_plus_i_mod_384x: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x2] ldp x6,x7,[x2,#16] @@ -617,15 +617,15 @@ ldp x14,x15,[x1,#80] bl __add_mod_384_ab_are_loaded // a->re + a->im - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -963,7 +963,7 @@ .Loop_is_zero_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq @@ -992,7 +992,7 @@ .Loop_is_equal_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq diff --git a/c-source/build/elf/add_mod_384-x86_64.s b/c-source/build/elf/add_mod_384-x86_64.s --- a/c-source/build/elf/add_mod_384-x86_64.s +++ b/c-source/build/elf/add_mod_384-x86_64.s @@ -48,7 +48,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size add_mod_384,.-add_mod_384 @@ -58,6 +66,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -101,7 +112,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __add_mod_384,.-__add_mod_384 @@ -163,7 +182,15 @@ leaq 24+48(%rsp),%rsp .cfi_adjust_cfa_offset -24-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size add_mod_384x,.-add_mod_384x @@ -199,6 +226,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -233,7 +263,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size rshift_mod_384,.-rshift_mod_384 @@ -290,7 +328,15 @@ orq %rbp,%r12 orq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r14 + lfence + jmpq *%r14 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __rshift_mod_384,.-__rshift_mod_384 @@ -325,6 +371,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq %rdx,%rcx movq 8(%rsi),%r9 @@ -357,7 +406,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -393,6 +450,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -456,7 +516,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size lshift_mod_384,.-lshift_mod_384 @@ -495,7 +563,15 @@ cmovcq %rbp,%r12 cmovcq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __lshift_mod_384,.-__lshift_mod_384 @@ -531,6 +607,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -542,6 +621,9 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -559,7 +641,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -594,6 +684,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -628,7 +721,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -664,6 +765,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -675,11 +779,17 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq (%rsp),%rsi leaq 48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%r8 movq 56(%rsi),%r9 movq 64(%rsi),%r10 @@ -691,6 +801,9 @@ movq $48,%rdx addq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -708,7 +821,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -743,6 +864,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -763,6 +887,9 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -796,7 +923,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -832,6 +967,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -898,7 +1036,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size cneg_mod_384,.-cneg_mod_384 @@ -951,7 +1097,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sub_mod_384,.-sub_mod_384 @@ -961,6 +1115,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1002,7 +1159,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __sub_mod_384,.-__sub_mod_384 @@ -1064,7 +1229,15 @@ leaq 24+48(%rsp),%rsp .cfi_adjust_cfa_offset -24-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sub_mod_384x,.-sub_mod_384x .globl mul_by_1_plus_i_mod_384x @@ -1098,6 +1271,9 @@ .cfi_adjust_cfa_offset 56 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1210,7 +1386,15 @@ leaq 56+48(%rsp),%rsp .cfi_adjust_cfa_offset -56-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x .globl sgn0_pty_mod_384 @@ -1223,6 +1407,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -1254,7 +1441,15 @@ orq %rdi,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0_pty_mod_384,.-sgn0_pty_mod_384 @@ -1277,6 +1472,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rdi),%r8 movq 56(%rdi),%r9 movq 64(%rdi),%r10 @@ -1369,7 +1567,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0_pty_mod_384x,.-sgn0_pty_mod_384x .globl vec_select_32 @@ -1383,6 +1589,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 16(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1400,7 +1609,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,16-16(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -1414,6 +1631,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 24(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1437,7 +1657,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,32-24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_48,.-vec_select_48 .globl vec_select_96 @@ -1451,6 +1679,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 48(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1492,7 +1723,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,80-48(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_96,.-vec_select_96 .globl vec_select_192 @@ -1506,6 +1745,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 96(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1583,7 +1825,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,176-96(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_192,.-vec_select_192 .globl vec_select_144 @@ -1597,6 +1847,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 72(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1656,7 +1909,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,128-72(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_144,.-vec_select_144 .globl vec_select_288 @@ -1670,6 +1931,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 144(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1783,7 +2047,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,272-144(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_select_288,.-vec_select_288 .globl vec_prefetch @@ -1797,6 +2069,9 @@ leaq -1(%rdi,%rsi,1),%rsi movq $64,%rax xorq %r8,%r8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif prefetchnta (%rdi) leaq (%rdi,%rax,1),%rdi cmpq %rsi,%rdi @@ -1827,7 +2102,15 @@ cmpq %rsi,%rdi cmovaq %rsi,%rdi prefetchnta (%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_prefetch,.-vec_prefetch .globl vec_is_zero_16x @@ -1839,6 +2122,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%esi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdi),%xmm0 leaq 16(%rdi),%rdi @@ -1858,7 +2144,15 @@ testq %rax,%rax cmovnzl %esi,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_is_zero_16x,.-vec_is_zero_16x .globl vec_is_equal_16x @@ -1870,6 +2164,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%edx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdi),%xmm0 movdqu (%rsi),%xmm1 subq %rdi,%rsi @@ -1894,14 +2191,24 @@ testq %rax,%rax cmovnzl %edx,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size vec_is_equal_16x,.-vec_is_equal_16x .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/add_mod_384x384-x86_64.s b/c-source/build/elf/add_mod_384x384-x86_64.s --- a/c-source/build/elf/add_mod_384x384-x86_64.s +++ b/c-source/build/elf/add_mod_384x384-x86_64.s @@ -1,11 +1,39 @@ .text -.type __add_mod_384x384,@function +.globl add_mod_384x384 +.hidden add_mod_384x384 +.type add_mod_384x384,@function .align 32 -__add_mod_384x384: +add_mod_384x384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -66,16 +94,67 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 -.cfi_endproc -.size __add_mod_384x384,.-__add_mod_384x384 +#endif +.cfi_endproc +.size add_mod_384x384,.-add_mod_384x384 -.type __sub_mod_384x384,@function +.globl sub_mod_384x384 +.hidden sub_mod_384x384 +.type sub_mod_384x384,@function .align 32 -__sub_mod_384x384: +sub_mod_384x384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -135,43 +214,6 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) - .byte 0xf3,0xc3 -.cfi_endproc -.size __sub_mod_384x384,.-__sub_mod_384x384 - -.globl add_mod_384x384 -.hidden add_mod_384x384 -.type add_mod_384x384,@function -.align 32 -add_mod_384x384: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 - - - call __add_mod_384x384 - movq 8(%rsp),%r15 .cfi_restore %r15 movq 16(%rsp),%r14 @@ -187,66 +229,24 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 - .byte 0xf3,0xc3 -.cfi_endproc -.size add_mod_384x384,.-add_mod_384x384 - -.globl sub_mod_384x384 -.hidden sub_mod_384x384 -.type sub_mod_384x384,@function -.align 32 -sub_mod_384x384: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 - - - call __sub_mod_384x384 - - movq 8(%rsp),%r15 -.cfi_restore %r15 - movq 16(%rsp),%r14 -.cfi_restore %r14 - movq 24(%rsp),%r13 -.cfi_restore %r13 - movq 32(%rsp),%r12 -.cfi_restore %r12 - movq 40(%rsp),%rbx -.cfi_restore %rbx - movq 48(%rsp),%rbp -.cfi_restore %rbp - leaq 56(%rsp),%rsp -.cfi_adjust_cfa_offset -56 - + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sub_mod_384x384,.-sub_mod_384x384 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/ct_inverse_mod_256-armv8.S b/c-source/build/elf/ct_inverse_mod_256-armv8.S --- a/c-source/build/elf/ct_inverse_mod_256-armv8.S +++ b/c-source/build/elf/ct_inverse_mod_256-armv8.S @@ -6,20 +6,25 @@ .align 5 ct_inverse_mod_256: .inst 0xd503233f - stp x29, x30, [sp,#-80]! + stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] sub sp, sp, #1040 ldp x4, x5, [x1,#8*0] ldp x6, x7, [x1,#8*2] +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#16+511 + alignd c1,c1,#9 +#else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - str x0, [sp] +#endif + str x0, [sp] // offload out_ptr ldp x8, x9, [x2,#8*0] ldp x10, x11, [x2,#8*2] @@ -33,27 +38,36 @@ bl .Lab_approximation_31_256_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to dst |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 str x12, [x0,#8*9] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| @@ -72,318 +86,399 @@ stp x5, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, ldr x7, [x1,#8*0] // just load @@ -395,7 +490,7 @@ ldr x0, [sp] // original out_ptr bl __smul_256x63 bl __smul_512x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] smulh x20, x7, x17 // figure out top-most limb ldp x8, x9, [x3,#8*0] @@ -441,11 +536,11 @@ stp x6, x7, [x0,#8*6] add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldr x29, [sp],#80 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldr x29, [sp],#10*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 diff --git a/c-source/build/elf/ct_inverse_mod_256-x86_64.s b/c-source/build/elf/ct_inverse_mod_256-x86_64.s --- a/c-source/build/elf/ct_inverse_mod_256-x86_64.s +++ b/c-source/build/elf/ct_inverse_mod_256-x86_64.s @@ -36,6 +36,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -567,6 +570,9 @@ movq %rdx,%r8 movq %rdx,%r9 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 movq %rdx,%r10 andq 8(%rsi),%r9 @@ -629,7 +635,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1072-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size ct_inverse_mod_256,.-ct_inverse_mod_256 .type __smulq_512x63,@function @@ -779,7 +793,15 @@ movq %r14,48(%rdi) movq %r15,56(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_512x63,.-__smulq_512x63 @@ -890,7 +912,15 @@ movq %r11,24(%rdi) movq %rbp,32(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_256x63,.-__smulq_256x63 .type __smulq_256_n_shift_by_31,@function @@ -1021,7 +1051,15 @@ addq %rax,%rdx addq %rax,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_256_n_shift_by_31,.-__smulq_256_n_shift_by_31 .type __ab_approximation_31_256,@function @@ -1077,7 +1115,15 @@ jmp __inner_loop_31_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __ab_approximation_31_256,.-__ab_approximation_31_256 .type __inner_loop_31_256,@function @@ -1127,7 +1173,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_31_256,.-__inner_loop_31_256 @@ -1173,14 +1227,24 @@ subl $1,%r15d jnz .Loop_62_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_62_256,.-__inner_loop_62_256 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/ct_inverse_mod_384-armv8.S b/c-source/build/elf/ct_inverse_mod_384-armv8.S --- a/c-source/build/elf/ct_inverse_mod_384-armv8.S +++ b/c-source/build/elf/ct_inverse_mod_384-armv8.S @@ -6,22 +6,27 @@ .align 5 ct_inverse_mod_383: .inst 0xd503233f - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] - sub sp, sp, #1040 + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] + sub sp, sp, #1056 ldp x22, x4, [x1,#8*0] ldp x5, x6, [x1,#8*2] ldp x7, x8, [x1,#8*4] - add x1, sp, #16+511 // find closest 512-byte-aligned spot +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#32+511 + alignd c1,c1,#9 +#else + add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - stp x0, x3, [sp] +#endif + stp x0, x3, [sp] // offload out_ptr, nx_ptr ldp x9, x10, [x2,#8*0] ldp x11, x12, [x2,#8*2] @@ -39,28 +44,37 @@ bl .Lab_approximation_62_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to dst |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 str x15, [x0,#8*12] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 ldr x7, [x1,#8*12] // |u| @@ -87,205 +101,262 @@ stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, ldp x3, x8, [x1,#8*0] // just load @@ -293,6 +364,9 @@ bl __inner_loop_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -300,17 +374,20 @@ mov x21, x16 // exact |g0| mov x15, x17 mov x16, x19 - add x0, x0, #8*12 // pointer to dst |u| + add x0,x0,#8*12 bl __smul_383x63 mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0, x0, #8*6 // pointer to dst |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #22 // 766 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load @@ -321,10 +398,10 @@ mov x20, x17 mov x21, x19 - ldp x0, x15, [sp] // original out_ptr and n_ptr + ldp x0, x15, [sp] // original out_ptr and n_ptr bl __smul_383x63 bl __smul_767x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] asr x22, x8, #63 // sign as mask ldp x9, x10, [x15,#8*0] @@ -347,13 +424,13 @@ adc x8, x8, x14 stp x7, x8, [x0,#8*10] - add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + add sp, sp, #1056 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size ct_inverse_mod_383,.-ct_inverse_mod_383 diff --git a/c-source/build/elf/ct_is_square_mod_384-armv8.S b/c-source/build/elf/ct_is_square_mod_384-armv8.S --- a/c-source/build/elf/ct_is_square_mod_384-armv8.S +++ b/c-source/build/elf/ct_is_square_mod_384-armv8.S @@ -6,13 +6,13 @@ .align 5 ct_is_square_mod_384: .inst 0xd503233f - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] sub sp, sp, #512 ldp x3, x4, [x0,#8*0] // load input @@ -21,6 +21,9 @@ add x0, sp, #255 // find closest 256-byte-aligned spot and x0, x0, #-256 // in the frame... +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif ldp x9, x10, [x1,#8*0] // load modulus ldp x11, x12, [x1,#8*2] @@ -43,15 +46,21 @@ sub x15, x15, #1 eor x1, x0, #128 // pointer to dst |b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __smul_384_n_shift_by_30 mov x19, x16 // |f0| mov x20, x17 // |g0| - add x1, x1, #8*6 // pointer to dst |a| + add x1,x1,#8*6 bl __smul_384_n_shift_by_30 ldp x9, x10, [x1,#-8*6] eor x0, x0, #128 // flip-flop src |a|b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif and x27, x27, x9 // if |a| was negative, add x2, x2, x27, lsr#1 // adjust |L| @@ -63,18 +72,18 @@ //ldr x14, [x0,#8*0] mov x15, #48 // 48 is 768%30 + 30 bl __inner_loop_48 - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] and x0, x2, #1 eor x0, x0, #1 add sp, sp, #512 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 diff --git a/c-source/build/elf/ct_is_square_mod_384-x86_64.s b/c-source/build/elf/ct_is_square_mod_384-x86_64.s --- a/c-source/build/elf/ct_is_square_mod_384-x86_64.s +++ b/c-source/build/elf/ct_is_square_mod_384-x86_64.s @@ -34,6 +34,9 @@ leaq 24+255(%rsp),%rax andq $-256,%rax +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -120,7 +123,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -536-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -296,7 +307,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_384_n_shift_by_30,.-__smulq_384_n_shift_by_30 .type __ab_approximation_30,@function @@ -363,7 +382,15 @@ jmp __inner_loop_30 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __ab_approximation_30,.-__ab_approximation_30 .type __inner_loop_30,@function @@ -426,7 +453,15 @@ subq %r15,%rdx subq %r15,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_30,.-__inner_loop_30 @@ -467,14 +502,24 @@ subl $1,%edi jnz .Loop_48 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_48,.-__inner_loop_48 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/ctq_inverse_mod_384-x86_64.s b/c-source/build/elf/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/elf/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/elf/ctq_inverse_mod_384-x86_64.s @@ -539,7 +539,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1112-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size ct_inverse_mod_383,.-ct_inverse_mod_383 .type __smulq_767x63,@function @@ -752,7 +760,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_767x63,.-__smulq_767x63 .type __smulq_383x63,@function @@ -896,7 +912,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_383x63,.-__smulq_383x63 .type __smulq_383_n_shift_by_62,@function @@ -1075,7 +1099,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulq_383_n_shift_by_62,.-__smulq_383_n_shift_by_62 .type __ab_approximation_62,@function @@ -1133,7 +1165,15 @@ jmp __inner_loop_62 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __ab_approximation_62,.-__ab_approximation_62 .type __inner_loop_62,@function @@ -1188,14 +1228,24 @@ jnz .Loop_62 movq 8(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_62,.-__inner_loop_62 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/ctx_inverse_mod_384-x86_64.s b/c-source/build/elf/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/elf/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/elf/ctx_inverse_mod_384-x86_64.s @@ -37,6 +37,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -838,6 +841,9 @@ movq %rax,%r8 movq %rax,%r9 movq %rax,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 movq %rax,%r11 @@ -877,7 +883,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1112-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size ctx_inverse_mod_383,.-ctx_inverse_mod_383 .type __smulx_767x63,@function @@ -1044,7 +1058,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulx_767x63,.-__smulx_767x63 .type __smulx_383x63,@function @@ -1154,7 +1176,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulx_383x63,.-__smulx_383x63 .type __smulx_383_n_shift_by_31,@function @@ -1302,7 +1332,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulx_383_n_shift_by_31,.-__smulx_383_n_shift_by_31 .type __smulx_191_n_shift_by_31,@function @@ -1396,7 +1434,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __smulx_191_n_shift_by_31,.-__smulx_191_n_shift_by_31 .type __ab_approximation_31,@function @@ -1469,7 +1515,15 @@ jmp __inner_loop_31 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __ab_approximation_31,.-__ab_approximation_31 .type __inner_loop_31,@function @@ -1519,7 +1573,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __inner_loop_31,.-__inner_loop_31 @@ -1563,14 +1625,24 @@ subl $1,%edi jnz .Loop_53 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __tail_loop_53,.-__tail_loop_53 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/div3w-armv8.S b/c-source/build/elf/div3w-armv8.S --- a/c-source/build/elf/div3w-armv8.S +++ b/c-source/build/elf/div3w-armv8.S @@ -1,6 +1,7 @@ .text .globl div_3_limbs +.hidden div_3_limbs .type div_3_limbs,%function .align 5 div_3_limbs: @@ -34,6 +35,7 @@ ret .size div_3_limbs,.-div_3_limbs .globl quot_rem_128 +.hidden quot_rem_128 .type quot_rem_128,%function .align 5 quot_rem_128: @@ -70,6 +72,7 @@ .size quot_rem_128,.-quot_rem_128 .globl quot_rem_64 +.hidden quot_rem_64 .type quot_rem_64,%function .align 5 quot_rem_64: diff --git a/c-source/build/elf/div3w-x86_64.s b/c-source/build/elf/div3w-x86_64.s --- a/c-source/build/elf/div3w-x86_64.s +++ b/c-source/build/elf/div3w-x86_64.s @@ -10,6 +10,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq (%rdi),%r8 movq 8(%rdi),%r9 xorq %rax,%rax @@ -42,7 +45,15 @@ orq %rcx,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size div_3_limbs,.-div_3_limbs .globl quot_rem_128 @@ -55,6 +66,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax movq %rdx,%rcx @@ -90,7 +104,15 @@ movq %rcx,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size quot_rem_128,.-quot_rem_128 @@ -108,6 +130,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax imulq 0(%rsi),%rdx @@ -119,14 +144,24 @@ movq %rax,8(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size quot_rem_64,.-quot_rem_64 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/mul_mont_256-armv8.S b/c-source/build/elf/mul_mont_256-armv8.S --- a/c-source/build/elf/mul_mont_256-armv8.S +++ b/c-source/build/elf/mul_mont_256-armv8.S @@ -5,11 +5,11 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldr x9, [x2] @@ -185,10 +185,10 @@ stp x19,x20,[x0] stp x21,x22,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ ret .size mul_mont_sparse_256,.-mul_mont_sparse_256 .globl sqr_mont_sparse_256 @@ -197,10 +197,10 @@ .align 5 sqr_mont_sparse_256: .inst 0xd503233f - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x5,x6,[x1] ldp x7,x8,[x1,#16] @@ -272,7 +272,7 @@ adc x22,x22,x8 bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] adds x10,x10,x19 // accumulate upper half adcs x11,x11,x20 @@ -294,9 +294,9 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 @@ -306,7 +306,7 @@ .align 5 from_mont_256: .inst 0xd503233f - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -314,7 +314,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x14,x10,x5 sbcs x15,x11,x6 @@ -329,7 +329,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size from_mont_256,.-from_mont_256 @@ -340,7 +340,7 @@ .align 5 redc_mont_256: .inst 0xd503233f - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -348,7 +348,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x14,x15,[x1,#32] ldp x16,x17,[x1,#48] @@ -373,7 +373,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size redc_mont_256,.-redc_mont_256 diff --git a/c-source/build/elf/mul_mont_384-armv8.S b/c-source/build/elf/mul_mont_384-armv8.S --- a/c-source/build/elf/mul_mont_384-armv8.S +++ b/c-source/build/elf/mul_mont_384-armv8.S @@ -5,23 +5,23 @@ .align 5 add_mod_384x384: .inst 0xd503233f - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __add_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size add_mod_384x384,.-add_mod_384x384 @@ -86,23 +86,23 @@ .align 5 sub_mod_384x384: .inst 0xd503233f - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -240,23 +240,23 @@ .align 5 mul_mont_384x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#288 // space for 3 768-bit vectors mov x26,x0 // save r_ptr mov x27,x1 // save b_ptr mov x28,x2 // save b_ptr - sub x0,sp,#0 // mul_384(t0, a->re, b->re) + add x0,sp,#0 bl __mul_384 - add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x1,x1,#48 add x2,x2,#48 add x0,sp,#96 bl __mul_384 @@ -271,7 +271,7 @@ add x1,x28,#0 add x2,x28,#48 - add x0,sp,#192 // t2 + add x0,sp,#192 bl __add_mod_384 add x1,x0,#0 @@ -294,24 +294,24 @@ add x0,sp,#0 bl __sub_mod_384x384 // t0 = t0-t1 - add x1,sp,#0 // ret->re = redc(t0) + add x1,sp,#0 add x0,x26,#0 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - add x1,sp,#192 // ret->im = redc(t2) + add x1,sp,#192 add x0,x0,#48 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#288 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_mont_384x,.-mul_mont_384x @@ -322,14 +322,14 @@ .align 5 sqr_mont_384x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 2 384-bit vectors mov x4,x3 // adjust for missing b_ptr @@ -384,19 +384,19 @@ add x2,sp,#48 bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_mont_384x,.-sqr_mont_384x @@ -407,14 +407,14 @@ .align 5 mul_mont_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there ldp x11,x12,[x1] ldr x17, [x2] @@ -426,18 +426,18 @@ ldp x9,x10,[x3,#32] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_mont_384,.-mul_mont_384 @@ -502,7 +502,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -562,7 +562,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -622,7 +622,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -682,7 +682,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -742,7 +742,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -786,7 +786,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adcs x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adc x17,x17,xzr adds x19,x20,x26 @@ -820,13 +820,13 @@ .align 5 sqr_mont_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for 768-bit vector mov x4,x3 // adjust for missing b_ptr @@ -847,15 +847,15 @@ mov x0,x3 // restore r_ptr bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_mont_384,.-sqr_mont_384 @@ -866,14 +866,14 @@ .align 5 sqr_n_mul_mont_383: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 768-bit vector mov x17,x5 // save b_ptr @@ -908,19 +908,19 @@ mov x2,x17 ldr x17,[x17] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 @@ -1043,27 +1043,27 @@ .align 5 sqr_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x13,x14,[x1,#16] ldp x15,x16,[x1,#32] bl __sqr_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_384,.-sqr_384 @@ -1074,13 +1074,13 @@ .align 5 redc_mont_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1089,14 +1089,14 @@ bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size redc_mont_384,.-redc_mont_384 @@ -1107,13 +1107,13 @@ .align 5 from_mont_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1121,7 +1121,7 @@ ldp x9,x10,[x2,#32] bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x19,x11,x5 sbcs x20,x12,x6 @@ -1141,12 +1141,12 @@ stp x13,x14,[x0,#16] stp x15,x16,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size from_mont_384,.-from_mont_384 @@ -1366,23 +1366,23 @@ .align 5 mul_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_384,.-mul_384 @@ -1570,13 +1570,13 @@ .align 5 mul_382x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for two 384-bit vectors ldp x11,x12,[x1] @@ -1616,12 +1616,12 @@ bl __mul_384 // mul_384(ret->re, a->re, b->re) - add x1,sp,#0 // mul_384(ret->im, t0, t1) + add x1,sp,#0 add x2,sp,#48 add x0,x26,#96 bl __mul_384 - add x1,x27,#48 // mul_384(tx, a->im, b->im) + add x1,x27,#48 add x2,x28,#48 add x0,sp,#0 bl __mul_384 @@ -1630,27 +1630,27 @@ ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] - add x1,x26,#96 // ret->im -= tx + add x1,x26,#96 add x2,sp,#0 add x0,x26,#96 bl __sub_mod_384x384 - add x2,x26,#0 // ret->im -= ret->re + add x2,x26,#0 bl __sub_mod_384x384 - add x1,x26,#0 // ret->re -= tx + add x1,x26,#0 add x2,sp,#0 add x0,x26,#0 bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size mul_382x,.-mul_382x @@ -1661,13 +1661,13 @@ .align 5 sqr_382x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x19,x20,[x1,#48] @@ -1713,15 +1713,15 @@ stp x15,x16,[x0,#80] mov x4,x1 // save a_ptr - add x1,x0,#0 // mul_384(ret->re, t0, t1) + add x1,x0,#0 add x2,x0,#48 bl __mul_384 - add x1,x4,#0 // mul_384(ret->im, a->re, a->im) + add x1,x4,#0 add x2,x4,#48 add x0,x0,#96 bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x11,x12,[x0] ldp x13,x14,[x0,#16] @@ -1745,12 +1745,12 @@ stp x21,x22,[x0,#64] stp x23,x24,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_382x,.-sqr_382x @@ -1761,14 +1761,14 @@ .align 5 sqr_mont_382x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#112 // space for two 384-bit vectors + word mov x4,x3 // adjust for missing b_ptr @@ -1828,7 +1828,7 @@ add x2,sp,#48 bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldr x25,[sp,#96] // account for sign from a->re - a->im ldp x19,x20,[sp] @@ -1869,12 +1869,12 @@ stp x15,x16,[x2,#32] add sp,sp,#112 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sqr_mont_382x,.-sqr_mont_382x @@ -1924,7 +1924,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -1981,7 +1981,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2038,7 +2038,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2095,7 +2095,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2152,7 +2152,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2207,7 +2207,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adc x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adds x11,x20,x26 adcs x12,x21,x27 @@ -2225,13 +2225,13 @@ .align 5 sgn0_pty_mont_384: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2240,7 +2240,7 @@ mov x1,x0 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 adds x11,x11,x11 @@ -2263,12 +2263,12 @@ and x17,x17,#2 orr x0,x0,x17 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2279,13 +2279,13 @@ .align 5 sgn0_pty_mont_384x: .inst 0xd503233f - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2323,7 +2323,7 @@ orr x2,x2,x17 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 orr x1,x11,x12 @@ -2361,12 +2361,12 @@ and x1,x1,#2 orr x0,x1,x3 // pack sign and parity - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .inst 0xd50323bf ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x diff --git a/c-source/build/elf/mulq_mont_256-x86_64.s b/c-source/build/elf/mulq_mont_256-x86_64.s --- a/c-source/build/elf/mulq_mont_256-x86_64.s +++ b/c-source/build/elf/mulq_mont_256-x86_64.s @@ -65,7 +65,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_mont_sparse_256,.-mul_mont_sparse_256 @@ -134,7 +142,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .type __mulq_mont_sparse_256,@function @@ -416,7 +432,15 @@ movq %r15,16(%rsi) movq %r9,24(%rsi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_mont_sparse_256,.-__mulq_mont_sparse_256 .globl from_mont_256 @@ -494,7 +518,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size from_mont_256,.-from_mont_256 @@ -579,7 +611,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size redc_mont_256,.-redc_mont_256 .type __mulq_by_1_mont_256,@function @@ -718,14 +758,24 @@ addq %r9,%r15 adcq $0,%rdx movq %rdx,%r9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_by_1_mont_256,.-__mulq_by_1_mont_256 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/mulq_mont_384-x86_64.s b/c-source/build/elf/mulq_mont_384-x86_64.s --- a/c-source/build/elf/mulq_mont_384-x86_64.s +++ b/c-source/build/elf/mulq_mont_384-x86_64.s @@ -72,7 +72,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __subq_mod_384x384,.-__subq_mod_384x384 @@ -124,7 +132,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __addq_mod_384,.-__addq_mod_384 @@ -176,7 +192,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __subq_mod_384,.-__subq_mod_384 .globl mul_mont_384x @@ -296,7 +320,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -328-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_mont_384x,.-mul_mont_384x .globl sqr_mont_384x @@ -423,7 +455,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_mont_384x,.-sqr_mont_384x @@ -563,7 +603,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_382x,.-mul_382x .globl sqr_382x @@ -700,7 +748,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -8*7 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_382x,.-sqr_382x .globl mul_384 @@ -739,7 +795,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_384,.-mul_384 @@ -1028,7 +1092,15 @@ movq %r11,80(%rdi) movq %r12,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_384,.-__mulq_384 .globl sqr_384 @@ -1083,7 +1155,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_384,.-sqr_384 @@ -1277,7 +1357,15 @@ movq %rax,80(%rdi) movq %rdx,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __sqrq_384,.-__sqrq_384 @@ -1346,7 +1434,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*21 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_mont_384,.-sqr_mont_384 @@ -1406,7 +1502,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size redc_mont_384,.-redc_mont_384 @@ -1496,7 +1600,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size from_mont_384,.-from_mont_384 .type __mulq_by_1_mont_384,@function @@ -1795,7 +1907,15 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_by_1_mont_384,.-__mulq_by_1_mont_384 @@ -1844,7 +1964,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __redq_tail_mont_384,.-__redq_tail_mont_384 @@ -1926,7 +2054,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2058,7 +2194,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x .globl mul_mont_384 @@ -2122,7 +2266,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -72 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mul_mont_384,.-mul_mont_384 .type __mulq_mont_384,@function @@ -2726,7 +2878,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_mont_384,.-__mulq_mont_384 .globl sqr_n_mul_mont_384 @@ -2817,7 +2977,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*23 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_n_mul_mont_384,.-sqr_n_mul_mont_384 @@ -2922,7 +3090,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*23 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __mulq_mont_383_nonred,@function @@ -3483,7 +3659,15 @@ adcq $0,%rdx addq %r12,%r10 adcq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulq_mont_383_nonred,.-__mulq_mont_383_nonred .globl sqr_mont_382x @@ -3668,14 +3852,24 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqr_mont_382x,.-sqr_mont_382x .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/mulx_mont_256-x86_64.s b/c-source/build/elf/mulx_mont_256-x86_64.s --- a/c-source/build/elf/mulx_mont_256-x86_64.s +++ b/c-source/build/elf/mulx_mont_256-x86_64.s @@ -33,6 +33,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -59,7 +62,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mulx_mont_sparse_256,.-mulx_mont_sparse_256 @@ -98,6 +109,9 @@ movq %rsi,%rbx movq %rcx,%r8 movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rbp @@ -123,7 +137,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_mont_sparse_256,.-sqrx_mont_sparse_256 .type __mulx_mont_sparse_256,@function @@ -322,7 +344,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_mont_sparse_256,.-__mulx_mont_sparse_256 .globl fromx_mont_256 @@ -397,7 +427,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size fromx_mont_256,.-fromx_mont_256 @@ -479,7 +517,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size redcx_mont_256,.-redcx_mont_256 .type __mulx_by_1_mont_256,@function @@ -488,6 +534,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rax movq 8(%rsi),%r11 movq 16(%rsi),%r12 @@ -618,14 +667,24 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_by_1_mont_256,.-__mulx_by_1_mont_256 .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/mulx_mont_384-x86_64.s b/c-source/build/elf/mulx_mont_384-x86_64.s --- a/c-source/build/elf/mulx_mont_384-x86_64.s +++ b/c-source/build/elf/mulx_mont_384-x86_64.s @@ -71,7 +71,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __subx_mod_384x384,.-__subx_mod_384x384 @@ -81,6 +89,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -123,7 +134,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __addx_mod_384,.-__addx_mod_384 @@ -133,6 +152,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -175,7 +197,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __subx_mod_384,.-__subx_mod_384 .globl mulx_mont_384x @@ -221,6 +251,9 @@ leaq 40(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -234,11 +267,17 @@ leaq (%rbx),%rsi leaq -48(%rbx),%rdx leaq 40+192+48(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 movq 24(%rsp),%rsi leaq 48(%rsi),%rdx leaq -48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 leaq (%rdi),%rbx @@ -249,6 +288,9 @@ leaq (%rdi),%rsi leaq 40(%rsp),%rdx movq 8(%rsp),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 leaq (%rdi),%rsi @@ -293,7 +335,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -328-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mulx_mont_384x,.-mulx_mont_384x .globl sqrx_mont_384x @@ -349,6 +399,9 @@ movq 24(%rsp),%rsi leaq 48(%rsi),%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -428,7 +481,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_mont_384x,.-sqrx_mont_384x @@ -471,6 +532,9 @@ movq %rcx,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -523,6 +587,9 @@ movq 0(%rsp),%rsi movq 8(%rsp),%rbx leaq -96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -536,6 +603,9 @@ leaq 32(%rsp),%rdx movq 24(%rsp),%rcx movq %rsi,%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 @@ -565,7 +635,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mulx_382x,.-mulx_382x .globl sqrx_382x @@ -603,6 +681,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r14 movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -645,6 +726,9 @@ movq (%rsp),%rsi leaq 48(%rsi),%rbx leaq 96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rdi),%r8 @@ -699,7 +783,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -8*7 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_382x,.-sqrx_382x .globl mulx_384 @@ -733,6 +825,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rsp),%r15 @@ -750,7 +845,15 @@ leaq 48(%rsp),%rsp .cfi_adjust_cfa_offset -48 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mulx_384,.-mulx_384 @@ -926,7 +1029,15 @@ movq %r12,80(%rdi) movq %r13,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_384,.-__mulx_384 .globl sqrx_384 @@ -961,6 +1072,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __sqrx_384 movq 8(%rsp),%r15 @@ -978,7 +1092,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_384,.-sqrx_384 .type __sqrx_384,@function @@ -1118,7 +1240,15 @@ movq %rbx,80(%rdi) movq %rbp,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __sqrx_384,.-__sqrx_384 @@ -1157,6 +1287,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 call __redx_tail_mont_384 @@ -1175,7 +1308,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size redcx_mont_384,.-redcx_mont_384 @@ -1215,6 +1356,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 @@ -1262,7 +1406,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size fromx_mont_384,.-fromx_mont_384 .type __mulx_by_1_mont_384,@function @@ -1452,7 +1604,15 @@ adcxq %rax,%r10 adoxq %r11,%rbp adcxq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_by_1_mont_384,.-__mulx_by_1_mont_384 @@ -1501,7 +1661,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __redx_tail_mont_384,.-__redx_tail_mont_384 @@ -1540,6 +1708,9 @@ movq %rsi,%rbx leaq 0(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 xorq %rax,%rax @@ -1580,7 +1751,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0x_pty_mont_384,.-sgn0x_pty_mont_384 @@ -1619,6 +1798,9 @@ movq %rsi,%rbx leaq 48(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 movq %r14,%r12 @@ -1709,7 +1891,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sgn0x_pty_mont_384x,.-sgn0x_pty_mont_384x .globl mulx_mont_384 @@ -1745,6 +1935,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -1775,7 +1968,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -8*9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size mulx_mont_384,.-mulx_mont_384 .type __mulx_mont_384,@function @@ -2177,7 +2378,15 @@ movq %rdi,32(%rbx) movq %rbp,40(%rbx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_mont_384,.-__mulx_mont_384 .globl sqrx_mont_384 @@ -2214,6 +2423,9 @@ movq %rcx,%r8 leaq -128(%rdx),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2244,7 +2456,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -8*9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_mont_384,.-sqrx_mont_384 @@ -2281,6 +2501,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2330,7 +2553,15 @@ leaq 88(%rsp),%rsp .cfi_adjust_cfa_offset -8*11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_n_mul_mont_384,.-sqrx_n_mul_mont_384 @@ -2367,6 +2598,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2415,7 +2649,15 @@ leaq 88(%rsp),%rsp .cfi_adjust_cfa_offset -8*11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_n_mul_mont_383,.-sqrx_n_mul_mont_383 .type __mulx_mont_383_nonred,@function @@ -2778,7 +3020,15 @@ movq %r10,40(%rbx) movq %r10,%rbp + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size __mulx_mont_383_nonred,.-__mulx_mont_383_nonred .globl sqrx_mont_382x @@ -2819,6 +3069,9 @@ movq %rsi,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -2970,14 +3223,24 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size sqrx_mont_382x,.-sqrx_mont_382x .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/sha256-armv8.S b/c-source/build/elf/sha256-armv8.S --- a/c-source/build/elf/sha256-armv8.S +++ b/c-source/build/elf/sha256-armv8.S @@ -47,7 +47,7 @@ .align 6 blst_sha256_block_armv8: .Lv8_entry: - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 ld1 {v0.4s,v1.4s},[x0] @@ -178,7 +178,7 @@ st1 {v0.4s,v1.4s},[x0] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ ret .size blst_sha256_block_armv8,.-blst_sha256_block_armv8 .globl blst_sha256_block_data_order @@ -190,7 +190,7 @@ tst w16,#1 b.ne .Lv8_entry - stp x29, x30, [sp, #-16]! + stp x29, x30, [sp, #-2*__SIZEOF_POINTER__]! mov x29, sp sub sp,sp,#16*4 @@ -671,11 +671,11 @@ sub x17,x17,#64 bne .L_00_48 - sub x16,x16,#256 // rewind x16 + sub x16,x16,#256 cmp x1,x2 - mov x17, #64 + mov x17, #-64 csel x17, x17, xzr, eq - sub x1,x1,x17 // avoid SEGV + add x1,x1,x17 mov x17,sp add w10,w10,w12 add w3,w3,w15 @@ -1026,7 +1026,7 @@ b.ne .L_00_48 ldr x29,[x29] - add sp,sp,#16*4+16 + add sp,sp,#16*4+2*__SIZEOF_POINTER__ ret .size blst_sha256_block_data_order,.-blst_sha256_block_data_order .globl blst_sha256_emit diff --git a/c-source/build/elf/sha256-portable-x86_64.s b/c-source/build/elf/sha256-portable-x86_64.s --- a/c-source/build/elf/sha256-portable-x86_64.s +++ b/c-source/build/elf/sha256-portable-x86_64.s @@ -1651,11 +1651,20 @@ .cfi_restore %rbp .cfi_restore %rbx leaq (%r11),%rsp + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_block_data_order,.-blst_sha256_block_data_order #ifndef __BLST_PORTABLE__ +.section .rodata .align 64 .type K256,@object K256: @@ -1705,7 +1714,15 @@ shrq $32,%r11 movl %r10d,16(%rdi) movl %r11d,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_emit,.-blst_sha256_emit @@ -1724,7 +1741,15 @@ movb %al,-1(%rdi,%rsi,1) decq %rdx jnz .Loop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_bcopy,.-blst_sha256_bcopy @@ -1744,15 +1769,25 @@ movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_hcopy,.-blst_sha256_hcopy #endif .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/elf/sha256-x86_64.s b/c-source/build/elf/sha256-x86_64.s --- a/c-source/build/elf/sha256-x86_64.s +++ b/c-source/build/elf/sha256-x86_64.s @@ -1,6 +1,6 @@ .comm __blst_platform_cap,4 -.text +.section .rodata .align 64 .type K256,@object K256: @@ -25,6 +25,7 @@ .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.text .globl blst_sha256_block_data_order_shaext .hidden blst_sha256_block_data_order_shaext .type blst_sha256_block_data_order_shaext,@function @@ -41,6 +42,9 @@ .cfi_def_cfa_register %rbp .Lblst_sha256_block_data_order$2: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif leaq K256+128(%rip),%rcx movdqu (%rdi),%xmm1 movdqu 16(%rdi),%xmm2 @@ -247,7 +251,15 @@ .cfi_adjust_cfa_offset -8 .cfi_restore %rbp + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_block_data_order_shaext,.-blst_sha256_block_data_order_shaext .globl blst_sha256_block_data_order @@ -264,8 +276,10 @@ .cfi_offset %rbp,-16 movq %rsp,%rbp .cfi_def_cfa_register %rbp +#ifndef __SGX_LVI_HARDENING__ testl $2,__blst_platform_cap(%rip) jnz .Lblst_sha256_block_data_order$2 +#endif pushq %rbx .cfi_offset %rbx,-24 pushq %r12 @@ -286,6 +300,9 @@ leaq -64(%rsp),%rsp +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movl 0(%rdi),%eax andq $-64,%rsp movl 4(%rdi),%ebx @@ -1330,6 +1347,9 @@ movl %r14d,%eax movq -56(%rbp),%rsi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif addl 0(%rdi),%eax addl 4(%rdi),%ebx addl 8(%rdi),%ecx @@ -1372,7 +1392,15 @@ .cfi_restore %r14 .cfi_restore %r15 .cfi_restore %rbx + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_block_data_order,.-blst_sha256_block_data_order .globl blst_sha256_emit @@ -1383,6 +1411,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1403,7 +1434,15 @@ shrq $32,%r11 movl %r10d,16(%rdi) movl %r11d,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_emit,.-blst_sha256_emit @@ -1415,6 +1454,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif subq %rsi,%rdi .Loop_bcopy: movzbl (%rsi),%eax @@ -1422,7 +1464,15 @@ movb %al,-1(%rdi,%rsi,1) decq %rdx jnz .Loop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_bcopy,.-blst_sha256_bcopy @@ -1434,6 +1484,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1442,14 +1495,24 @@ movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .size blst_sha256_hcopy,.-blst_sha256_hcopy .section .note.GNU-stack,"",@progbits +#ifndef __SGX_LVI_HARDENING__ .section .note.gnu.property,"a",@note .long 4,2f-1f,5 .byte 0x47,0x4E,0x55,0 1: .long 0xc0000002,4,3 .align 8 2: +#endif diff --git a/c-source/build/mach-o/add_mod_256-x86_64.s b/c-source/build/mach-o/add_mod_256-x86_64.s --- a/c-source/build/mach-o/add_mod_256-x86_64.s +++ b/c-source/build/mach-o/add_mod_256-x86_64.s @@ -19,6 +19,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -57,7 +60,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -83,6 +94,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -101,7 +115,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -132,7 +154,15 @@ cmovcq %rbx,%r10 cmovcq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -157,6 +187,9 @@ .cfi_offset %r12,-32 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -181,7 +214,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -205,6 +246,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rbp movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -261,7 +305,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -286,6 +338,9 @@ .cfi_offset %r12,-32 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r12 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -331,7 +386,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -355,6 +418,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -391,7 +457,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -405,6 +479,9 @@ .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%rax movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -426,7 +503,15 @@ cmovneq %rdx,%rax andq %rsi,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -450,6 +535,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -493,7 +581,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -517,6 +613,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -559,6 +658,14 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/add_mod_384-armv8.S b/c-source/build/mach-o/add_mod_384-armv8.S --- a/c-source/build/mach-o/add_mod_384-armv8.S +++ b/c-source/build/mach-o/add_mod_384-armv8.S @@ -6,25 +6,25 @@ .align 5 _add_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -72,10 +72,10 @@ .align 5 _add_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -90,15 +90,15 @@ stp x14,x15,[x0,#32] bl __add_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -109,10 +109,10 @@ .align 5 _rshift_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -127,14 +127,14 @@ bl __rshift_mod_384 cbnz x2,Loop_rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -171,10 +171,10 @@ .align 5 _div_by_2_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -186,14 +186,14 @@ bl __rshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -204,10 +204,10 @@ .align 5 _lshift_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -222,14 +222,14 @@ bl __lshift_mod_384 cbnz x2,Loop_lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -269,10 +269,10 @@ .align 5 _mul_by_3_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -289,15 +289,15 @@ ldp x21,x22,[x1,#32] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -308,10 +308,10 @@ .align 5 _mul_by_8_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -324,15 +324,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -343,10 +343,10 @@ .align 5 _mul_by_3_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -378,15 +378,15 @@ ldp x21,x22,[x1,#80] bl __add_mod_384_ab_are_loaded - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -397,10 +397,10 @@ .align 5 _mul_by_8_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x12,x13,[x1,#16] @@ -424,15 +424,15 @@ bl __lshift_mod_384 bl __lshift_mod_384 bl __lshift_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -443,10 +443,10 @@ .align 5 _cneg_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldp x4,x5,[x3] @@ -481,9 +481,9 @@ csel x15,x15,x22,eq stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -494,25 +494,25 @@ .align 5 _sub_mod_384: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] ldp x8,x9,[x3,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0] stp x12,x13,[x0,#16] stp x14,x15,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -557,10 +557,10 @@ .align 5 _sub_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x3] ldp x6,x7,[x3,#16] @@ -575,15 +575,15 @@ stp x14,x15,[x0,#32] bl __sub_mod_384 - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -594,10 +594,10 @@ .align 5 _mul_by_1_plus_i_mod_384x: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x4,x5,[x2] ldp x6,x7,[x2,#16] @@ -617,15 +617,15 @@ ldp x14,x15,[x1,#80] bl __add_mod_384_ab_are_loaded // a->re + a->im - ldr x30,[sp,#8] + ldr x30,[sp,#__SIZEOF_POINTER__] stp x10,x11,[x0,#48] stp x12,x13,[x0,#64] stp x14,x15,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -963,7 +963,7 @@ Loop_is_zero_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq @@ -992,7 +992,7 @@ Loop_is_equal_done: dup v1.2d, v0.d[1] orr v0.16b, v0.16b, v1.16b - mov x1, v0.d[0] + umov x1, v0.d[0] mov x0, #1 cmp x1, #0 csel x0, x0, xzr, eq diff --git a/c-source/build/mach-o/add_mod_384-x86_64.s b/c-source/build/mach-o/add_mod_384-x86_64.s --- a/c-source/build/mach-o/add_mod_384-x86_64.s +++ b/c-source/build/mach-o/add_mod_384-x86_64.s @@ -48,7 +48,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -58,6 +66,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -101,7 +112,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -163,7 +182,15 @@ leaq 24+48(%rsp),%rsp .cfi_adjust_cfa_offset -24-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -199,6 +226,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -233,7 +263,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -290,7 +328,15 @@ orq %rbp,%r12 orq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r14 + lfence + jmpq *%r14 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -325,6 +371,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq %rdx,%rcx movq 8(%rsi),%r9 @@ -357,7 +406,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -393,6 +450,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -456,7 +516,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -495,7 +563,15 @@ cmovcq %rbp,%r12 cmovcq %rsi,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -531,6 +607,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -542,6 +621,9 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -559,7 +641,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -594,6 +684,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -628,7 +721,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -664,6 +765,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -675,11 +779,17 @@ call __lshift_mod_384 movq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq (%rsp),%rsi leaq 48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%r8 movq 56(%rsi),%r9 movq 64(%rsi),%r10 @@ -691,6 +801,9 @@ movq $48,%rdx addq (%rsp),%rdx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __add_mod_384_a_is_loaded movq 8(%rsp),%r15 @@ -708,7 +821,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -743,6 +864,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -763,6 +887,9 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -796,7 +923,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -832,6 +967,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -898,7 +1036,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -951,7 +1097,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -961,6 +1115,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1002,7 +1159,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1064,7 +1229,15 @@ leaq 24+48(%rsp),%rsp .cfi_adjust_cfa_offset -24-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mul_by_1_plus_i_mod_384x @@ -1098,6 +1271,9 @@ .cfi_adjust_cfa_offset 56 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1210,7 +1386,15 @@ leaq 56+48(%rsp),%rsp .cfi_adjust_cfa_offset -56-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sgn0_pty_mod_384 @@ -1223,6 +1407,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -1254,7 +1441,15 @@ orq %rdi,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1277,6 +1472,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rdi),%r8 movq 56(%rdi),%r9 movq 64(%rdi),%r10 @@ -1369,7 +1567,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_32 @@ -1383,6 +1589,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 16(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1400,7 +1609,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,16-16(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_48 @@ -1414,6 +1631,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 24(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1437,7 +1657,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,32-24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_96 @@ -1451,6 +1679,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 48(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1492,7 +1723,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,80-48(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_192 @@ -1506,6 +1745,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 96(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1583,7 +1825,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,176-96(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_144 @@ -1597,6 +1847,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 72(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1656,7 +1909,15 @@ pand %xmm5,%xmm1 por %xmm1,%xmm0 movdqu %xmm0,128-72(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_select_288 @@ -1670,6 +1931,9 @@ movd %ecx,%xmm5 pxor %xmm4,%xmm4 pshufd $0,%xmm5,%xmm5 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rsi),%xmm0 leaq 144(%rsi),%rsi pcmpeqd %xmm4,%xmm5 @@ -1783,7 +2047,15 @@ pand %xmm5,%xmm3 por %xmm3,%xmm2 movdqu %xmm2,272-144(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_prefetch @@ -1797,6 +2069,9 @@ leaq -1(%rdi,%rsi,1),%rsi movq $64,%rax xorq %r8,%r8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif prefetchnta (%rdi) leaq (%rdi,%rax,1),%rdi cmpq %rsi,%rdi @@ -1827,7 +2102,15 @@ cmpq %rsi,%rdi cmovaq %rsi,%rdi prefetchnta (%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_is_zero_16x @@ -1839,6 +2122,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%esi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdi),%xmm0 leaq 16(%rdi),%rdi @@ -1858,7 +2144,15 @@ testq %rax,%rax cmovnzl %esi,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _vec_is_equal_16x @@ -1870,6 +2164,9 @@ .byte 0xf3,0x0f,0x1e,0xfa shrl $4,%edx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movdqu (%rdi),%xmm0 movdqu (%rsi),%xmm1 subq %rdi,%rsi @@ -1894,6 +2191,14 @@ testq %rax,%rax cmovnzl %edx,%eax xorl $1,%eax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/add_mod_384x384-x86_64.s b/c-source/build/mach-o/add_mod_384x384-x86_64.s --- a/c-source/build/mach-o/add_mod_384x384-x86_64.s +++ b/c-source/build/mach-o/add_mod_384x384-x86_64.s @@ -1,11 +1,39 @@ .text +.globl _add_mod_384x384 +.private_extern _add_mod_384x384 .p2align 5 -__add_mod_384x384: +_add_mod_384x384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -66,16 +94,67 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 -.cfi_endproc +#endif +.cfi_endproc +.globl _sub_mod_384x384 +.private_extern _sub_mod_384x384 .p2align 5 -__sub_mod_384x384: +_sub_mod_384x384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -135,43 +214,6 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) - .byte 0xf3,0xc3 -.cfi_endproc - - -.globl _add_mod_384x384 -.private_extern _add_mod_384x384 - -.p2align 5 -_add_mod_384x384: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 - - - call __add_mod_384x384 - movq 8(%rsp),%r15 .cfi_restore %r15 movq 16(%rsp),%r14 @@ -187,58 +229,14 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 - .byte 0xf3,0xc3 -.cfi_endproc - - -.globl _sub_mod_384x384 -.private_extern _sub_mod_384x384 - -.p2align 5 -_sub_mod_384x384: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 - - - call __sub_mod_384x384 - - movq 8(%rsp),%r15 -.cfi_restore %r15 - movq 16(%rsp),%r14 -.cfi_restore %r14 - movq 24(%rsp),%r13 -.cfi_restore %r13 - movq 32(%rsp),%r12 -.cfi_restore %r12 - movq 40(%rsp),%rbx -.cfi_restore %rbx - movq 48(%rsp),%rbp -.cfi_restore %rbp - leaq 56(%rsp),%rsp -.cfi_adjust_cfa_offset -56 - + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/ct_inverse_mod_256-armv8.S b/c-source/build/mach-o/ct_inverse_mod_256-armv8.S --- a/c-source/build/mach-o/ct_inverse_mod_256-armv8.S +++ b/c-source/build/mach-o/ct_inverse_mod_256-armv8.S @@ -6,20 +6,25 @@ .align 5 _ct_inverse_mod_256: .long 3573752639 - stp x29, x30, [sp,#-80]! + stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] sub sp, sp, #1040 ldp x4, x5, [x1,#8*0] ldp x6, x7, [x1,#8*2] +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#16+511 + alignd c1,c1,#9 +#else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - str x0, [sp] +#endif + str x0, [sp] // offload out_ptr ldp x8, x9, [x2,#8*0] ldp x10, x11, [x2,#8*2] @@ -33,27 +38,36 @@ bl Lab_approximation_31_256_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to dst |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 str x12, [x0,#8*9] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| @@ -72,318 +86,399 @@ stp x5, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 adc x22, x22, x23 stp x22, x22, [x0,#8*4] stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| mov x17, x13 // corrected |g0| mov x12, x14 // |f1| mov x13, x15 // |g1| - add x0, x0, #8*4 // pointer to destination |b| + add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - add x0, x0, #8*4 // pointer to destination |u| + add x0,x0,#8*4 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0, x0, #8*5 // pointer to destination |v| + add x0,x0,#8*5 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, ldr x7, [x1,#8*0] // just load @@ -395,7 +490,7 @@ ldr x0, [sp] // original out_ptr bl __smul_256x63 bl __smul_512x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] smulh x20, x7, x17 // figure out top-most limb ldp x8, x9, [x3,#8*0] @@ -441,11 +536,11 @@ stp x6, x7, [x0,#8*6] add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldr x29, [sp],#80 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldr x29, [sp],#10*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/mach-o/ct_inverse_mod_256-x86_64.s b/c-source/build/mach-o/ct_inverse_mod_256-x86_64.s --- a/c-source/build/mach-o/ct_inverse_mod_256-x86_64.s +++ b/c-source/build/mach-o/ct_inverse_mod_256-x86_64.s @@ -36,6 +36,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -567,6 +570,9 @@ movq %rdx,%r8 movq %rdx,%r9 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 movq %rdx,%r10 andq 8(%rsi),%r9 @@ -629,7 +635,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1072-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -779,7 +793,15 @@ movq %r14,48(%rdi) movq %r15,56(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -890,7 +912,15 @@ movq %r11,24(%rdi) movq %rbp,32(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1021,7 +1051,15 @@ addq %rax,%rdx addq %rax,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1077,7 +1115,15 @@ jmp __inner_loop_31_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1127,7 +1173,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1173,6 +1227,14 @@ subl $1,%r15d jnz L$oop_62_256 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/ct_inverse_mod_384-armv8.S b/c-source/build/mach-o/ct_inverse_mod_384-armv8.S --- a/c-source/build/mach-o/ct_inverse_mod_384-armv8.S +++ b/c-source/build/mach-o/ct_inverse_mod_384-armv8.S @@ -6,22 +6,27 @@ .align 5 _ct_inverse_mod_383: .long 3573752639 - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] - sub sp, sp, #1040 + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] + sub sp, sp, #1056 ldp x22, x4, [x1,#8*0] ldp x5, x6, [x1,#8*2] ldp x7, x8, [x1,#8*4] - add x1, sp, #16+511 // find closest 512-byte-aligned spot +#ifdef __CHERI_PURE_CAPABILITY__ + add x1,sp,#32+511 + alignd c1,c1,#9 +#else + add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... - stp x0, x3, [sp] +#endif + stp x0, x3, [sp] // offload out_ptr, nx_ptr ldp x9, x10, [x2,#8*0] ldp x11, x12, [x2,#8*2] @@ -39,28 +44,37 @@ bl Lab_approximation_62_loaded eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to dst |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 str x15, [x0,#8*12] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 ldr x7, [x1,#8*12] // |u| @@ -87,205 +101,262 @@ stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif bl __smul_383_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| - add x0, x0, #8*6 // pointer to destination |b| + add x0,x0,#8*6 bl __smul_383_n_shift_by_62 - add x0, x0, #8*6 // pointer to destination |u| + add x0,x0,#8*6 bl __smul_383x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0, x0, #8*6 // pointer to destination |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, ldp x3, x8, [x1,#8*0] // just load @@ -293,6 +364,9 @@ bl __inner_loop_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -300,17 +374,20 @@ mov x21, x16 // exact |g0| mov x15, x17 mov x16, x19 - add x0, x0, #8*12 // pointer to dst |u| + add x0,x0,#8*12 bl __smul_383x63 mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0, x0, #8*6 // pointer to dst |v| + add x0,x0,#8*6 bl __smul_383x63 bl __smul_767x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif mov x2, #22 // 766 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load @@ -321,10 +398,10 @@ mov x20, x17 mov x21, x19 - ldp x0, x15, [sp] // original out_ptr and n_ptr + ldp x0, x15, [sp] // original out_ptr and n_ptr bl __smul_383x63 bl __smul_767x63_tail - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] asr x22, x8, #63 // sign as mask ldp x9, x10, [x15,#8*0] @@ -347,13 +424,13 @@ adc x8, x8, x14 stp x7, x8, [x0,#8*10] - add sp, sp, #1040 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + add sp, sp, #1056 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/mach-o/ct_is_square_mod_384-armv8.S b/c-source/build/mach-o/ct_is_square_mod_384-armv8.S --- a/c-source/build/mach-o/ct_is_square_mod_384-armv8.S +++ b/c-source/build/mach-o/ct_is_square_mod_384-armv8.S @@ -6,13 +6,13 @@ .align 5 _ct_is_square_mod_384: .long 3573752639 - stp x29, x30, [sp,#-128]! + stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] + stp x19, x20, [sp,#2*__SIZEOF_POINTER__] + stp x21, x22, [sp,#4*__SIZEOF_POINTER__] + stp x23, x24, [sp,#6*__SIZEOF_POINTER__] + stp x25, x26, [sp,#8*__SIZEOF_POINTER__] + stp x27, x28, [sp,#10*__SIZEOF_POINTER__] sub sp, sp, #512 ldp x3, x4, [x0,#8*0] // load input @@ -21,6 +21,9 @@ add x0, sp, #255 // find closest 256-byte-aligned spot and x0, x0, #-256 // in the frame... +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif ldp x9, x10, [x1,#8*0] // load modulus ldp x11, x12, [x1,#8*2] @@ -43,15 +46,21 @@ sub x15, x15, #1 eor x1, x0, #128 // pointer to dst |b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c1,csp,x1 +#endif bl __smul_384_n_shift_by_30 mov x19, x16 // |f0| mov x20, x17 // |g0| - add x1, x1, #8*6 // pointer to dst |a| + add x1,x1,#8*6 bl __smul_384_n_shift_by_30 ldp x9, x10, [x1,#-8*6] eor x0, x0, #128 // flip-flop src |a|b| +#ifdef __CHERI_PURE_CAPABILITY__ + scvalue c0,csp,x0 +#endif and x27, x27, x9 // if |a| was negative, add x2, x2, x27, lsr#1 // adjust |L| @@ -63,18 +72,18 @@ //ldr x14, [x0,#8*0] mov x15, #48 // 48 is 768%30 + 30 bl __inner_loop_48 - ldr x30, [x29,#8] + ldr x30, [x29,#__SIZEOF_POINTER__] and x0, x2, #1 eor x0, x0, #1 add sp, sp, #512 - ldp x19, x20, [x29,#16] - ldp x21, x22, [x29,#32] - ldp x23, x24, [x29,#48] - ldp x25, x26, [x29,#64] - ldp x27, x28, [x29,#80] - ldr x29, [sp],#128 + ldp x19, x20, [x29,#2*__SIZEOF_POINTER__] + ldp x21, x22, [x29,#4*__SIZEOF_POINTER__] + ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] + ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] + ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] + ldr x29, [sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/mach-o/ct_is_square_mod_384-x86_64.s b/c-source/build/mach-o/ct_is_square_mod_384-x86_64.s --- a/c-source/build/mach-o/ct_is_square_mod_384-x86_64.s +++ b/c-source/build/mach-o/ct_is_square_mod_384-x86_64.s @@ -34,6 +34,9 @@ leaq 24+255(%rsp),%rax andq $-256,%rax +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdi),%r8 movq 8(%rdi),%r9 movq 16(%rdi),%r10 @@ -120,7 +123,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -536-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -296,7 +307,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -363,7 +382,15 @@ jmp __inner_loop_30 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -426,7 +453,15 @@ subq %r15,%rdx subq %r15,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -467,6 +502,14 @@ subl $1,%edi jnz L$oop_48 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s b/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s @@ -12,7 +12,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz L$ct_inverse_mod_383$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -539,7 +539,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1112-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -752,7 +760,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -896,7 +912,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1075,7 +1099,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1133,7 +1165,15 @@ jmp __inner_loop_62 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1188,6 +1228,14 @@ jnz L$oop_62 movq 8(%rsp),%rsi + +#ifdef __SGX_LVI_HARDENING__ + popq %rax + lfence + jmpq *%rax + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s b/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s @@ -9,7 +9,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -ct_inverse_mod_383$1: +L$ct_inverse_mod_383$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -37,6 +37,9 @@ movq %rdi,32(%rsp) movq %rcx,40(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -838,6 +841,9 @@ movq %rax,%r8 movq %rax,%r9 movq %rax,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 movq %rax,%r11 @@ -877,7 +883,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -1112-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1044,7 +1058,15 @@ movq %rcx,80(%rdx) movq %rax,88(%rdx) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1154,7 +1176,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1302,7 +1332,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1396,7 +1434,15 @@ addq %rbp,%rdx addq %rbp,%rcx + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1469,7 +1515,15 @@ jmp __inner_loop_31 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1519,7 +1573,15 @@ subq %r15,%r12 subq %r15,%r13 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1563,6 +1625,14 @@ subl $1,%edi jnz L$oop_53 + +#ifdef __SGX_LVI_HARDENING__ + popq %r8 + lfence + jmpq *%r8 + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/div3w-armv8.S b/c-source/build/mach-o/div3w-armv8.S --- a/c-source/build/mach-o/div3w-armv8.S +++ b/c-source/build/mach-o/div3w-armv8.S @@ -1,6 +1,7 @@ .text .globl _div_3_limbs +.private_extern _div_3_limbs .align 5 _div_3_limbs: @@ -34,6 +35,7 @@ ret .globl _quot_rem_128 +.private_extern _quot_rem_128 .align 5 _quot_rem_128: @@ -70,6 +72,7 @@ .globl _quot_rem_64 +.private_extern _quot_rem_64 .align 5 _quot_rem_64: diff --git a/c-source/build/mach-o/div3w-x86_64.s b/c-source/build/mach-o/div3w-x86_64.s --- a/c-source/build/mach-o/div3w-x86_64.s +++ b/c-source/build/mach-o/div3w-x86_64.s @@ -10,6 +10,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq (%rdi),%r8 movq 8(%rdi),%r9 xorq %rax,%rax @@ -42,7 +45,15 @@ orq %rcx,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _quot_rem_128 @@ -55,6 +66,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax movq %rdx,%rcx @@ -90,7 +104,15 @@ movq %rcx,%rax + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -108,6 +130,9 @@ +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq %rdx,%rax imulq 0(%rsi),%rdx @@ -119,6 +144,14 @@ movq %rax,8(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/mul_mont_256-armv8.S b/c-source/build/mach-o/mul_mont_256-armv8.S --- a/c-source/build/mach-o/mul_mont_256-armv8.S +++ b/c-source/build/mach-o/mul_mont_256-armv8.S @@ -5,11 +5,11 @@ .align 5 _mul_mont_sparse_256: - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x10,x11,[x1] ldr x9, [x2] @@ -185,10 +185,10 @@ stp x19,x20,[x0] stp x21,x22,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ ret .globl _sqr_mont_sparse_256 @@ -197,10 +197,10 @@ .align 5 _sqr_mont_sparse_256: .long 3573752639 - stp x29,x30,[sp,#-48]! + stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] ldp x5,x6,[x1] ldp x7,x8,[x1,#16] @@ -272,7 +272,7 @@ adc x22,x22,x8 bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] adds x10,x10,x19 // accumulate upper half adcs x11,x11,x20 @@ -294,9 +294,9 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldr x29,[sp],#48 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldr x29,[sp],#6*__SIZEOF_POINTER__ .long 3573752767 ret @@ -306,7 +306,7 @@ .align 5 _from_mont_256: .long 3573752639 - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -314,7 +314,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x14,x10,x5 sbcs x15,x11,x6 @@ -329,7 +329,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .long 3573752767 ret @@ -340,7 +340,7 @@ .align 5 _redc_mont_256: .long 3573752639 - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 mov x4,x3 @@ -348,7 +348,7 @@ ldp x12,x13,[x1,#16] bl __mul_by_1_mont_256 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x14,x15,[x1,#32] ldp x16,x17,[x1,#48] @@ -373,7 +373,7 @@ stp x10,x11,[x0] stp x12,x13,[x0,#16] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/mach-o/mul_mont_384-armv8.S b/c-source/build/mach-o/mul_mont_384-armv8.S --- a/c-source/build/mach-o/mul_mont_384-armv8.S +++ b/c-source/build/mach-o/mul_mont_384-armv8.S @@ -5,23 +5,23 @@ .align 5 _add_mod_384x384: .long 3573752639 - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __add_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .long 3573752767 ret @@ -86,23 +86,23 @@ .align 5 _sub_mod_384x384: .long 3573752639 - stp x29,x30,[sp,#-64]! + stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] ldp x5,x6,[x3] ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldr x29,[sp],#64 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldr x29,[sp],#8*__SIZEOF_POINTER__ .long 3573752767 ret @@ -240,23 +240,23 @@ .align 5 _mul_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#288 // space for 3 768-bit vectors mov x26,x0 // save r_ptr mov x27,x1 // save b_ptr mov x28,x2 // save b_ptr - sub x0,sp,#0 // mul_384(t0, a->re, b->re) + add x0,sp,#0 bl __mul_384 - add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x1,x1,#48 add x2,x2,#48 add x0,sp,#96 bl __mul_384 @@ -271,7 +271,7 @@ add x1,x28,#0 add x2,x28,#48 - add x0,sp,#192 // t2 + add x0,sp,#192 bl __add_mod_384 add x1,x0,#0 @@ -294,24 +294,24 @@ add x0,sp,#0 bl __sub_mod_384x384 // t0 = t0-t1 - add x1,sp,#0 // ret->re = redc(t0) + add x1,sp,#0 add x0,x26,#0 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - add x1,sp,#192 // ret->im = redc(t2) + add x1,sp,#192 add x0,x0,#48 bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#288 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -322,14 +322,14 @@ .align 5 _sqr_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 2 384-bit vectors mov x4,x3 // adjust for missing b_ptr @@ -384,19 +384,19 @@ add x2,sp,#48 bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -407,14 +407,14 @@ .align 5 _mul_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there ldp x11,x12,[x1] ldr x17, [x2] @@ -426,18 +426,18 @@ ldp x9,x10,[x3,#32] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -502,7 +502,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -562,7 +562,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -622,7 +622,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -682,7 +682,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -742,7 +742,7 @@ adcs x24,x25,x3 mul x3,x16,x17 adc x25,x4,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x19,x26 umulh x26,x11,x17 @@ -786,7 +786,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adcs x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adc x17,x17,xzr adds x19,x20,x26 @@ -820,13 +820,13 @@ .align 5 _sqr_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for 768-bit vector mov x4,x3 // adjust for missing b_ptr @@ -847,15 +847,15 @@ mov x0,x3 // restore r_ptr bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -866,14 +866,14 @@ .align 5 _sqr_n_mul_mont_383: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x4,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#96 // space for 768-bit vector mov x17,x5 // save b_ptr @@ -908,19 +908,19 @@ mov x2,x17 ldr x17,[x17] bl __mul_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] stp x11,x12,[x2] stp x13,x14,[x2,#16] stp x15,x16,[x2,#32] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1043,27 +1043,27 @@ .align 5 _sqr_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x13,x14,[x1,#16] ldp x15,x16,[x1,#32] bl __sqr_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1074,13 +1074,13 @@ .align 5 _redc_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1089,14 +1089,14 @@ bl __mul_by_1_mont_384 bl __redc_tail_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1107,13 +1107,13 @@ .align 5 _from_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x3 // adjust for missing b_ptr ldp x5,x6,[x2] @@ -1121,7 +1121,7 @@ ldp x9,x10,[x2,#32] bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] subs x19,x11,x5 sbcs x20,x12,x6 @@ -1141,12 +1141,12 @@ stp x13,x14,[x0,#16] stp x15,x16,[x0,#32] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1366,23 +1366,23 @@ .align 5 _mul_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1570,13 +1570,13 @@ .align 5 _mul_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] sub sp,sp,#96 // space for two 384-bit vectors ldp x11,x12,[x1] @@ -1616,12 +1616,12 @@ bl __mul_384 // _mul_384(ret->re, a->re, b->re) - add x1,sp,#0 // _mul_384(ret->im, t0, t1) + add x1,sp,#0 add x2,sp,#48 add x0,x26,#96 bl __mul_384 - add x1,x27,#48 // _mul_384(tx, a->im, b->im) + add x1,x27,#48 add x2,x28,#48 add x0,sp,#0 bl __mul_384 @@ -1630,27 +1630,27 @@ ldp x7,x8,[x3,#16] ldp x9,x10,[x3,#32] - add x1,x26,#96 // ret->im -= tx + add x1,x26,#96 add x2,sp,#0 add x0,x26,#96 bl __sub_mod_384x384 - add x2,x26,#0 // ret->im -= ret->re + add x2,x26,#0 bl __sub_mod_384x384 - add x1,x26,#0 // ret->re -= tx + add x1,x26,#0 add x2,sp,#0 add x0,x26,#0 bl __sub_mod_384x384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] add sp,sp,#96 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1661,13 +1661,13 @@ .align 5 _sqr_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] ldp x11,x12,[x1] ldp x19,x20,[x1,#48] @@ -1713,15 +1713,15 @@ stp x15,x16,[x0,#80] mov x4,x1 // save a_ptr - add x1,x0,#0 // _mul_384(ret->re, t0, t1) + add x1,x0,#0 add x2,x0,#48 bl __mul_384 - add x1,x4,#0 // _mul_384(ret->im, a->re, a->im) + add x1,x4,#0 add x2,x4,#48 add x0,x0,#96 bl __mul_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldp x11,x12,[x0] ldp x13,x14,[x0,#16] @@ -1745,12 +1745,12 @@ stp x21,x22,[x0,#64] stp x23,x24,[x0,#80] - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1761,14 +1761,14 @@ .align 5 _sqr_mont_382x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] + stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there sub sp,sp,#112 // space for two 384-bit vectors + word mov x4,x3 // adjust for missing b_ptr @@ -1828,7 +1828,7 @@ add x2,sp,#48 bl __mul_mont_383_nonred // _mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] ldr x25,[sp,#96] // account for sign from a->re - a->im ldp x19,x20,[sp] @@ -1869,12 +1869,12 @@ stp x15,x16,[x2,#32] add sp,sp,#112 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -1924,7 +1924,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -1981,7 +1981,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2038,7 +2038,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2095,7 +2095,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2152,7 +2152,7 @@ umulh x3,x10,x4 adc x25,x25,xzr - ldr x4,[x29,#96] + ldr x4,[x29,#12*__SIZEOF_POINTER__] adds x19,x20,x26 mul x26,x11,x17 adcs x20,x21,x27 @@ -2207,7 +2207,7 @@ adcs x24,x24,x3 umulh x3,x10,x4 adc x25,x25,xzr - ldp x4,x2,[x29,#96] // pull r_ptr + ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr adds x11,x20,x26 adcs x12,x21,x27 @@ -2225,13 +2225,13 @@ .align 5 _sgn0_pty_mont_384: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2240,7 +2240,7 @@ mov x1,x0 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 adds x11,x11,x11 @@ -2263,12 +2263,12 @@ and x17,x17,#2 orr x0,x0,x17 - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret @@ -2279,13 +2279,13 @@ .align 5 _sgn0_pty_mont_384x: .long 3573752639 - stp x29,x30,[sp,#-128]! + stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] + stp x19,x20,[sp,#2*__SIZEOF_POINTER__] + stp x21,x22,[sp,#4*__SIZEOF_POINTER__] + stp x23,x24,[sp,#6*__SIZEOF_POINTER__] + stp x25,x26,[sp,#8*__SIZEOF_POINTER__] + stp x27,x28,[sp,#10*__SIZEOF_POINTER__] mov x4,x2 ldp x5,x6,[x1] @@ -2323,7 +2323,7 @@ orr x2,x2,x17 bl __mul_by_1_mont_384 - ldr x30,[x29,#8] + ldr x30,[x29,#__SIZEOF_POINTER__] and x0,x11,#1 orr x1,x11,x12 @@ -2361,12 +2361,12 @@ and x1,x1,#2 orr x0,x1,x3 // pack sign and parity - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldr x29,[sp],#128 + ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] + ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] + ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] + ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] + ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] + ldr x29,[sp],#16*__SIZEOF_POINTER__ .long 3573752767 ret diff --git a/c-source/build/mach-o/mulq_mont_256-x86_64.s b/c-source/build/mach-o/mulq_mont_256-x86_64.s --- a/c-source/build/mach-o/mulq_mont_256-x86_64.s +++ b/c-source/build/mach-o/mulq_mont_256-x86_64.s @@ -12,7 +12,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz mul_mont_sparse_256$1 + jnz L$mul_mont_sparse_256$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -65,7 +65,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -80,7 +88,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_mont_sparse_256$1 + jnz L$sqr_mont_sparse_256$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -134,7 +142,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -416,7 +432,15 @@ movq %r15,16(%rsi) movq %r9,24(%rsi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _from_mont_256 @@ -430,7 +454,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz from_mont_256$1 + jnz L$from_mont_256$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -494,7 +518,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -509,7 +541,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz redc_mont_256$1 + jnz L$redc_mont_256$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -579,7 +611,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -718,6 +758,14 @@ addq %r9,%r15 adcq $0,%rdx movq %rdx,%r9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/mulq_mont_384-x86_64.s b/c-source/build/mach-o/mulq_mont_384-x86_64.s --- a/c-source/build/mach-o/mulq_mont_384-x86_64.s +++ b/c-source/build/mach-o/mulq_mont_384-x86_64.s @@ -72,7 +72,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -124,7 +132,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -176,7 +192,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mul_mont_384x @@ -190,7 +214,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz mul_mont_384x$1 + jnz L$mul_mont_384x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -296,7 +320,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -328-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqr_mont_384x @@ -310,7 +342,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_mont_384x$1 + jnz L$sqr_mont_384x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -423,7 +455,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -438,7 +478,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz mul_382x$1 + jnz L$mul_382x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -563,7 +603,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqr_382x @@ -577,7 +625,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_382x$1 + jnz L$sqr_382x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -700,7 +748,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -8*7 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mul_384 @@ -714,7 +770,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz mul_384$1 + jnz L$mul_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -739,7 +795,15 @@ leaq 24(%rsp),%rsp .cfi_adjust_cfa_offset -24 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1028,7 +1092,15 @@ movq %r11,80(%rdi) movq %r12,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqr_384 @@ -1042,7 +1114,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_384$1 + jnz L$sqr_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -1083,7 +1155,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1277,7 +1357,15 @@ movq %rax,80(%rdi) movq %rdx,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1292,7 +1380,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_mont_384$1 + jnz L$sqr_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -1346,7 +1434,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*21 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1363,7 +1459,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz redc_mont_384$1 + jnz L$redc_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -1406,7 +1502,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1424,7 +1528,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz from_mont_384$1 + jnz L$from_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -1496,7 +1600,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1795,7 +1907,15 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1844,7 +1964,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1859,7 +1987,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sgn0_pty_mont_384$1 + jnz L$sgn0_pty_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -1926,7 +2054,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1941,7 +2077,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sgn0_pty_mont_384x$1 + jnz L$sgn0_pty_mont_384x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -2058,7 +2194,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mul_mont_384 @@ -2072,7 +2216,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz mul_mont_384$1 + jnz L$mul_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -2122,7 +2266,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -72 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2726,7 +2878,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqr_n_mul_mont_384 @@ -2740,7 +2900,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_n_mul_mont_384$1 + jnz L$sqr_n_mul_mont_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -2817,7 +2977,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*23 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2832,7 +3000,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_n_mul_mont_383$1 + jnz L$sqr_n_mul_mont_383$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -2922,7 +3090,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -8*23 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -3483,7 +3659,15 @@ adcq $0,%rdx addq %r12,%r10 adcq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqr_mont_382x @@ -3497,7 +3681,7 @@ #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz sqr_mont_382x$1 + jnz L$sqr_mont_382x$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -3668,6 +3852,14 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/mulx_mont_256-x86_64.s b/c-source/build/mach-o/mulx_mont_256-x86_64.s --- a/c-source/build/mach-o/mulx_mont_256-x86_64.s +++ b/c-source/build/mach-o/mulx_mont_256-x86_64.s @@ -9,7 +9,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -mul_mont_sparse_256$1: +L$mul_mont_sparse_256$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -33,6 +33,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -59,7 +62,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -72,7 +83,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_mont_sparse_256$1: +L$sqr_mont_sparse_256$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -98,6 +109,9 @@ movq %rsi,%rbx movq %rcx,%r8 movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rbp @@ -123,7 +137,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -322,7 +344,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _fromx_mont_256 @@ -334,7 +364,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -from_mont_256$1: +L$from_mont_256$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -397,7 +427,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -410,7 +448,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -redc_mont_256$1: +L$redc_mont_256$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -479,7 +517,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -488,6 +534,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rax movq 8(%rsi),%r11 movq 16(%rsi),%r12 @@ -618,6 +667,14 @@ addq %r11,%r10 adcq $0,%rdx movq %rdx,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/mulx_mont_384-x86_64.s b/c-source/build/mach-o/mulx_mont_384-x86_64.s --- a/c-source/build/mach-o/mulx_mont_384-x86_64.s +++ b/c-source/build/mach-o/mulx_mont_384-x86_64.s @@ -71,7 +71,15 @@ movq %rbp,80(%rdi) movq %rsi,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -81,6 +89,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -123,7 +134,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -133,6 +152,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -175,7 +197,15 @@ movq %r12,32(%rdi) movq %r13,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mulx_mont_384x @@ -187,7 +217,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -mul_mont_384x$1: +L$mul_mont_384x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -221,6 +251,9 @@ leaq 40(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -234,11 +267,17 @@ leaq (%rbx),%rsi leaq -48(%rbx),%rdx leaq 40+192+48(%rsp),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 movq 24(%rsp),%rsi leaq 48(%rsi),%rdx leaq -48(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __addx_mod_384 leaq (%rdi),%rbx @@ -249,6 +288,9 @@ leaq (%rdi),%rsi leaq 40(%rsp),%rdx movq 8(%rsp),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 leaq (%rdi),%rsi @@ -293,7 +335,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -328-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqrx_mont_384x @@ -305,7 +355,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_mont_384x$1: +L$sqr_mont_384x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -349,6 +399,9 @@ movq 24(%rsp),%rsi leaq 48(%rsi),%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 48(%rsi),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -428,7 +481,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -441,7 +502,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -mul_382x$1: +L$mul_382x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -471,6 +532,9 @@ movq %rcx,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -523,6 +587,9 @@ movq 0(%rsp),%rsi movq 8(%rsp),%rbx leaq -96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 @@ -536,6 +603,9 @@ leaq 32(%rsp),%rdx movq 24(%rsp),%rcx movq %rsi,%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __subx_mod_384x384 @@ -565,7 +635,15 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqrx_382x @@ -577,7 +655,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_382x$1: +L$sqr_382x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -603,6 +681,9 @@ movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r14 movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -645,6 +726,9 @@ movq (%rsp),%rsi leaq 48(%rsi),%rbx leaq 96(%rdi),%rdi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rdi),%r8 @@ -699,7 +783,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -8*7 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mulx_384 @@ -711,7 +803,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -mul_384$1: +L$mul_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -733,6 +825,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_384 movq 0(%rsp),%r15 @@ -750,7 +845,15 @@ leaq 48(%rsp),%rsp .cfi_adjust_cfa_offset -48 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -926,7 +1029,15 @@ movq %r12,80(%rdi) movq %r13,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqrx_384 @@ -938,7 +1049,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_384$1: +L$sqr_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -961,6 +1072,9 @@ .cfi_adjust_cfa_offset 8 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __sqrx_384 movq 8(%rsp),%r15 @@ -978,7 +1092,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1118,7 +1240,15 @@ movq %rbx,80(%rdi) movq %rbp,88(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1133,7 +1263,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -redc_mont_384$1: +L$redc_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -1157,6 +1287,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 call __redx_tail_mont_384 @@ -1175,7 +1308,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1191,7 +1332,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -from_mont_384$1: +L$from_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -1215,6 +1356,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 @@ -1262,7 +1406,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1452,7 +1604,15 @@ adcxq %rax,%r10 adoxq %r11,%rbp adcxq %rbp,%r11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1501,7 +1661,15 @@ movq %r10,32(%rdi) movq %r11,40(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1514,7 +1682,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sgn0_pty_mont_384$1: +L$sgn0_pty_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -1540,6 +1708,9 @@ movq %rsi,%rbx leaq 0(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 xorq %rax,%rax @@ -1580,7 +1751,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1593,7 +1772,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sgn0_pty_mont_384x$1: +L$sgn0_pty_mont_384x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -1619,6 +1798,9 @@ movq %rsi,%rbx leaq 48(%rdi),%rsi movq %rdx,%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif call __mulx_by_1_mont_384 movq %r14,%r12 @@ -1709,7 +1891,15 @@ leaq 56(%rsp),%rsp .cfi_adjust_cfa_offset -56 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _mulx_mont_384 @@ -1721,7 +1911,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -mul_mont_384$1: +L$mul_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -1745,6 +1935,9 @@ movq %rdx,%rbx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rdx),%rdx movq 0(%rsi),%r14 movq 8(%rsi),%r15 @@ -1775,7 +1968,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -8*9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2177,7 +2378,15 @@ movq %rdi,32(%rbx) movq %rbp,40(%rbx) + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqrx_mont_384 @@ -2189,7 +2398,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_mont_384$1: +L$sqr_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -2214,6 +2423,9 @@ movq %rcx,%r8 leaq -128(%rdx),%rcx +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2244,7 +2456,15 @@ leaq 72(%rsp),%rsp .cfi_adjust_cfa_offset -8*9 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2257,7 +2477,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_n_mul_mont_384$1: +L$sqr_n_mul_mont_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -2281,6 +2501,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2330,7 +2553,15 @@ leaq 88(%rsp),%rsp .cfi_adjust_cfa_offset -8*11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2343,7 +2574,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_n_mul_mont_383$1: +L$sqr_n_mul_mont_383$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -2367,6 +2598,9 @@ movq %rdx,%r10 +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%rdx movq 8(%rsi),%r15 movq 16(%rsi),%rax @@ -2415,7 +2649,15 @@ leaq 88(%rsp),%rsp .cfi_adjust_cfa_offset -8*11 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -2778,7 +3020,15 @@ movq %r10,40(%rbx) movq %r10,%rbp + +#ifdef __SGX_LVI_HARDENING__ + popq %rsi + lfence + jmpq *%rsi + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _sqrx_mont_382x @@ -2790,7 +3040,7 @@ .byte 0xf3,0x0f,0x1e,0xfa -sqr_mont_382x$1: +L$sqr_mont_382x$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -2819,6 +3069,9 @@ movq %rsi,24(%rsp) +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -2970,6 +3223,14 @@ leaq 48(%r8),%rsp .cfi_adjust_cfa_offset -136-8*6 + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/build/mach-o/sha256-armv8.S b/c-source/build/mach-o/sha256-armv8.S --- a/c-source/build/mach-o/sha256-armv8.S +++ b/c-source/build/mach-o/sha256-armv8.S @@ -47,7 +47,7 @@ .align 6 _blst_sha256_block_armv8: Lv8_entry: - stp x29,x30,[sp,#-16]! + stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 ld1 {v0.4s,v1.4s},[x0] @@ -178,7 +178,7 @@ st1 {v0.4s,v1.4s},[x0] - ldr x29,[sp],#16 + ldr x29,[sp],#2*__SIZEOF_POINTER__ ret .globl _blst_sha256_block_data_order @@ -190,7 +190,7 @@ tst w16,#1 b.ne Lv8_entry - stp x29, x30, [sp, #-16]! + stp x29, x30, [sp, #-2*__SIZEOF_POINTER__]! mov x29, sp sub sp,sp,#16*4 @@ -671,11 +671,11 @@ sub x17,x17,#64 bne L_00_48 - sub x16,x16,#256 // rewind x16 + sub x16,x16,#256 cmp x1,x2 - mov x17, #64 + mov x17, #-64 csel x17, x17, xzr, eq - sub x1,x1,x17 // avoid SEGV + add x1,x1,x17 mov x17,sp add w10,w10,w12 add w3,w3,w15 @@ -1026,7 +1026,7 @@ b.ne L_00_48 ldr x29,[x29] - add sp,sp,#16*4+16 + add sp,sp,#16*4+2*__SIZEOF_POINTER__ ret .globl _blst_sha256_emit diff --git a/c-source/build/mach-o/sha256-portable-x86_64.s b/c-source/build/mach-o/sha256-portable-x86_64.s --- a/c-source/build/mach-o/sha256-portable-x86_64.s +++ b/c-source/build/mach-o/sha256-portable-x86_64.s @@ -1651,11 +1651,20 @@ .cfi_restore %rbp .cfi_restore %rbx leaq (%r11),%rsp + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc #ifndef __BLST_PORTABLE__ +.section __TEXT,__const .p2align 6 K256: @@ -1705,7 +1714,15 @@ shrq $32,%r11 movl %r10d,16(%rdi) movl %r11d,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1724,7 +1741,15 @@ movb %al,-1(%rdi,%rsi,1) decq %rdx jnz L$oop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1744,7 +1769,15 @@ movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc #endif diff --git a/c-source/build/mach-o/sha256-x86_64.s b/c-source/build/mach-o/sha256-x86_64.s --- a/c-source/build/mach-o/sha256-x86_64.s +++ b/c-source/build/mach-o/sha256-x86_64.s @@ -1,6 +1,6 @@ .comm ___blst_platform_cap,4 -.text +.section __TEXT,__const .p2align 6 K256: @@ -25,6 +25,7 @@ .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.text .globl _blst_sha256_block_data_order_shaext .private_extern _blst_sha256_block_data_order_shaext @@ -41,6 +42,9 @@ .cfi_def_cfa_register %rbp L$blst_sha256_block_data_order$2: +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif leaq K256+128(%rip),%rcx movdqu (%rdi),%xmm1 movdqu 16(%rdi),%xmm2 @@ -247,7 +251,15 @@ .cfi_adjust_cfa_offset -8 .cfi_restore %rbp + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _blst_sha256_block_data_order @@ -264,8 +276,10 @@ .cfi_offset %rbp,-16 movq %rsp,%rbp .cfi_def_cfa_register %rbp +#ifndef __SGX_LVI_HARDENING__ testl $2,___blst_platform_cap(%rip) jnz L$blst_sha256_block_data_order$2 +#endif pushq %rbx .cfi_offset %rbx,-24 pushq %r12 @@ -286,6 +300,9 @@ leaq -64(%rsp),%rsp +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movl 0(%rdi),%eax andq $-64,%rsp movl 4(%rdi),%ebx @@ -1330,6 +1347,9 @@ movl %r14d,%eax movq -56(%rbp),%rsi +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif addl 0(%rdi),%eax addl 4(%rdi),%ebx addl 8(%rdi),%ecx @@ -1372,7 +1392,15 @@ .cfi_restore %r14 .cfi_restore %r15 .cfi_restore %rbx + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc .globl _blst_sha256_emit @@ -1383,6 +1411,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1403,7 +1434,15 @@ shrq $32,%r11 movl %r10d,16(%rdi) movl %r11d,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1415,6 +1454,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif subq %rsi,%rdi L$oop_bcopy: movzbl (%rsi),%eax @@ -1422,7 +1464,15 @@ movb %al,-1(%rdi,%rsi,1) decq %rdx jnz L$oop_bcopy + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc @@ -1434,6 +1484,9 @@ .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa +#ifdef __SGX_LVI_HARDENING__ + lfence +#endif movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 @@ -1442,6 +1495,14 @@ movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) + +#ifdef __SGX_LVI_HARDENING__ + popq %rdx + lfence + jmpq *%rdx + ud2 +#else .byte 0xf3,0xc3 +#endif .cfi_endproc diff --git a/c-source/src/cpuid.c b/c-source/src/cpuid.c --- a/c-source/src/cpuid.c +++ b/c-source/src/cpuid.c @@ -9,7 +9,7 @@ #endif int __blst_platform_cap = 0; -#if defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) +#if defined(__x86_64__) || defined(__x86_64) || (defined(_M_X64) && !defined(_M_ARM64EC)) # if defined(__GNUC__) || defined(__clang__) || defined(__SUNPRO_C) static void __cpuidex(int info[4], int func, int sub) @@ -54,7 +54,7 @@ # pragma init(__blst_cpuid) # endif -#elif defined(__aarch64__) || defined(__aarch64) +#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) # if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) extern unsigned long getauxval(unsigned long type) __attribute__ ((weak)); @@ -80,6 +80,35 @@ __blst_platform_cap = 1; /* SHA256 */ return 0; } +# elif defined(__FreeBSD__) && __FreeBSD__ >= 12 +# include +__attribute__((constructor)) +static int __blst_cpuid() +{ + unsigned long cap; + + if (elf_aux_info(AT_HWCAP, &cap, sizeof(cap)) == 0) + __blst_platform_cap = (cap & HWCAP_SHA2) != 0; + + return 0; +} +# elif defined(_WIN64) +int IsProcessorFeaturePresent(int); + +# if defined(__GNUC__) || defined(__clang__) +__attribute__((constructor)) +# endif +static int __blst_cpuid(void) +{ + __blst_platform_cap = IsProcessorFeaturePresent(30); /* AES, SHA1, SHA2 */ + + return 0; +} + +# if defined(_MSC_VER) && !defined(__clang__) +# pragma section(".CRT$XCU",read) +__declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; +# endif # endif #endif diff --git a/c-source/src/ec_mult.h b/c-source/src/ec_mult.h --- a/c-source/src/ec_mult.h +++ b/c-source/src/ec_mult.h @@ -66,20 +66,22 @@ * infinity "naturally," since resulting Z is product of original Z.] */ #define POINT_MULT_SCALAR_WX_IMPL(ptype, SZ) \ -static void ptype##_gather_booth_w##SZ(ptype *restrict p, \ - const ptype table[1<<(SZ-1)], \ - limb_t booth_idx) \ +static bool_t ptype##_gather_booth_w##SZ(ptype *restrict p, \ + const ptype table[1<<(SZ-1)], \ + limb_t booth_idx) \ { \ size_t i; \ bool_t booth_sign = (booth_idx >> SZ) & 1; \ \ booth_idx &= (1< 0) { \ @@ -136,8 +139,14 @@ scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ wval = get_wval(scalar, bits - 1, window + 1) & wmask; \ wval = booth_encode(wval, SZ); \ - ptype##_gather_booth_w##SZ(temp, table[i], wval); \ - ptype##_dadd(ret, ret, temp, NULL); \ + row_is_inf = ptype##_gather_booth_w##SZ(row, table[i], wval); \ + ptype##_dadd(sum, ret, row, NULL); \ + ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ + sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ + ret_is_inf |= sum_is_inf; \ + row_is_inf |= sum_is_inf; \ + ptype##_ccopy(ret, row, ret_is_inf); \ + ret_is_inf &= row_is_inf; \ } \ \ for (j = 0; j < SZ; j++) \ @@ -153,9 +162,17 @@ scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ wval = (scalar[0] << 1) & wmask; \ wval = booth_encode(wval, SZ); \ - ptype##_gather_booth_w##SZ(temp, table[i], wval); \ - ptype##_dadd(ret, ret, temp, NULL); \ + row_is_inf = ptype##_gather_booth_w##SZ(row, table[i], wval); \ + ptype##_dadd(sum, ret, row, NULL); \ + ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ + sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ + ret_is_inf |= sum_is_inf; \ + row_is_inf |= sum_is_inf; \ + ptype##_ccopy(ret, row, ret_is_inf); \ + ret_is_inf &= row_is_inf; \ } \ +\ + vec_czero(ret->Z, sizeof(ret->Z), ret_is_inf); \ } \ \ static void ptype##_mult_w##SZ(ptype *ret, const ptype *point, \ @@ -163,7 +180,8 @@ { \ limb_t wmask, wval; \ size_t j, window; \ - ptype temp[1]; \ + ptype sum[1], row[1]; \ + bool_t sum_is_inf, row_is_inf, ret_is_inf; \ ptype table[1<<(SZ-1)]; \ \ ptype##_precompute_w##SZ(table, point); \ @@ -177,7 +195,7 @@ : (limb_t)scalar[0] << 1; \ wval &= wmask; \ wval = booth_encode(wval, SZ); \ - ptype##_gather_booth_w##SZ(ret, table, wval); \ + ret_is_inf = ptype##_gather_booth_w##SZ(ret, table, wval); \ \ while (bits > 0) { \ for (j = 0; j < SZ; j++) \ @@ -191,10 +209,18 @@ : (limb_t)scalar[0] << 1; \ wval &= wmask; \ wval = booth_encode(wval, SZ); \ - ptype##_gather_booth_w##SZ(temp, table, wval); \ - if (bits > 0) ptype##_add(ret, ret, temp); \ - else ptype##_dadd(ret, ret, temp, NULL); \ + row_is_inf = ptype##_gather_booth_w##SZ(row, table, wval); \ + if (bits > 0) ptype##_add(sum, ret, row); \ + else ptype##_dadd(sum, ret, row, NULL); \ + ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ + sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ + ret_is_inf |= sum_is_inf; \ + row_is_inf |= sum_is_inf; \ + ptype##_ccopy(ret, row, ret_is_inf); \ + ret_is_inf &= row_is_inf; \ } \ +\ + vec_czero(ret->Z, sizeof(ret->Z), ret_is_inf); \ } #if 0 diff --git a/c-source/src/server.c b/c-source/src/server.c --- a/c-source/src/server.c +++ b/c-source/src/server.c @@ -27,4 +27,6 @@ #ifdef BLST_FR_PENTAROOT # include "pentaroot.c" #endif -#include "cpuid.c" +#ifndef __BLST_NO_CPUID__ +# include "cpuid.c" +#endif diff --git a/c-source/src/vect.h b/c-source/src/vect.h --- a/c-source/src/vect.h +++ b/c-source/src/vect.h @@ -381,6 +381,21 @@ #endif } +static inline void vec_czero(void *ret, size_t num, bool_t cbit) +{ + limb_t *rp = (limb_t *)ret; + size_t i; + limb_t mask; + + launder(cbit); + mask = (limb_t)0 - (cbit^1); + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) + rp[i] &= mask; +} + /* * Some compilers get arguably overzealous(*) when passing pointer to * multi-dimensional array [such as vec384x] as 'const' argument. diff --git a/hsblst.cabal b/hsblst.cabal --- a/hsblst.cabal +++ b/hsblst.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: hsblst -version: 0.0.3 +version: 0.0.4 synopsis: Haskell bindings to BLST description: HsBLST is low-level Haskell bindings and a high-level interface to [BLST](https://github.com/supranational/blst) -- a multilingual BLS12-381 signature library. category: Cryptography @@ -18,9 +18,8 @@ license-file: LICENSE build-type: Simple tested-with: - GHC == 9.4.8 - , GHC == 9.6.3 - , GHC == 9.8.1 + GHC == 9.8.4 + , GHC == 9.10.1 extra-source-files: c-source/build/coff/add_mod_256-x86_64.s c-source/build/coff/add_mod_384-x86_64.s @@ -65,6 +64,15 @@ c-source/build/mach-o/sha256-portable-x86_64.s c-source/build/mach-o/sha256-x86_64.s c-source/build/assembly.S + c-source/build/cheri/add_mod_256-armv8.S + c-source/build/cheri/add_mod_384-armv8.S + c-source/build/cheri/ct_inverse_mod_256-armv8.S + c-source/build/cheri/ct_inverse_mod_384-armv8.S + c-source/build/cheri/ct_is_square_mod_384-armv8.S + c-source/build/cheri/div3w-armv8.S + c-source/build/cheri/mul_mont_256-armv8.S + c-source/build/cheri/mul_mont_384-armv8.S + c-source/build/cheri/sha256-armv8.S c-source/build/coff/add_mod_256-armv8.S c-source/build/coff/add_mod_384-armv8.S c-source/build/coff/ct_inverse_mod_256-armv8.S @@ -208,8 +216,8 @@ c-sources: c-source/src/server.c build-depends: - base >=4.17 && <4.20 - , deepseq >=1.4.5.0 && <1.5.1 + base >=4.17 && <4.21 + , deepseq >=1.4.5.0 && <1.6.0 , memory >=0.16.0 && <0.19 if impl(ghc >= 9.2) ghc-options: -Wno-missing-kind-signatures @@ -283,7 +291,7 @@ build-tool-depends: tasty-discover:tasty-discover build-depends: - base >=4.17 && <4.20 + base >=4.17 && <4.21 , base16-bytestring ==1.0.* , bytestring >=0.10.12.1 && <0.13 , hsblst