diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ - SPDX-License-Identifier: MPL-2.0 --> +## v0.0.5 + +* Moved to ram instead of memory +* Added ForeignFunctionInterface to extensions for cabal to build successfully +* Bundled blst version bumped to v0.3.16 + ## v0.0.4 * Bundled blst version bumped to v0.3.13 diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -11,6 +11,10 @@ This package includes sources from v0.3.13 version of BLST, licensed under [Apache License 2.0](https://github.com/supranational/blst/blob/master/LICENSE) +## Known Issues + +- Fails to build on Windows with GHC 9.12.3, seems to be a GHC bug. + ## Build Instructions [↑](#hsblst) Run `make` to build everything. diff --git a/c-source/README.md b/c-source/README.md --- a/c-source/README.md +++ b/c-source/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://api.travis-ci.com/supranational/blst.svg?branch=master)](https://travis-ci.com/github/supranational/blst) [![Actions status](https://github.com/supranational/blst/workflows/build/badge.svg)](https://github.com/supranational/blst/actions) [![CodeQL status](https://github.com/supranational/blst/workflows/CodeQL/badge.svg)](https://github.com/supranational/blst/actions/workflows/codeql-analysis.yml) +[![Actions status](https://github.com/supranational/blst/workflows/build/badge.svg)](https://github.com/supranational/blst/actions) [![CodeQL status](https://github.com/supranational/blst/workflows/CodeQL/badge.svg)](https://github.com/supranational/blst/actions/workflows/codeql-analysis.yml)
@@ -212,7 +212,7 @@ * **Asm** - folder containing Perl scripts that are used to generate assembly code for different hardware platforms including x86 with ADX instructions, x86 without ADX instructions, and ARMv8, and [ABI](https://en.wikipedia.org/wiki/Application_binary_interface)[1] * **Build** - this folder containing a set of pre-generated assembly files for a variety of operating systems and maintenance scripts. * **Cheri** - assembly code for use on [CHERI](https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/) platforms - * **Coff** - assembly code for use on Window systems with GNU toolchain + * **Coff** - assembly code for use on Windows systems with GNU or LLVM toolchain * **Elf** - assembly code for use on Unix systems * **Mach-o** - assembly code for use on Apple operating systems * **Win64** - assembly code for use on Windows systems with Microsoft toolchain diff --git a/c-source/bindings/blst.h b/c-source/bindings/blst.h --- a/c-source/bindings/blst.h +++ b/c-source/bindings/blst.h @@ -23,15 +23,17 @@ #ifdef __cplusplus extern "C" { -#elif defined(__BLST_CGO__) +#elif !defined(__STDC_VERSION__) || __STDC_VERSION__<202311 +# if defined(__BLST_CGO__) typedef _Bool bool; /* it's assumed that cgo calls modern enough compiler */ -#elif !defined(bool) -# if defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 -# define bool _Bool -# else -# define bool int +# elif !defined(bool) +# if defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 +# define bool _Bool +# else +# define bool int +# endif +# define __blst_h_bool__ # endif -# define __blst_h_bool__ #endif #ifdef SWIG diff --git a/c-source/build/cheri/add_mod_256-armv8.S b/c-source/build/cheri/add_mod_256-armv8.S --- a/c-source/build/cheri/add_mod_256-armv8.S +++ b/c-source/build/cheri/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[c1] ldp x12,x13,[c2] @@ -39,6 +48,7 @@ .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[c1] ldp x14,x15,[c1,#16] @@ -88,6 +98,7 @@ .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -126,6 +137,7 @@ .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -377,3 +389,13 @@ ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/add_mod_384-armv8.S b/c-source/build/cheri/add_mod_384-armv8.S --- a/c-source/build/cheri/add_mod_384-armv8.S +++ b/c-source/build/cheri/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -672,6 +681,7 @@ .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -753,14 +763,14 @@ .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + ld1 {v0.2d, v1.2d}, [c1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + ld1 {v3.2d, v4.2d}, [c2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [c0] + st1 {v0.2d, v1.2d}, [c0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [c0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [c0], #16 ld1 {v1.2d}, [c1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/ct_inverse_mod_256-armv8.S b/c-source/build/cheri/ct_inverse_mod_256-armv8.S --- a/c-source/build/cheri/ct_inverse_mod_256-armv8.S +++ b/c-source/build/cheri/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-10*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -48,7 +56,7 @@ mov x13, x15 // |g1| add c0,c0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [c0,#8*9] // initialize |v| with |f1| + str x12, [c0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -71,19 +79,17 @@ bl __smul_256_n_shift_by_31 ldr x8, [c1,#8*8] // |u| - ldr x9, [c1,#8*13] // |v| + ldr x9, [c1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [c0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*5] - stp x5, x5, [c0,#8*7] + stp x4, x5, [c0,#8*4] + stp x5, x5, [c0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [c0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*10] + stp x4, x5, [c0,#8*10] stp x5, x5, [c0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -106,16 +112,10 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -137,16 +137,10 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -168,16 +162,10 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -199,16 +187,10 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -230,16 +212,10 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -261,16 +237,15 @@ add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - + asr x24, x24, #63 + str x24, [c0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [c0,#8*4] + stp x24, x24, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -294,10 +269,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -323,10 +297,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -352,10 +325,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -381,10 +353,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -410,10 +381,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -439,10 +409,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -468,10 +437,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations @@ -541,7 +509,7 @@ ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldr c29, [csp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +547,11 @@ adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [c1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [c1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [c1,#8*2+104] + ldp x10, x11, [c1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [c1,#8*4+104] + ldr x23, [c1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +593,9 @@ .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [c1,#8*18] // load rest of |v| + ldr x5, [c1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [c1,#8*20] + ldp x6, x7, [c1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +846,13 @@ ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/ct_inverse_mod_384-armv8.S b/c-source/build/cheri/ct_inverse_mod_384-armv8.S --- a/c-source/build/cheri/ct_inverse_mod_384-armv8.S +++ b/c-source/build/cheri/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -47,14 +55,14 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[c0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [c0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [c0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -68,17 +76,17 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [c1,#8*12] // |u| - ldr x8, [c1,#8*18] // |v| + ldr x8, [c1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,10 +104,10 @@ smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [c0,#8*12] + stp x3, x4, [c0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*14] stp x5, x5, [c0,#8*16] + stp x5, x5, [c0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -111,22 +119,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -138,22 +145,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -165,22 +171,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -192,22 +197,23 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [c0,#8*6] stp x27, x27, [c0,#8*8] @@ -223,23 +229,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -251,23 +258,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -279,23 +287,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -307,23 +316,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -335,23 +345,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -375,20 +386,22 @@ mov x15, x17 mov x16, x19 add c0,c0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [c1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +412,60 @@ mov x20, x17 mov x21, x19 ldp c0, c15, [csp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr c30, [c29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [c15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [c15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [c15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [c15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [c0,#8*6] adcs x7, x7, x13 stp x5, x6, [c0,#8*8] @@ -431,15 +479,15 @@ ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [c1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [c1,#8*2+96] @@ -447,6 +495,7 @@ ldp x7, x8, [c1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [c1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +510,33 @@ umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [c1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [c1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [c1,#8*2+144] + ldp x11, x12, [c1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [c1,#8*4+144] + ldp x13, x14, [c1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [c1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +551,25 @@ umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +581,41 @@ stp x5, x6, [c0,#8*2] adcs x27, x27, x28 stp x7, x27, [c0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [c1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [c1,#8*26] - ldp x7, x8, [c1,#8*28] +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [c1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [c1,#8*28] + and x25, x25, x20 + ldp x7, x8, [c1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +623,11 @@ adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +636,15 @@ stp x3, x4, [c0,#8*6] adcs x7, x7, x28 stp x5, x6, [c0,#8*8] - adc x8, x8, x28 - stp x7, x8, [c0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [c0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [c1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [c1,#8*2+0] @@ -605,25 +664,27 @@ adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [c1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [c1,#8*2+48] @@ -643,25 +704,27 @@ adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +763,7 @@ sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +856,13 @@ ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/ct_is_square_mod_384-armv8.S b/c-source/build/cheri/ct_is_square_mod_384-armv8.S --- a/c-source/build/cheri/ct_is_square_mod_384-armv8.S +++ b/c-source/build/cheri/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/div3w-armv8.S b/c-source/build/cheri/div3w-armv8.S --- a/c-source/build/cheri/div3w-armv8.S +++ b/c-source/build/cheri/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[c0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[c1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[c1] ldr x8,[c0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/mul_mont_256-armv8.S b/c-source/build/cheri/mul_mont_256-armv8.S --- a/c-source/build/cheri/mul_mont_256-armv8.S +++ b/c-source/build/cheri/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -330,7 +339,7 @@ stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -374,7 +383,7 @@ stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/mul_mont_384-armv8.S b/c-source/build/cheri/mul_mont_384-armv8.S --- a/c-source/build/cheri/mul_mont_384-armv8.S +++ b/c-source/build/cheri/mul_mont_384-armv8.S @@ -1,10 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 +.hidden add_mod_384x384 .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -22,7 +31,7 @@ ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -82,10 +91,11 @@ .size __add_mod_384x384,.-__add_mod_384x384 .globl sub_mod_384x384 +.hidden sub_mod_384x384 .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -103,7 +113,7 @@ ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -239,7 +249,7 @@ .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -312,7 +322,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -321,7 +331,7 @@ .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -397,7 +407,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -406,7 +416,7 @@ .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -438,7 +448,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -819,7 +829,7 @@ .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -856,7 +866,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -865,7 +875,7 @@ .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -921,7 +931,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1042,7 +1052,7 @@ .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1064,7 +1074,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1073,7 +1083,7 @@ .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1097,7 +1107,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1106,7 +1116,7 @@ .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1147,7 +1157,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1365,7 +1375,7 @@ .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1383,7 +1393,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1569,7 +1579,7 @@ .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1651,7 +1661,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1660,7 +1670,7 @@ .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1751,7 +1761,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x @@ -1760,7 +1770,7 @@ .type sqr_mont_382x,%function .align 5 sqr_mont_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1875,7 +1885,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_382x,.-sqr_mont_382x @@ -2224,7 +2234,7 @@ .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2269,7 +2279,7 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2278,7 +2288,7 @@ .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2367,6 +2377,16 @@ ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/cheri/sha256-armv8.S b/c-source/build/cheri/sha256-armv8.S --- a/c-source/build/cheri/sha256-armv8.S +++ b/c-source/build/cheri/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -43,9 +51,11 @@ .align 2 .align 2 .globl blst_sha256_block_armv8 +.hidden blst_sha256_block_armv8 .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -182,9 +192,11 @@ ret .size blst_sha256_block_armv8,.-blst_sha256_block_armv8 .globl blst_sha256_block_data_order +.hidden blst_sha256_block_data_order .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp c16,__blst_platform_cap ldr w16,[c16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1034,6 +1046,7 @@ .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] #ifndef __AARCH64EB__ @@ -1062,6 +1075,7 @@ .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[c1],#1 sub x2,x2,#1 @@ -1075,9 +1089,20 @@ .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] stp x4,x5,[c0] stp x6,x7,[c0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/coff/add_mod_256-armv8.S b/c-source/build/coff/add_mod_256-armv8.S --- a/c-source/build/coff/add_mod_256-armv8.S +++ b/c-source/build/coff/add_mod_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -43,6 +44,7 @@ .endef .p2align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -94,6 +96,7 @@ .endef .p2align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -134,6 +137,7 @@ .endef .p2align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/c-source/build/coff/add_mod_384-armv8.S b/c-source/build/coff/add_mod_384-armv8.S --- a/c-source/build/coff/add_mod_384-armv8.S +++ b/c-source/build/coff/add_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -27,7 +27,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -77,7 +77,7 @@ .endef .p2align 5 add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +105,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -116,7 +116,7 @@ .endef .p2align 5 rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -143,7 +143,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -182,7 +182,7 @@ .endef .p2align 5 div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -206,7 +206,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -217,7 +217,7 @@ .endef .p2align 5 lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -244,7 +244,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -286,7 +286,7 @@ .endef .p2align 5 mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -316,7 +316,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -327,7 +327,7 @@ .endef .p2align 5 mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -353,7 +353,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -364,7 +364,7 @@ .endef .p2align 5 mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -409,7 +409,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -420,7 +420,7 @@ .endef .p2align 5 mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -457,7 +457,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -468,7 +468,7 @@ .endef .p2align 5 cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -510,7 +510,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -521,7 +521,7 @@ .endef .p2align 5 sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -541,7 +541,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -588,7 +588,7 @@ .endef .p2align 5 sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -616,7 +616,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -627,7 +627,7 @@ .endef .p2align 5 mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -660,7 +660,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -671,6 +671,7 @@ .endef .p2align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -710,6 +711,7 @@ .endef .p2align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -793,14 +795,14 @@ .endef .p2align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl vec_select_48 @@ -810,6 +812,7 @@ .endef .p2align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -827,6 +830,7 @@ .endef .p2align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -850,6 +854,7 @@ .endef .p2align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -885,6 +890,7 @@ .endef .p2align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -914,6 +920,7 @@ .endef .p2align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -961,6 +968,7 @@ .endef .p2align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -1003,6 +1011,7 @@ .endef .p2align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -1030,6 +1039,7 @@ .endef .p2align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/c-source/build/coff/ct_inverse_mod_256-armv8.S b/c-source/build/coff/ct_inverse_mod_256-armv8.S --- a/c-source/build/coff/ct_inverse_mod_256-armv8.S +++ b/c-source/build/coff/ct_inverse_mod_256-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -50,7 +50,7 @@ mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -73,19 +73,17 @@ bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -108,16 +106,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -139,16 +131,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -170,16 +156,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -201,16 +181,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -232,16 +206,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -263,16 +231,15 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -296,10 +263,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -325,10 +291,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -354,10 +319,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -383,10 +347,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -412,10 +375,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -441,10 +403,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -470,10 +431,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations @@ -543,7 +503,7 @@ ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -583,11 +543,11 @@ adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -631,9 +591,9 @@ .p2align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/c-source/build/coff/ct_inverse_mod_384-armv8.S b/c-source/build/coff/ct_inverse_mod_384-armv8.S --- a/c-source/build/coff/ct_inverse_mod_384-armv8.S +++ b/c-source/build/coff/ct_inverse_mod_384-armv8.S @@ -1,13 +1,13 @@ .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; +.def ct_inverse_mod_384; .type 32; .endef .p2align 5 -ct_inverse_mod_383: -.long 3573752639 +ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -49,14 +49,14 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -70,17 +70,17 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -98,10 +98,10 @@ smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -113,22 +113,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -140,22 +139,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -167,22 +165,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -194,22 +191,23 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] @@ -225,23 +223,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -253,23 +252,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -281,23 +281,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -309,23 +310,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -337,23 +339,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -377,20 +380,22 @@ mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -401,25 +406,60 @@ mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -433,17 +473,17 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.def __smul_383x63; +.def __smul_384x63; .type 32; .endef .p2align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -451,6 +491,7 @@ ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -465,28 +506,33 @@ umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -501,21 +547,25 @@ umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -527,43 +577,43 @@ stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.def __smul_767x63_tail; +.def __smul_768x63_tail; .type 32; .endef .p2align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -571,10 +621,11 @@ adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -583,17 +634,17 @@ stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.def __smul_383_n_shift_by_62; +.def __smul_384_n_shift_by_62; .type 32; .endef .p2align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -613,25 +664,27 @@ adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -651,25 +704,27 @@ adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/c-source/build/coff/ct_is_square_mod_384-armv8.S b/c-source/build/coff/ct_is_square_mod_384-armv8.S --- a/c-source/build/coff/ct_is_square_mod_384-armv8.S +++ b/c-source/build/coff/ct_is_square_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -86,7 +86,7 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/coff/ctq_inverse_mod_384-x86_64.s b/c-source/build/coff/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/coff/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/coff/ctq_inverse_mod_384-x86_64.s @@ -1,16 +1,16 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; .scl 2; .type 32; .endef +.def ct_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ct_inverse_mod_383: +ct_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ct_inverse_mod_383: +.LSEH_begin_ct_inverse_mod_384: movq %rcx,%rdi @@ -19,7 +19,7 @@ movq %r9,%rcx #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp @@ -35,7 +35,7 @@ subq $1112,%rsp -.LSEH_body_ct_inverse_mod_383: +.LSEH_body_ct_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -82,7 +82,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -90,10 +90,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -106,19 +106,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -135,6 +135,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -145,13 +146,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -162,14 +164,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -177,12 +179,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -193,14 +195,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -208,12 +210,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -224,14 +226,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -239,12 +241,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -255,14 +257,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -270,19 +272,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -293,14 +293,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -308,12 +308,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -324,14 +324,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -339,12 +339,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -355,14 +355,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -370,12 +370,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -386,14 +386,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -401,12 +401,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -417,14 +417,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -432,12 +432,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -461,16 +461,16 @@ leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -493,37 +493,77 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -540,7 +580,7 @@ leaq 48(%r8),%rsp -.LSEH_epilogue_ct_inverse_mod_383: +.LSEH_epilogue_ct_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -554,10 +594,10 @@ .byte 0xf3,0xc3 #endif -.LSEH_end_ct_inverse_mod_383: -.def __smulq_767x63; .scl 3; .type 32; .endef +.LSEH_end_ct_inverse_mod_384: +.def __smulq_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_767x63: +__smulq_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -566,6 +606,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -574,7 +615,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -585,16 +626,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -620,14 +665,14 @@ adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -730,39 +775,41 @@ movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -774,9 +821,9 @@ .byte 0xf3,0xc3 #endif -.def __smulq_383x63; .scl 3; .type 32; .endef +.def __smulq_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_383x63: +__smulq_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -785,6 +832,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -800,16 +848,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -831,10 +883,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -842,13 +895,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -864,16 +919,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -895,17 +954,19 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -913,6 +974,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -924,9 +986,9 @@ .byte 0xf3,0xc3 #endif -.def __smulq_383_n_shift_by_62; .scl 3; .type 32; .endef +.def __smulq_384_n_shift_by_62; .scl 3; .type 32; .endef .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx @@ -951,6 +1013,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1236,28 +1303,28 @@ .section .pdata .p2align 2 -.rva .LSEH_begin_ct_inverse_mod_383 -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_prologue +.rva .LSEH_begin_ct_inverse_mod_384 +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_prologue -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_body +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_body -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_end_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_end_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ct_inverse_mod_383_prologue: +.LSEH_info_ct_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ct_inverse_mod_383_body: +.LSEH_info_ct_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1270,7 +1337,7 @@ .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ct_inverse_mod_383_epilogue: +.LSEH_info_ct_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/c-source/build/coff/ctx_inverse_mod_384-x86_64.s b/c-source/build/coff/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/coff/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/coff/ctx_inverse_mod_384-x86_64.s @@ -1,22 +1,22 @@ .text -.globl ctx_inverse_mod_383 +.globl ctx_inverse_mod_384 -.def ctx_inverse_mod_383; .scl 2; .type 32; .endef +.def ctx_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ctx_inverse_mod_383: +.LSEH_begin_ctx_inverse_mod_384: movq %rcx,%rdi movq %rdx,%rsi movq %r8,%rdx movq %r9,%rcx -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp pushq %rbx @@ -31,7 +31,7 @@ subq $1112,%rsp -.LSEH_body_ctx_inverse_mod_383: +.LSEH_body_ctx_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -81,7 +81,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -89,10 +89,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -105,19 +105,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -134,6 +134,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -144,13 +145,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -161,14 +163,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -176,12 +178,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -192,14 +194,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -207,12 +209,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -223,14 +225,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -238,12 +240,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -254,14 +256,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -269,12 +271,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -285,14 +287,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -300,12 +302,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -316,14 +318,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -331,12 +333,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -347,14 +349,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -362,12 +364,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -378,14 +380,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -393,12 +395,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -409,14 +411,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -424,12 +426,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -440,14 +442,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -455,19 +457,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -478,14 +478,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -493,12 +493,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -509,14 +509,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -524,12 +524,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -540,14 +540,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -555,12 +555,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -571,14 +571,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -586,12 +586,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -602,14 +602,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -617,12 +617,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -633,14 +633,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -648,12 +648,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -664,14 +664,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -679,12 +679,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -710,12 +710,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -741,12 +741,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -772,12 +772,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -803,21 +803,21 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -834,40 +834,80 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -884,7 +924,7 @@ leaq 48(%r8),%rsp -.LSEH_epilogue_ctx_inverse_mod_383: +.LSEH_epilogue_ctx_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -898,10 +938,10 @@ .byte 0xf3,0xc3 #endif -.LSEH_end_ctx_inverse_mod_383: -.def __smulx_767x63; .scl 3; .type 32; .endef +.LSEH_end_ctx_inverse_mod_384: +.def __smulx_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_767x63: +__smulx_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -910,6 +950,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -918,7 +959,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -928,37 +969,41 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -993,7 +1038,7 @@ xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1005,63 +1050,65 @@ adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax - adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax - adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx - movq 16(%rsp),%rsi - adcq %rdi,%rax + adcq $0,%rax - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi adcq %rdi,%rbx - adcq %rdi,%rbp + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi adcq %rdi,%rcx - adcq %rdi,%rax + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx - movq %rdx,%rdi + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 + adcq %rsi,%rbx + adcq %rsi,%rbp + adcq %rsi,%rcx + adcq %rsi,%rax + adcq %rsi,%rdx - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq 16(%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) + #ifdef __SGX_LVI_HARDENING__ popq %r8 @@ -1072,9 +1119,9 @@ .byte 0xf3,0xc3 #endif -.def __smulx_383x63; .scl 3; .type 32; .endef +.def __smulx_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_383x63: +__smulx_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0+0(%rsi),%r8 @@ -1083,6 +1130,7 @@ movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1098,13 +1146,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1117,19 +1170,22 @@ mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1145,13 +1201,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1163,13 +1224,15 @@ adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1177,6 +1240,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1188,13 +1252,12 @@ .byte 0xf3,0xc3 #endif -.def __smulx_383_n_shift_by_31; .scl 3; .type 32; .endef +.def __smulx_384_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1578,9 +1646,9 @@ #endif -.def __tail_loop_53; .scl 3; .type 32; .endef +.def __tail_loop_55; .scl 3; .type 32; .endef .p2align 5 -__tail_loop_53: +__tail_loop_55: .byte 0xf3,0x0f,0x1e,0xfa movq $1,%rdx @@ -1588,7 +1656,7 @@ xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1615,7 +1683,7 @@ subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1629,28 +1697,28 @@ .section .pdata .p2align 2 -.rva .LSEH_begin_ctx_inverse_mod_383 -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_prologue +.rva .LSEH_begin_ctx_inverse_mod_384 +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_prologue -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_body +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_body -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_end_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_end_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ctx_inverse_mod_383_prologue: +.LSEH_info_ctx_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ctx_inverse_mod_383_body: +.LSEH_info_ctx_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1663,7 +1731,7 @@ .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ctx_inverse_mod_383_epilogue: +.LSEH_info_ctx_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/c-source/build/coff/div3w-armv8.S b/c-source/build/coff/div3w-armv8.S --- a/c-source/build/coff/div3w-armv8.S +++ b/c-source/build/coff/div3w-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -43,6 +44,7 @@ .endef .p2align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -82,6 +84,7 @@ .endef .p2align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/c-source/build/coff/mul_mont_256-armv8.S b/c-source/build/coff/mul_mont_256-armv8.S --- a/c-source/build/coff/mul_mont_256-armv8.S +++ b/c-source/build/coff/mul_mont_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -200,7 +201,7 @@ .endef .p2align 5 sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -301,7 +302,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl from_mont_256 @@ -311,7 +312,7 @@ .endef .p2align 5 from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -336,7 +337,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -347,7 +348,7 @@ .endef .p2align 5 redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -382,7 +383,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/coff/mul_mont_384-armv8.S b/c-source/build/coff/mul_mont_384-armv8.S --- a/c-source/build/coff/mul_mont_384-armv8.S +++ b/c-source/build/coff/mul_mont_384-armv8.S @@ -1,12 +1,13 @@ .text .globl add_mod_384x384 + .def add_mod_384x384; .type 32; .endef .p2align 5 add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -24,7 +25,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -86,12 +87,13 @@ .globl sub_mod_384x384 + .def sub_mod_384x384; .type 32; .endef .p2align 5 sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -109,7 +111,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -253,7 +255,7 @@ .endef .p2align 5 mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -326,7 +328,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -337,7 +339,7 @@ .endef .p2align 5 sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -413,7 +415,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -424,7 +426,7 @@ .endef .p2align 5 mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -456,7 +458,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -841,7 +843,7 @@ .endef .p2align 5 sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -878,7 +880,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -889,7 +891,7 @@ .endef .p2align 5 sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -945,7 +947,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .def __sqr_384; @@ -1070,7 +1072,7 @@ .endef .p2align 5 sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1092,7 +1094,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1103,7 +1105,7 @@ .endef .p2align 5 redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1127,7 +1129,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1138,7 +1140,7 @@ .endef .p2align 5 from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1179,7 +1181,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1403,7 +1405,7 @@ .endef .p2align 5 mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1421,7 +1423,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1611,7 +1613,7 @@ .endef .p2align 5 mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1693,7 +1695,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1704,7 +1706,7 @@ .endef .p2align 5 sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1795,7 +1797,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1806,7 +1808,7 @@ .endef .p2align 5 sqr_mont_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1921,7 +1923,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2274,7 +2276,7 @@ .endef .p2align 5 sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2319,7 +2321,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2330,7 +2332,7 @@ .endef .p2align 5 sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2419,6 +2421,6 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/coff/sha256-armv8.S b/c-source/build/coff/sha256-armv8.S --- a/c-source/build/coff/sha256-armv8.S +++ b/c-source/build/coff/sha256-armv8.S @@ -43,11 +43,13 @@ .align 2 .p2align 2 .globl blst_sha256_block_armv8 + .def blst_sha256_block_armv8; .type 32; .endef .p2align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -184,11 +186,13 @@ ret .globl blst_sha256_block_data_order + .def blst_sha256_block_data_order; .type 32; .endef .p2align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1040,6 +1044,7 @@ .endef .p2align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1070,6 +1075,7 @@ .endef .p2align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1085,6 +1091,7 @@ .endef .p2align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/c-source/build/elf/add_mod_256-armv8.S b/c-source/build/elf/add_mod_256-armv8.S --- a/c-source/build/elf/add_mod_256-armv8.S +++ b/c-source/build/elf/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +48,7 @@ .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +98,7 @@ .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +137,7 @@ .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -377,3 +389,13 @@ ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/add_mod_384-armv8.S b/c-source/build/elf/add_mod_384-armv8.S --- a/c-source/build/elf/add_mod_384-armv8.S +++ b/c-source/build/elf/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +681,7 @@ .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +763,14 @@ .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/ct_inverse_mod_256-armv8.S b/c-source/build/elf/ct_inverse_mod_256-armv8.S --- a/c-source/build/elf/ct_inverse_mod_256-armv8.S +++ b/c-source/build/elf/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -48,7 +56,7 @@ mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -71,19 +79,17 @@ bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -106,16 +112,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -137,16 +137,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -168,16 +162,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -199,16 +187,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -230,16 +212,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -261,16 +237,15 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -294,10 +269,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -323,10 +297,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -352,10 +325,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -381,10 +353,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -410,10 +381,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -439,10 +409,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -468,10 +437,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations @@ -541,7 +509,7 @@ ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +547,11 @@ adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +593,9 @@ .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +846,13 @@ ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/ct_inverse_mod_384-armv8.S b/c-source/build/elf/ct_inverse_mod_384-armv8.S --- a/c-source/build/elf/ct_inverse_mod_384-armv8.S +++ b/c-source/build/elf/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -47,14 +55,14 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -68,17 +76,17 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,10 +104,10 @@ smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -111,22 +119,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -138,22 +145,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -165,22 +171,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -192,22 +197,23 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] @@ -223,23 +229,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -251,23 +258,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -279,23 +287,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -307,23 +316,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -335,23 +345,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -375,20 +386,22 @@ mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +412,60 @@ mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,15 +479,15 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +495,7 @@ ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +510,33 @@ umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +551,25 @@ umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +581,41 @@ stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +623,11 @@ adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +636,15 @@ stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +664,27 @@ adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +704,27 @@ adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +763,7 @@ sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +856,13 @@ ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/ct_is_square_mod_384-armv8.S b/c-source/build/elf/ct_is_square_mod_384-armv8.S --- a/c-source/build/elf/ct_is_square_mod_384-armv8.S +++ b/c-source/build/elf/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/ctq_inverse_mod_384-x86_64.s b/c-source/build/elf/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/elf/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/elf/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383,@function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384,@function .align 32 -ct_inverse_mod_383: +ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -549,10 +589,10 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size ct_inverse_mod_383,.-ct_inverse_mod_383 -.type __smulq_767x63,@function +.size ct_inverse_mod_384,.-ct_inverse_mod_384 +.type __smulq_768x63,@function .align 32 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -770,10 +817,10 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_767x63,.-__smulq_767x63 -.type __smulq_383x63,@function +.size __smulq_768x63,.-__smulq_768x63 +.type __smulq_384x63,@function .align 32 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -922,10 +984,10 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383x63,.-__smulq_383x63 -.type __smulq_383_n_shift_by_62,@function +.size __smulq_384x63,.-__smulq_384x63 +.type __smulq_384_n_shift_by_62,@function .align 32 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1109,7 +1176,7 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383_n_shift_by_62,.-__smulq_383_n_shift_by_62 +.size __smulq_384_n_shift_by_62,.-__smulq_384_n_shift_by_62 .type __ab_approximation_62,@function .align 32 __ab_approximation_62: diff --git a/c-source/build/elf/ctx_inverse_mod_384-x86_64.s b/c-source/build/elf/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/elf/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/elf/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl ctx_inverse_mod_383 -.hidden ctx_inverse_mod_383 -.type ctx_inverse_mod_383,@function +.globl ctx_inverse_mod_384 +.hidden ctx_inverse_mod_384 +.type ctx_inverse_mod_384,@function .align 32 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -893,10 +933,10 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size ctx_inverse_mod_383,.-ctx_inverse_mod_383 -.type __smulx_767x63,@function +.size ctx_inverse_mod_384,.-ctx_inverse_mod_384 +.type __smulx_768x63,@function .align 32 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,63 +1046,65 @@ adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax - adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax - adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx - movq 16(%rsp),%rsi - adcq %rdi,%rax + adcq $0,%rax - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi adcq %rdi,%rbx - adcq %rdi,%rbp + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi adcq %rdi,%rcx - adcq %rdi,%rax + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx - movq %rdx,%rdi + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 + adcq %rsi,%rbx + adcq %rsi,%rbp + adcq %rsi,%rcx + adcq %rsi,%rax + adcq %rsi,%rdx - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq 16(%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) + #ifdef __SGX_LVI_HARDENING__ popq %r8 @@ -1068,10 +1115,10 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_767x63,.-__smulx_767x63 -.type __smulx_383x63,@function +.size __smulx_768x63,.-__smulx_768x63 +.type __smulx_384x63,@function .align 32 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,13 +1144,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1115,19 +1168,22 @@ mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,13 +1199,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1161,13 +1222,15 @@ adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1186,15 +1250,14 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383x63,.-__smulx_383x63 -.type __smulx_383_n_shift_by_31,@function +.size __smulx_384x63,.-__smulx_384x63 +.type __smulx_384_n_shift_by_31,@function .align 32 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1342,7 +1410,7 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383_n_shift_by_31,.-__smulx_383_n_shift_by_31 +.size __smulx_384_n_shift_by_31,.-__smulx_384_n_shift_by_31 .type __smulx_191_n_shift_by_31,@function .align 32 __smulx_191_n_shift_by_31: @@ -1585,9 +1653,9 @@ .cfi_endproc .size __inner_loop_31,.-__inner_loop_31 -.type __tail_loop_53,@function +.type __tail_loop_55,@function .align 32 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1635,7 +1703,7 @@ .byte 0xf3,0xc3 #endif .cfi_endproc -.size __tail_loop_53,.-__tail_loop_53 +.size __tail_loop_55,.-__tail_loop_55 .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/c-source/build/elf/div3w-armv8.S b/c-source/build/elf/div3w-armv8.S --- a/c-source/build/elf/div3w-armv8.S +++ b/c-source/build/elf/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/mul_mont_256-armv8.S b/c-source/build/elf/mul_mont_256-armv8.S --- a/c-source/build/elf/mul_mont_256-armv8.S +++ b/c-source/build/elf/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +339,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +383,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/mul_mont_384-armv8.S b/c-source/build/elf/mul_mont_384-armv8.S --- a/c-source/build/elf/mul_mont_384-armv8.S +++ b/c-source/build/elf/mul_mont_384-armv8.S @@ -1,10 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 +.hidden add_mod_384x384 .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -22,7 +31,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -82,10 +91,11 @@ .size __add_mod_384x384,.-__add_mod_384x384 .globl sub_mod_384x384 +.hidden sub_mod_384x384 .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -103,7 +113,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -239,7 +249,7 @@ .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -312,7 +322,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -321,7 +331,7 @@ .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -397,7 +407,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -406,7 +416,7 @@ .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -438,7 +448,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -819,7 +829,7 @@ .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -856,7 +866,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -865,7 +875,7 @@ .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -921,7 +931,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1042,7 +1052,7 @@ .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1064,7 +1074,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1073,7 +1083,7 @@ .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1097,7 +1107,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1106,7 +1116,7 @@ .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1147,7 +1157,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1365,7 +1375,7 @@ .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1383,7 +1393,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1569,7 +1579,7 @@ .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1651,7 +1661,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1660,7 +1670,7 @@ .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1751,7 +1761,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x @@ -1760,7 +1770,7 @@ .type sqr_mont_382x,%function .align 5 sqr_mont_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1875,7 +1885,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_382x,.-sqr_mont_382x @@ -2224,7 +2234,7 @@ .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2269,7 +2279,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2278,7 +2288,7 @@ .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2367,6 +2377,16 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/elf/sha256-armv8.S b/c-source/build/elf/sha256-armv8.S --- a/c-source/build/elf/sha256-armv8.S +++ b/c-source/build/elf/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -43,9 +51,11 @@ .align 2 .align 2 .globl blst_sha256_block_armv8 +.hidden blst_sha256_block_armv8 .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -182,9 +192,11 @@ ret .size blst_sha256_block_armv8,.-blst_sha256_block_armv8 .globl blst_sha256_block_data_order +.hidden blst_sha256_block_data_order .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1034,6 +1046,7 @@ .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1062,6 +1075,7 @@ .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1075,9 +1089,20 @@ .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] stp x6,x7,[x0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/c-source/build/mach-o/add_mod_256-armv8.S b/c-source/build/mach-o/add_mod_256-armv8.S --- a/c-source/build/mach-o/add_mod_256-armv8.S +++ b/c-source/build/mach-o/add_mod_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +40,7 @@ .align 5 _mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +90,7 @@ .align 5 _lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +129,7 @@ .align 5 _rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/c-source/build/mach-o/add_mod_384-armv8.S b/c-source/build/mach-o/add_mod_384-armv8.S --- a/c-source/build/mach-o/add_mod_384-armv8.S +++ b/c-source/build/mach-o/add_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -71,7 +71,7 @@ .align 5 _add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +99,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -108,7 +108,7 @@ .align 5 _rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +135,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -170,7 +170,7 @@ .align 5 _div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +194,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -203,7 +203,7 @@ .align 5 _lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +230,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -268,7 +268,7 @@ .align 5 _mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +298,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -307,7 +307,7 @@ .align 5 _mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +333,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -342,7 +342,7 @@ .align 5 _mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +387,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -396,7 +396,7 @@ .align 5 _mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +433,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -442,7 +442,7 @@ .align 5 _cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +484,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -493,7 +493,7 @@ .align 5 _sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +513,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -556,7 +556,7 @@ .align 5 _sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +584,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -593,7 +593,7 @@ .align 5 _mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +626,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -635,6 +635,7 @@ .align 5 _sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +673,7 @@ .align 5 _sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +755,14 @@ .align 5 _vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl _vec_select_48 @@ -768,6 +770,7 @@ .align 5 _vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +786,7 @@ .align 5 _vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +808,7 @@ .align 5 _vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +842,7 @@ .align 5 _vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +870,7 @@ .align 5 _vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +916,7 @@ .align 5 _vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +957,7 @@ .align 5 _vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +983,7 @@ .align 5 _vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/c-source/build/mach-o/ct_inverse_mod_256-armv8.S b/c-source/build/mach-o/ct_inverse_mod_256-armv8.S --- a/c-source/build/mach-o/ct_inverse_mod_256-armv8.S +++ b/c-source/build/mach-o/ct_inverse_mod_256-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -48,7 +48,7 @@ mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -71,19 +71,17 @@ bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -106,16 +104,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -137,16 +129,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -168,16 +154,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -199,16 +179,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -230,16 +204,10 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -261,16 +229,15 @@ add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -294,10 +261,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -323,10 +289,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -352,10 +317,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -381,10 +345,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -410,10 +373,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -439,10 +401,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -468,10 +429,9 @@ bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations @@ -541,7 +501,7 @@ ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -579,11 +539,11 @@ adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +585,9 @@ .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/c-source/build/mach-o/ct_inverse_mod_384-armv8.S b/c-source/build/mach-o/ct_inverse_mod_384-armv8.S --- a/c-source/build/mach-o/ct_inverse_mod_384-armv8.S +++ b/c-source/build/mach-o/ct_inverse_mod_384-armv8.S @@ -1,11 +1,11 @@ .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .align 5 -_ct_inverse_mod_383: -.long 3573752639 +_ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -47,14 +47,14 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| @@ -68,17 +68,17 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,10 +96,10 @@ smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -111,22 +111,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -138,22 +137,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -165,22 +163,21 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -192,22 +189,23 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] @@ -223,23 +221,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -251,23 +250,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -279,23 +279,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -307,23 +308,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 @@ -335,23 +337,24 @@ #ifdef __CHERI_PURE_CAPABILITY__ scvalue c0,csp,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ @@ -375,20 +378,22 @@ mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ scvalue c1,csp,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +404,60 @@ mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,7 +471,7 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -439,7 +479,7 @@ // see corresponding commentary in ctx_inverse_mod_384-x86_64... .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +487,7 @@ ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +502,33 @@ umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +543,25 @@ umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +573,41 @@ stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +615,11 @@ adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +628,15 @@ stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +656,27 @@ adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +696,27 @@ adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/c-source/build/mach-o/ct_is_square_mod_384-armv8.S b/c-source/build/mach-o/ct_is_square_mod_384-armv8.S --- a/c-source/build/mach-o/ct_is_square_mod_384-armv8.S +++ b/c-source/build/mach-o/ct_is_square_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +84,7 @@ ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s b/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s --- a/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s +++ b/c-source/build/mach-o/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm ___blst_platform_cap,4 .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .p2align 5 -_ct_inverse_mod_383: +_ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz L$ct_inverse_mod_383$1 + jnz L$ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -552,7 +592,7 @@ .p2align 5 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -773,7 +820,7 @@ .p2align 5 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -925,7 +987,7 @@ .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 diff --git a/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s b/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s --- a/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s +++ b/c-source/build/mach-o/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl _ctx_inverse_mod_383 -.private_extern _ctx_inverse_mod_383 +.globl _ctx_inverse_mod_384 +.private_extern _ctx_inverse_mod_384 .p2align 5 -_ctx_inverse_mod_383: +_ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -L$ct_inverse_mod_383$1: +L$ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax + movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -896,7 +936,7 @@ .p2align 5 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,63 +1046,65 @@ adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax - adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax - adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx - movq 16(%rsp),%rsi - adcq %rdi,%rax + adcq $0,%rax - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi adcq %rdi,%rbx - adcq %rdi,%rbp + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi adcq %rdi,%rcx - adcq %rdi,%rax + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx - movq %rdx,%rdi + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 + adcq %rsi,%rbx + adcq %rsi,%rbp + adcq %rsi,%rcx + adcq %rsi,%rax + adcq %rsi,%rdx - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq 16(%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) + #ifdef __SGX_LVI_HARDENING__ popq %r8 @@ -1071,7 +1118,7 @@ .p2align 5 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,13 +1144,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1115,19 +1168,22 @@ mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,13 +1199,18 @@ xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + andq %rdx,%r14 + negq %r14 + mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax addq %rbp,%r9 @@ -1161,13 +1222,15 @@ adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1189,12 +1253,11 @@ .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + andq %rdx,%rax + negq %rax + mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1587,7 +1655,7 @@ .p2align 5 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ xorq %r12,%r12 movq $1,%r13 -L$oop_53: +L$oop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz L$oop_53 + jnz L$oop_55 #ifdef __SGX_LVI_HARDENING__ diff --git a/c-source/build/mach-o/div3w-armv8.S b/c-source/build/mach-o/div3w-armv8.S --- a/c-source/build/mach-o/div3w-armv8.S +++ b/c-source/build/mach-o/div3w-armv8.S @@ -5,6 +5,7 @@ .align 5 _div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +40,7 @@ .align 5 _quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +78,7 @@ .align 5 _quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/c-source/build/mach-o/mul_mont_256-armv8.S b/c-source/build/mach-o/mul_mont_256-armv8.S --- a/c-source/build/mach-o/mul_mont_256-armv8.S +++ b/c-source/build/mach-o/mul_mont_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +197,7 @@ .align 5 _sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +298,7 @@ ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl _from_mont_256 @@ -305,7 +306,7 @@ .align 5 _from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +331,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -339,7 +340,7 @@ .align 5 _redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +375,7 @@ stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/mach-o/mul_mont_384-armv8.S b/c-source/build/mach-o/mul_mont_384-armv8.S --- a/c-source/build/mach-o/mul_mont_384-armv8.S +++ b/c-source/build/mach-o/mul_mont_384-armv8.S @@ -1,10 +1,11 @@ .text .globl _add_mod_384x384 +.private_extern _add_mod_384x384 .align 5 _add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -22,7 +23,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -82,10 +83,11 @@ .globl _sub_mod_384x384 +.private_extern _sub_mod_384x384 .align 5 _sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -103,7 +105,7 @@ ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -239,7 +241,7 @@ .align 5 _mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -312,7 +314,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -321,7 +323,7 @@ .align 5 _sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -397,7 +399,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -406,7 +408,7 @@ .align 5 _mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -438,7 +440,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -819,7 +821,7 @@ .align 5 _sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -856,7 +858,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -865,7 +867,7 @@ .align 5 _sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -921,7 +923,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1042,7 +1044,7 @@ .align 5 _sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1064,7 +1066,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1073,7 +1075,7 @@ .align 5 _redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1097,7 +1099,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1106,7 +1108,7 @@ .align 5 _from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1147,7 +1149,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1365,7 +1367,7 @@ .align 5 _mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1383,7 +1385,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1569,7 +1571,7 @@ .align 5 _mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1651,7 +1653,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1660,7 +1662,7 @@ .align 5 _sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1751,7 +1753,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1760,7 +1762,7 @@ .align 5 _sqr_mont_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1875,7 +1877,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2224,7 +2226,7 @@ .align 5 _sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2269,7 +2271,7 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2278,7 +2280,7 @@ .align 5 _sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2367,6 +2369,6 @@ ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/c-source/build/mach-o/sha256-armv8.S b/c-source/build/mach-o/sha256-armv8.S --- a/c-source/build/mach-o/sha256-armv8.S +++ b/c-source/build/mach-o/sha256-armv8.S @@ -43,9 +43,11 @@ .align 2 .align 2 .globl _blst_sha256_block_armv8 +.private_extern _blst_sha256_block_armv8 .align 6 _blst_sha256_block_armv8: + hint #34 Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -182,9 +184,11 @@ ret .globl _blst_sha256_block_data_order +.private_extern _blst_sha256_block_data_order .align 4 _blst_sha256_block_data_order: + hint #34 adrp x16,___blst_platform_cap@PAGE ldr w16,[x16,___blst_platform_cap@PAGEOFF] tst w16,#1 @@ -1034,6 +1038,7 @@ .align 4 _blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1062,6 +1067,7 @@ .align 4 _blst_sha256_bcopy: + hint #34 Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1075,6 +1081,7 @@ .align 4 _blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/c-source/src/cpuid.c b/c-source/src/cpuid.c --- a/c-source/src/cpuid.c +++ b/c-source/src/cpuid.c @@ -47,14 +47,14 @@ return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # elif defined(__SUNPRO_C) # pragma init(__blst_cpuid) # endif -#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) || defined(_M_ARM64EC) # if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) extern unsigned long getauxval(unsigned long type) __attribute__ ((weak)); @@ -105,10 +105,56 @@ return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # endif # endif +#endif + +#if defined(_WIN64) && defined(__BLST_DLL_MAIN__) +# define IsProcessorFeaturePresent mask_IsProcessorFeaturePresent +# define WIN32_LEAN_AND_MEAN +# include + +BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD dwReason, LPVOID lpvReserved) +{ + if (dwReason == DLL_PROCESS_ATTACH) { + DisableThreadLibraryCalls(hinstDLL); + __blst_cpuid(); + } + + return TRUE; + + (void)lpvReserved; +} + +# if defined(_MSC_VER) +/* + * Even though we don't have memcpy/memset anywhere, MSVC compiler + * generates calls to them as it recognizes corresponding patterns. + */ +#pragma function(memcpy) +void *memcpy(unsigned char *dst, const unsigned char *src, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = *src++; + + return ret; +} + +#pragma function(memset) +void *memset(unsigned char *dst, int c, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = (unsigned char)c; + + return ret; +} +# endif #endif diff --git a/c-source/src/ec_mult.h b/c-source/src/ec_mult.h --- a/c-source/src/ec_mult.h +++ b/c-source/src/ec_mult.h @@ -210,8 +210,7 @@ wval &= wmask; \ wval = booth_encode(wval, SZ); \ row_is_inf = ptype##_gather_booth_w##SZ(row, table, wval); \ - if (bits > 0) ptype##_add(sum, ret, row); \ - else ptype##_dadd(sum, ret, row, NULL); \ + ptype##_dadd(sum, ret, row, NULL); \ ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ ret_is_inf |= sum_is_inf; \ diff --git a/c-source/src/multi_scalar.c b/c-source/src/multi_scalar.c --- a/c-source/src/multi_scalar.c +++ b/c-source/src/multi_scalar.c @@ -7,9 +7,6 @@ #include "fields.h" #include "point.h" -/* - * Infinite point among inputs would be devastating. Shall we change it? - */ #define POINTS_TO_AFFINE_IMPL(prefix, ptype, bits, field) \ static void ptype##s_to_affine(ptype##_affine dst[], \ const ptype *const points[], size_t npoints) \ @@ -25,27 +22,36 @@ \ point = *points ? *points++ : point+1; \ acc = (vec##bits *)dst; \ - vec_copy(acc++, point->Z, sizeof(vec##bits)); \ - for (i = 1; i < delta; i++, acc++) \ - point = *points ? *points++ : point+1, \ - mul_##field(acc[0], acc[-1], point->Z); \ + vec_select(acc++, BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + for (i = 1; i < delta; i++, acc++) { \ + point = *points ? *points++ : point+1; \ + vec_select(acc[0], BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + mul_##field(acc[0], acc[0], acc[-1]); \ + } \ \ --acc; reciprocal_##field(acc[0], acc[0]); \ \ walkback = points-1, p = point, --delta, dst += delta; \ for (i = 0; i < delta; i++, acc--, dst--) { \ + bool_t is_inf = vec_is_zero(p->Z, sizeof(p->Z)); \ mul_##field(acc[-1], acc[-1], acc[0]); /* 1/Z */\ sqr_##field(ZZ, acc[-1]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[-1]); /* 1/Z^3 */\ - mul_##field(acc[-1], p->Z, acc[0]); \ + vec_select(acc[-1], BLS12_381_Rx.p, p->Z, sizeof(vec##bits), \ + is_inf); \ + mul_##field(acc[-1], acc[-1], acc[0]); \ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), is_inf); \ p = (p == *walkback) ? *--walkback : p-1; \ } \ sqr_##field(ZZ, acc[0]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[0]); /* 1/Z^3 */\ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), vec_is_zero(p->Z, sizeof(p->Z))); \ ++delta, dst += delta, npoints -= delta; \ } \ } \ @@ -74,18 +80,23 @@ #define SCRATCH_SZ(ptype) (sizeof(ptype)==sizeof(POINTonE1) ? 8192 : 4096) +/* The intermediate infinity points are encoded as [0, 0, 1]. */ + #define PRECOMPUTE_WBITS_IMPL(prefix, ptype, bits, field, one) \ static void ptype##_precompute_row_wbits(ptype row[], size_t wbits, \ const ptype##_affine *point) \ { \ size_t i, j, n = (size_t)1 << (wbits-1); \ + bool_t inf = vec_is_zero(point, sizeof(*point)); \ /* row[-1] is implicit infinity */\ vec_copy(&row[0], point, sizeof(*point)); /* row[0]=p*1 */\ vec_copy(&row[0].Z, one, sizeof(row[0].Z)); \ ptype##_double(&row[1], &row[0]); /* row[1]=p*(1+1) */\ + vec_select(&row[1].Z, one, &row[1].Z, sizeof(row[1].Z), inf); \ for (i = 2, j = 1; i < n; i += 2, j++) \ ptype##_add_affine(&row[i], &row[i-1], point), /* row[2]=p*(2+1) */\ - ptype##_double(&row[i+1], &row[j]); /* row[3]=p*(2+2) */\ + ptype##_double(&row[i+1], &row[j]), /* row[3]=p*(2+2) */\ + vec_select(&row[i+1].Z, one, &row[i+1].Z, sizeof(row[i+1].Z), inf); \ } /* row[4] ... */\ \ static void ptype##s_to_affine_row_wbits(ptype##_affine dst[], ptype src[], \ @@ -202,7 +213,7 @@ \ nbits -= window; \ z = is_zero(nbits); \ - wval = (get_wval_limb(scalar, nbits - (z^1), wbits + (z^1)) << z) & wmask; \ + wval = (get_wval_limb(scalar, nbits - (z^1), window + (z^1)) << z) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[0], row, wbits, wval); \ row += nwin; \ @@ -232,7 +243,7 @@ if (j == scratch_sz) \ ptype##s_accumulate(ret, scratch, j), j = 0; \ scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ - wval = (get_wval_limb(scalar, 0, wbits) << 1) & wmask; \ + wval = (get_wval_limb(scalar, 0, window) << 1) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[j], row, wbits, wval); \ } \ @@ -267,7 +278,14 @@ for (wbits=0; npoints>>=1; wbits++) ; - return wbits>12 ? wbits-3 : (wbits>4 ? wbits-2 : (wbits ? 2 : 1)); + if (wbits > 12) + return wbits - 3; + else if (wbits > 8) + return wbits - 2; + else if (wbits > 4) + return wbits - 1; + + return wbits ? 2 : 1; } #define DECLARE_PRIVATE_POINTXYZZ(ptype, bits) \ @@ -402,10 +420,11 @@ { \ if (npoints == 1) { \ prefix##_from_affine(ret, points[0]); \ - prefix##_mult(ret, ret, scalars[0], nbits); \ + ptype##_mult_w5(ret, ret, scalars[0], nbits); \ return; \ } \ - if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT) { \ + if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT && \ + npoints < 32) { \ ptype##_affine *table = alloca(npoints * sizeof(ptype##_affine) * 8); \ ptype##s_precompute_wbits(table, 4, points, npoints); \ ptype##s_mult_wbits(ret, table, 4, npoints, scalars, nbits, NULL); \ diff --git a/c-source/src/recip.c b/c-source/src/recip.c --- a/c-source/src/recip.c +++ b/c-source/src/recip.c @@ -62,20 +62,11 @@ TO_LIMB_T(0x39869507b587b120), TO_LIMB_T(0x23ba5c279c2895fb), TO_LIMB_T(0x58dd3db21a5d66bb), TO_LIMB_T(0xd0088f51cbff34d2) }; -#ifdef __BLST_NO_ASM__ -# define RRx4 BLS12_381_RR -#else - static const vec384 RRx4 = { /* (4<<768)%P */ - TO_LIMB_T(0x5f7e7cd070d107c2), TO_LIMB_T(0xec839a9ac49c13c8), - TO_LIMB_T(0x6933786f44f4ef0b), TO_LIMB_T(0xd6bf8b9c676be983), - TO_LIMB_T(0xd3adaaaa4dcefb06), TO_LIMB_T(0x12601bc1d82bc175) - }; -#endif union { vec768 x; vec384 r[2]; } temp; - ct_inverse_mod_383(temp.x, inp, BLS12_381_P, Px8); + ct_inverse_mod_384(temp.x, inp, BLS12_381_P, Px8); redc_mont_384(temp.r[0], temp.x, BLS12_381_P, p0); - mul_mont_384(temp.r[0], temp.r[0], RRx4, BLS12_381_P, p0); + mul_mont_384(temp.r[0], temp.r[0], BLS12_381_RR, BLS12_381_P, p0); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* sign goes straight to flt_reciprocal */ @@ -88,7 +79,6 @@ #else vec_copy(out, temp.r[0], sizeof(vec384)); #endif -#undef RRx4 } void blst_fp_inverse(vec384 out, const vec384 inp) diff --git a/c-source/src/vect.h b/c-source/src/vect.h --- a/c-source/src/vect.h +++ b/c-source/src/vect.h @@ -85,9 +85,7 @@ # define from_mont_384 fromx_mont_384 # define sgn0_pty_mont_384 sgn0x_pty_mont_384 # define sgn0_pty_mont_384x sgn0x_pty_mont_384x -# define ct_inverse_mod_383 ctx_inverse_mod_383 -#elif defined(__BLST_NO_ASM__) -# define ct_inverse_mod_383 ct_inverse_mod_384 +# define ct_inverse_mod_384 ctx_inverse_mod_384 #endif void mul_mont_sparse_256(vec256 ret, const vec256 a, const vec256 b, @@ -137,7 +135,7 @@ void lshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void rshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void div_by_2_mod_384(vec384 ret, const vec384 a, const vec384 p); -void ct_inverse_mod_383(vec768 ret, const vec384 inp, const vec384 mod, +void ct_inverse_mod_384(vec768 ret, const vec384 inp, const vec384 mod, const vec384 modx); void ct_inverse_mod_256(vec512 ret, const vec256 inp, const vec256 mod, const vec256 modx); @@ -407,7 +405,7 @@ #if defined(__INTEL_COMPILER) # pragma warning(disable:167) # pragma warning(disable:556) -#elif defined(__GNUC__) && !defined(__clang__) +#elif defined(__GNUC__) && !defined(__clang__) && (__STDC_VERSION__-0) < 202311 # pragma GCC diagnostic ignored "-Wpedantic" #elif defined(_MSC_VER) # pragma warning(disable: 4127 4189) diff --git a/hsblst.cabal b/hsblst.cabal --- a/hsblst.cabal +++ b/hsblst.cabal @@ -1,11 +1,11 @@ cabal-version: 3.0 --- This file has been generated from package.yaml by hpack version 0.34.4. +-- This file has been generated from package.yaml by hpack version 0.37.0. -- -- see: https://github.com/sol/hpack name: hsblst -version: 0.0.4 +version: 0.0.5 synopsis: Haskell bindings to BLST description: HsBLST is low-level Haskell bindings and a high-level interface to [BLST](https://github.com/supranational/blst) -- a multilingual BLS12-381 signature library. category: Cryptography @@ -19,7 +19,9 @@ build-type: Simple tested-with: GHC == 9.8.4 - , GHC == 9.10.1 + , GHC == 9.10.3 + , GHC == 9.12.2 + , GHC == 9.12.3 extra-source-files: c-source/build/coff/add_mod_256-x86_64.s c-source/build/coff/add_mod_384-x86_64.s @@ -180,6 +182,7 @@ EmptyCase FlexibleContexts FlexibleInstances + ForeignFunctionInterface GADTs GeneralizedNewtypeDeriving ImportQualifiedPost @@ -216,12 +219,12 @@ c-sources: c-source/src/server.c build-depends: - base >=4.17 && <4.21 - , deepseq >=1.4.5.0 && <1.6.0 - , memory >=0.16.0 && <0.19 + base >=4.17 && <4.22 + , deepseq >=1.4.5 && <1.6 + , ram >=0.19 && <0.22 + default-language: Haskell2010 if impl(ghc >= 9.2) ghc-options: -Wno-missing-kind-signatures - default-language: Haskell2010 asm-sources: c-source/build/assembly.S build-tool-depends: c2hs:c2hs @@ -257,6 +260,7 @@ EmptyCase FlexibleContexts FlexibleInstances + ForeignFunctionInterface GADTs GeneralizedNewtypeDeriving ImportQualifiedPost @@ -291,16 +295,16 @@ build-tool-depends: tasty-discover:tasty-discover build-depends: - base >=4.17 && <4.21 - , base16-bytestring ==1.0.* + base >=4.17 && <4.22 + , base16-bytestring >=1 && <1.1 , bytestring >=0.10.12.1 && <0.13 , hsblst - , memory >=0.16.0 && <0.19 + , ram >=0.19 && <0.22 , tasty >=1.4.2.1 && <1.6 , tasty-hunit >=0.10.0.3 && <0.11 - , text >=1.2.5.0 && <1.3 || >=2.0.1 && <2.1 || >=2.1 && <2.2 + , text >=1.2.5 && <1.3 || >=2.0.1 && <2.1 || >=2.1 && <2.2 + default-language: Haskell2010 if impl(ghc >= 9.2) ghc-options: -Wno-missing-kind-signatures if impl(ghc < 9.4) ghc-options: -eventlog - default-language: Haskell2010