From 6c94aa73c885cd49f82ef4c485918b42d8027b58 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 17:07:21 -0500 Subject: [PATCH] blst v0.3.17 --- blst_src/README.md | 6 +- blst_src/build/cheri/add_mod_256-armv8.S | 22 + blst_src/build/cheri/add_mod_384-armv8.S | 88 +- .../build/cheri/ct_inverse_mod_256-armv8.S | 179 ++-- .../build/cheri/ct_inverse_mod_384-armv8.S | 426 ++++---- .../build/cheri/ct_is_square_mod_384-armv8.S | 22 +- blst_src/build/cheri/div3w-armv8.S | 21 + blst_src/build/cheri/mul_mont_256-armv8.S | 31 +- blst_src/build/cheri/mul_mont_384-armv8.S | 542 +--------- blst_src/build/cheri/sha256-armv8.S | 23 + blst_src/build/coff/add_mod_256-armv8.S | 4 + blst_src/build/coff/add_mod_384-armv8.S | 70 +- .../build/coff/ct_inverse_mod_256-armv8.S | 161 ++- .../build/coff/ct_inverse_mod_384-armv8.S | 398 ++++--- .../build/coff/ct_is_square_mod_384-armv8.S | 4 +- .../build/coff/ctq_inverse_mod_384-x86_64.s | 357 ++++--- .../build/coff/ctx_inverse_mod_384-x86_64.s | 592 ++++++----- blst_src/build/coff/div3w-armv8.S | 3 + blst_src/build/coff/mul_mont_256-armv8.S | 13 +- blst_src/build/coff/mul_mont_384-armv8.S | 528 +--------- blst_src/build/coff/mulq_mont_384-x86_64.s | 803 -------------- blst_src/build/coff/mulx_mont_256-x86_64.s | 34 +- blst_src/build/coff/mulx_mont_384-x86_64.s | 326 +----- blst_src/build/coff/sha256-armv8.S | 5 + blst_src/build/elf/add_mod_256-armv8.S | 22 + blst_src/build/elf/add_mod_384-armv8.S | 88 +- blst_src/build/elf/ct_inverse_mod_256-armv8.S | 179 ++-- blst_src/build/elf/ct_inverse_mod_384-armv8.S | 426 ++++---- .../build/elf/ct_is_square_mod_384-armv8.S | 22 +- .../build/elf/ctq_inverse_mod_384-x86_64.s | 335 +++--- .../build/elf/ctx_inverse_mod_384-x86_64.s | 572 +++++----- blst_src/build/elf/div3w-armv8.S | 21 + blst_src/build/elf/mul_mont_256-armv8.S | 31 +- blst_src/build/elf/mul_mont_384-armv8.S | 542 +--------- blst_src/build/elf/mulq_mont_384-x86_64.s | 762 -------------- blst_src/build/elf/mulx_mont_256-x86_64.s | 34 +- blst_src/build/elf/mulx_mont_384-x86_64.s | 283 +---- blst_src/build/elf/sha256-armv8.S | 23 + blst_src/build/mach-o/add_mod_256-armv8.S | 4 + blst_src/build/mach-o/add_mod_384-armv8.S | 70 +- .../build/mach-o/ct_inverse_mod_256-armv8.S | 161 ++- .../build/mach-o/ct_inverse_mod_384-armv8.S | 392 ++++--- .../build/mach-o/ct_is_square_mod_384-armv8.S | 4 +- .../build/mach-o/ctq_inverse_mod_384-x86_64.s | 319 +++--- .../build/mach-o/ctx_inverse_mod_384-x86_64.s | 552 +++++----- blst_src/build/mach-o/div3w-armv8.S | 3 + blst_src/build/mach-o/mul_mont_256-armv8.S | 13 +- blst_src/build/mach-o/mul_mont_384-armv8.S | 524 +--------- blst_src/build/mach-o/mulq_mont_384-x86_64.s | 762 -------------- blst_src/build/mach-o/mulx_mont_256-x86_64.s | 34 +- blst_src/build/mach-o/mulx_mont_384-x86_64.s | 283 +---- blst_src/build/mach-o/sha256-armv8.S | 5 + blst_src/build/win64/add_mod_256-armv8.asm | 4 + blst_src/build/win64/add_mod_384-armv8.asm | 70 +- blst_src/build/win64/blst.def | 2 + .../build/win64/ct_inverse_mod_256-armv8.asm | 161 ++- .../build/win64/ct_inverse_mod_384-armv8.asm | 382 ++++--- .../win64/ct_is_square_mod_384-armv8.asm | 4 +- .../win64/ctq_inverse_mod_384-x86_64.asm | 361 ++++--- .../win64/ctx_inverse_mod_384-x86_64.asm | 596 ++++++----- blst_src/build/win64/div3w-armv8.asm | 3 + blst_src/build/win64/mul_mont_256-armv8.asm | 13 +- blst_src/build/win64/mul_mont_384-armv8.asm | 524 +--------- blst_src/build/win64/mulq_mont_256-x86_64.asm | 2 + blst_src/build/win64/mulq_mont_384-x86_64.asm | 987 ++---------------- blst_src/build/win64/mulx_mont_256-x86_64.asm | 36 +- blst_src/build/win64/mulx_mont_384-x86_64.asm | 330 +----- blst_src/build/win64/sha256-armv8.asm | 5 + blst_src/cpuid.c | 52 +- blst_src/ec_mult.h | 3 +- blst_src/exports.c | 55 +- blst_src/keygen.c | 2 +- blst_src/multi_scalar.c | 93 +- blst_src/no_asm.h | 52 - blst_src/recip.c | 16 +- blst_src/vect.h | 10 +- internal/blst/blst.go | 137 ++- internal/blst/blst.h | 10 +- internal/blst/blst_aux.h | 3 + 79 files changed, 4876 insertions(+), 10161 deletions(-) diff --git a/blst_src/README.md b/blst_src/README.md index 48abe90e..0bb31ce3 100644 --- a/blst_src/README.md +++ b/blst_src/README.md @@ -1,5 +1,5 @@ All files in this folder contain source files copied from the BLST repo https://github.com/supranational/blst, -specifically from the tagged version `v0.3.14`. +specifically from the tagged version `v0.3.17`. Copyright Supranational LLC Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -18,8 +18,8 @@ The folder contains: - all `/build` (assembly generated files). - this `README` file. -To upgrade the BLST version: -- [ ] audit all BLST updates, with focus on `/src`: https://github.com/supranational/blst/compare/v0.3.14... +Follow these steps from the root of the package to upgrade the BLST version: +- [ ] audit all BLST updates, with focus on `/src`: https://github.com/supranational/blst/compare/v0.3.17... - [ ] delete all files in this folder `./blst_src/` but `blst_src.c` and `README.md`. - [ ] delete all files in `./internal/blst/` but `non_cgo.go`. - [ ] open BLST repository on the new version. diff --git a/blst_src/build/cheri/add_mod_256-armv8.S b/blst_src/build/cheri/add_mod_256-armv8.S index 9db59ce9..21e532c0 100644 --- a/blst_src/build/cheri/add_mod_256-armv8.S +++ b/blst_src/build/cheri/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[c1] ldp x12,x13,[c2] @@ -39,6 +48,7 @@ add_mod_256: .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[c1] ldp x14,x15,[c1,#16] @@ -88,6 +98,7 @@ mul_by_3_mod_256: .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -126,6 +137,7 @@ lshift_mod_256: .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -377,3 +389,13 @@ sub_n_check_mod_256: ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/add_mod_384-armv8.S b/blst_src/build/cheri/add_mod_384-armv8.S index 1f498079..d236bbbe 100644 --- a/blst_src/build/cheri/add_mod_384-armv8.S +++ b/blst_src/build/cheri/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ add_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ __add_mod_384_ab_are_loaded: .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ add_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ add_mod_384x: .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ rshift_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ __rshift_mod_384: .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ div_by_2_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ div_by_2_mod_384: .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ lshift_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ __lshift_mod_384: .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ mul_by_3_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ mul_by_3_mod_384: .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ mul_by_8_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ mul_by_8_mod_384: .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ mul_by_3_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ mul_by_3_mod_384x: .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ mul_by_8_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ mul_by_8_mod_384x: .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ cneg_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ cneg_mod_384: .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ sub_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ __sub_mod_384: .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ sub_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ sub_mod_384x: .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ mul_by_1_plus_i_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ mul_by_1_plus_i_mod_384x: .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -672,6 +681,7 @@ sgn0_pty_mod_384: .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -753,14 +763,14 @@ sgn0_pty_mod_384x: .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + ld1 {v0.2d, v1.2d}, [c1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + ld1 {v3.2d, v4.2d}, [c2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [c0] + st1 {v0.2d, v1.2d}, [c0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ vec_select_32: .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ vec_select_48: .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ vec_select_96: .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ vec_select_192: .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ vec_select_144: .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ vec_select_288: .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ vec_prefetch: .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [c0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ vec_is_zero_16x: .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [c0], #16 ld1 {v1.2d}, [c1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ vec_is_equal_16x: csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_inverse_mod_256-armv8.S b/blst_src/build/cheri/ct_inverse_mod_256-armv8.S index c451f941..02d12cb6 100644 --- a/blst_src/build/cheri/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/cheri/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-10*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -20,6 +28,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add c1,csp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +48,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[c0,#8*8] // initialize |u| with |f0| @@ -48,18 +57,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add c0,c0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [c0,#8*9] // initialize |v| with |f1| + str x12, [c0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +80,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [c1,#8*8] // |u| - ldr x9, [c1,#8*13] // |v| + ldr x9, [c1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [c0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*5] - stp x5, x5, [c0,#8*7] + stp x4, x5, [c0,#8*4] + stp x5, x5, [c0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [c0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*10] + stp x4, x5, [c0,#8*10] stp x5, x5, [c0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +113,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +138,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +163,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +188,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +213,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +238,24 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - + asr x24, x24, #63 + str x24, [c0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [c0,#8*4] + stp x24, x24, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +270,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +298,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +326,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +354,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +382,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +410,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +438,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +510,7 @@ ct_inverse_mod_256: ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldr c29, [csp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +548,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [c1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [c1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [c1,#8*2+104] + ldp x10, x11, [c1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [c1,#8*4+104] + ldr x23, [c1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +594,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [c1,#8*18] // load rest of |v| + ldr x5, [c1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [c1,#8*20] + ldp x6, x7, [c1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +847,13 @@ __inner_loop_62_256: ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_inverse_mod_384-armv8.S b/blst_src/build/cheri/ct_inverse_mod_384-armv8.S index 8d132974..160e727e 100644 --- a/blst_src/build/cheri/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/cheri/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -22,6 +30,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add c1,csp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +54,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[c0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [c0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [c0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [c1,#8*12] // |u| - ldr x8, [c1,#8*18] // |v| + ldr x8, [c1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +105,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [c0,#8*12] + stp x3, x4, [c0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*14] stp x5, x5, [c0,#8*16] + stp x5, x5, [c0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [c0,#8*6] stp x27, x27, [c0,#8*8] stp x27, x27, [c0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +377,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [c0,#8*0] str x9, [c0,#8*6] @@ -375,20 +387,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add c0,c0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [c1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +413,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp c0, c15, [csp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr c30, [c29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [c15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [c15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [c15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [c15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [c0,#8*6] adcs x7, x7, x13 stp x5, x6, [c0,#8*8] @@ -431,15 +480,15 @@ ct_inverse_mod_383: ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [c1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [c1,#8*2+96] @@ -447,6 +496,7 @@ __smul_383x63: ldp x7, x8, [c1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [c1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +511,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [c1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [c1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [c1,#8*2+144] + ldp x11, x12, [c1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [c1,#8*4+144] + ldp x13, x14, [c1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [c1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +552,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +582,41 @@ __smul_383x63: stp x5, x6, [c0,#8*2] adcs x27, x27, x28 stp x7, x27, [c0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [c1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [c1,#8*26] - ldp x7, x8, [c1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [c1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [c1,#8*28] + and x25, x25, x20 + ldp x7, x8, [c1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +624,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +637,15 @@ __smul_767x63_tail: stp x3, x4, [c0,#8*6] adcs x7, x7, x28 stp x5, x6, [c0,#8*8] - adc x8, x8, x28 - stp x7, x8, [c0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [c0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [c1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [c1,#8*2+0] @@ -605,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [c1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [c1,#8*2+48] @@ -643,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +764,7 @@ __smul_383_n_shift_by_62: sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +857,13 @@ __inner_loop_62: ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_is_square_mod_384-armv8.S b/blst_src/build/cheri/ct_is_square_mod_384-armv8.S index 269c300a..183a982e 100644 --- a/blst_src/build/cheri/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/cheri/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ct_is_square_mod_384: ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ __inner_loop_48: ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/div3w-armv8.S b/blst_src/build/cheri/div3w-armv8.S index 64149096..8e5cd764 100644 --- a/blst_src/build/cheri/div3w-armv8.S +++ b/blst_src/build/cheri/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[c0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ div_3_limbs: .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[c1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ quot_rem_128: .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[c1] ldr x8,[c0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ quot_rem_64: ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/mul_mont_256-armv8.S b/blst_src/build/cheri/mul_mont_256-armv8.S index 2dd8f459..3c94f7dc 100644 --- a/blst_src/build/cheri/mul_mont_256-armv8.S +++ b/blst_src/build/cheri/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ mul_mont_sparse_256: .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ sqr_mont_sparse_256: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ sqr_mont_sparse_256: .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -330,7 +339,7 @@ from_mont_256: stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ from_mont_256: .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -374,7 +383,7 @@ redc_mont_256: stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ __mul_by_1_mont_256: ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/mul_mont_384-armv8.S b/blst_src/build/cheri/mul_mont_384-armv8.S index 28e669e6..b475ed61 100644 --- a/blst_src/build/cheri/mul_mont_384-armv8.S +++ b/blst_src/build/cheri/mul_mont_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 @@ -5,7 +13,7 @@ .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -23,7 +31,7 @@ add_mod_384x384: ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -87,7 +95,7 @@ __add_mod_384x384: .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -105,7 +113,7 @@ sub_mod_384x384: ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -241,7 +249,7 @@ __sub_mod_384: .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -314,7 +322,7 @@ mul_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -323,7 +331,7 @@ mul_mont_384x: .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -399,7 +407,7 @@ sqr_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -408,7 +416,7 @@ sqr_mont_384x: .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -440,7 +448,7 @@ mul_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -821,7 +829,7 @@ __mul_mont_384: .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -858,7 +866,7 @@ sqr_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -867,7 +875,7 @@ sqr_mont_384: .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -923,7 +931,7 @@ sqr_n_mul_mont_383: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1044,7 +1052,7 @@ __sqr_384: .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1074,7 @@ sqr_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1075,7 +1083,7 @@ sqr_384: .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1107,7 @@ redc_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1108,7 +1116,7 @@ redc_mont_384: .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1157,7 @@ from_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1367,7 +1375,7 @@ __redc_tail_mont_384: .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1393,7 @@ mul_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1571,7 +1579,7 @@ __mul_384: .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1661,7 @@ mul_382x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1662,7 +1670,7 @@ mul_382x: .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1753,480 +1761,16 @@ sqr_382x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,%function -.align 5 -sqr_mont_382x: - .inst 0xd503233f - stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! - add c29,csp,#0 - stp c19,c20,[csp,#2*__SIZEOF_POINTER__] - stp c21,c22,[csp,#4*__SIZEOF_POINTER__] - stp c23,c24,[csp,#6*__SIZEOF_POINTER__] - stp c25,c26,[csp,#8*__SIZEOF_POINTER__] - stp c27,c28,[csp,#10*__SIZEOF_POINTER__] - stp c3,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub csp,csp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[c1] - ldp x13,x14,[c1,#16] - ldp x15,x16,[c1,#32] - - ldp x17,x20,[c1,#48] - ldp x21,x22,[c1,#64] - ldp x23,x24,[c1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[csp] - stp x7,x8,[csp,#16] - stp x9,x10,[csp,#32] - stp x19,x20,[csp,#48] - stp x21,x22,[csp,#64] - stp x23,x24,[csp,#80] - str x25,[csp,#96] - - ldp x5,x6,[c2] - ldp x7,x8,[c2,#16] - ldp x9,x10,[c2,#32] - - add c2,c1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[c2,#48] - stp x21,x22,[c2,#64] - stp x23,x24,[c2,#80] - - ldp x11,x12,[csp] - ldr x17,[csp,#48] - ldp x13,x14,[csp,#16] - ldp x15,x16,[csp,#32] - - add c2,csp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr c30,[c29,#__SIZEOF_POINTER__] - - ldr x25,[csp,#96] // account for sign from a->re - a->im - ldp x19,x20,[csp] - ldp x21,x22,[csp,#16] - ldp x23,x24,[csp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[c2] - stp x13,x14,[c2,#16] - stp x15,x16,[c2,#32] - - add csp,csp,#112 - ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] - ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] - ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] - ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] - ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] - ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf - ret -.size sqr_mont_382x,.-sqr_mont_382x - -.type __mul_mont_383_nonred,%function -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[c2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp c4,c2,[c29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - - ret -.size __mul_mont_383_nonred,.-__mul_mont_383_nonred - .globl sgn0_pty_mont_384 .hidden sgn0_pty_mont_384 .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1815,7 @@ sgn0_pty_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2280,7 +1824,7 @@ sgn0_pty_mont_384: .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1913,16 @@ sgn0_pty_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/sha256-armv8.S b/blst_src/build/cheri/sha256-armv8.S index 53d011ec..7b018952 100644 --- a/blst_src/build/cheri/sha256-armv8.S +++ b/blst_src/build/cheri/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -47,6 +55,7 @@ .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -187,6 +196,7 @@ blst_sha256_block_armv8: .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp c16,__blst_platform_cap ldr w16,[c16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1036,6 +1046,7 @@ blst_sha256_block_data_order: .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1075,7 @@ blst_sha256_emit: .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[c1],#1 sub x2,x2,#1 @@ -1077,9 +1089,20 @@ blst_sha256_bcopy: .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] stp x4,x5,[c0] stp x6,x7,[c0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/coff/add_mod_256-armv8.S b/blst_src/build/coff/add_mod_256-armv8.S index 27b64ef4..07495153 100644 --- a/blst_src/build/coff/add_mod_256-armv8.S +++ b/blst_src/build/coff/add_mod_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -43,6 +44,7 @@ add_mod_256: .endef .p2align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -94,6 +96,7 @@ mul_by_3_mod_256: .endef .p2align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -134,6 +137,7 @@ lshift_mod_256: .endef .p2align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/coff/add_mod_384-armv8.S b/blst_src/build/coff/add_mod_384-armv8.S index c1709000..65275894 100644 --- a/blst_src/build/coff/add_mod_384-armv8.S +++ b/blst_src/build/coff/add_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -27,7 +27,7 @@ add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -77,7 +77,7 @@ __add_mod_384_ab_are_loaded: .endef .p2align 5 add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +105,7 @@ add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -116,7 +116,7 @@ add_mod_384x: .endef .p2align 5 rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -143,7 +143,7 @@ rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -182,7 +182,7 @@ __rshift_mod_384: .endef .p2align 5 div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -206,7 +206,7 @@ div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -217,7 +217,7 @@ div_by_2_mod_384: .endef .p2align 5 lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -244,7 +244,7 @@ lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -286,7 +286,7 @@ __lshift_mod_384: .endef .p2align 5 mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -316,7 +316,7 @@ mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -327,7 +327,7 @@ mul_by_3_mod_384: .endef .p2align 5 mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -353,7 +353,7 @@ mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -364,7 +364,7 @@ mul_by_8_mod_384: .endef .p2align 5 mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -409,7 +409,7 @@ mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -420,7 +420,7 @@ mul_by_3_mod_384x: .endef .p2align 5 mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -457,7 +457,7 @@ mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -468,7 +468,7 @@ mul_by_8_mod_384x: .endef .p2align 5 cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -510,7 +510,7 @@ cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -521,7 +521,7 @@ cneg_mod_384: .endef .p2align 5 sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -541,7 +541,7 @@ sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -588,7 +588,7 @@ __sub_mod_384: .endef .p2align 5 sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -616,7 +616,7 @@ sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -627,7 +627,7 @@ sub_mod_384x: .endef .p2align 5 mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -660,7 +660,7 @@ mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -671,6 +671,7 @@ mul_by_1_plus_i_mod_384x: .endef .p2align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -710,6 +711,7 @@ sgn0_pty_mod_384: .endef .p2align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -793,14 +795,14 @@ sgn0_pty_mod_384x: .endef .p2align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl vec_select_48 @@ -810,6 +812,7 @@ vec_select_32: .endef .p2align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -827,6 +830,7 @@ vec_select_48: .endef .p2align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -850,6 +854,7 @@ vec_select_96: .endef .p2align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -885,6 +890,7 @@ vec_select_192: .endef .p2align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -914,6 +920,7 @@ vec_select_144: .endef .p2align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -961,6 +968,7 @@ vec_select_288: .endef .p2align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -1003,6 +1011,7 @@ vec_prefetch: .endef .p2align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -1030,6 +1039,7 @@ vec_is_zero_16x: .endef .p2align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/coff/ct_inverse_mod_256-armv8.S b/blst_src/build/coff/ct_inverse_mod_256-armv8.S index b8ac9bf7..90d01408 100644 --- a/blst_src/build/coff/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/coff/ct_inverse_mod_256-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -41,7 +42,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -50,18 +51,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -73,29 +74,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -108,25 +107,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -139,25 +132,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -170,25 +157,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -201,25 +182,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -232,25 +207,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -263,25 +232,24 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -296,21 +264,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -325,21 +292,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -354,21 +320,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -383,21 +348,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -412,21 +376,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -441,21 +404,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -470,16 +432,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -543,7 +504,7 @@ ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -583,11 +544,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -631,9 +592,9 @@ __smul_256x63: .p2align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/blst_src/build/coff/ct_inverse_mod_384-armv8.S b/blst_src/build/coff/ct_inverse_mod_384-armv8.S index aecb4ea7..4a94a0a8 100644 --- a/blst_src/build/coff/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/coff/ct_inverse_mod_384-armv8.S @@ -1,13 +1,13 @@ .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; +.def ct_inverse_mod_384; .type 32; .endef .p2align 5 -ct_inverse_mod_383: -.long 3573752639 +ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -24,6 +24,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -47,40 +48,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -98,266 +99,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -367,7 +371,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -377,20 +381,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -401,25 +407,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -433,17 +474,17 @@ ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.def __smul_383x63; +.def __smul_384x63; .type 32; .endef .p2align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -451,6 +492,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -465,28 +507,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -501,21 +548,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -527,43 +578,43 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.def __smul_767x63_tail; +.def __smul_768x63_tail; .type 32; .endef .p2align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -571,10 +622,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -583,17 +635,17 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.def __smul_383_n_shift_by_62; +.def __smul_384_n_shift_by_62; .type 32; .endef .p2align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -613,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -651,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/coff/ct_is_square_mod_384-armv8.S b/blst_src/build/coff/ct_is_square_mod_384-armv8.S index 07f3d9f0..ccf81578 100644 --- a/blst_src/build/coff/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/coff/ct_is_square_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -86,7 +86,7 @@ ct_is_square_mod_384: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s b/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s index d00335b9..b86dada5 100644 --- a/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s @@ -1,16 +1,16 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; .scl 2; .type 32; .endef +.def ct_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ct_inverse_mod_383: +ct_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ct_inverse_mod_383: +.LSEH_begin_ct_inverse_mod_384: movq %rcx,%rdi @@ -19,7 +19,7 @@ ct_inverse_mod_383: movq %r9,%rcx #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp @@ -35,7 +35,7 @@ ct_inverse_mod_383: subq $1112,%rsp -.LSEH_body_ct_inverse_mod_383: +.LSEH_body_ct_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -82,7 +82,7 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -90,10 +90,10 @@ ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -106,19 +106,19 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -135,6 +135,7 @@ ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -145,13 +146,14 @@ ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -162,14 +164,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -177,12 +179,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -193,14 +195,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -208,12 +210,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -224,14 +226,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -239,12 +241,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -255,14 +257,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -270,19 +272,17 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -293,14 +293,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -308,12 +308,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -324,14 +324,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -339,12 +339,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -355,14 +355,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -370,12 +370,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -386,14 +386,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -401,12 +401,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -417,14 +417,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -432,12 +432,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -461,16 +461,16 @@ ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -493,37 +493,77 @@ ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -540,7 +580,7 @@ ct_inverse_mod_383: leaq 48(%r8),%rsp -.LSEH_epilogue_ct_inverse_mod_383: +.LSEH_epilogue_ct_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -554,10 +594,10 @@ ct_inverse_mod_383: .byte 0xf3,0xc3 #endif -.LSEH_end_ct_inverse_mod_383: -.def __smulq_767x63; .scl 3; .type 32; .endef +.LSEH_end_ct_inverse_mod_384: +.def __smulq_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_767x63: +__smulq_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -566,6 +606,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -574,7 +615,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -585,16 +626,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -620,14 +665,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -730,39 +775,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -774,9 +821,9 @@ __smulq_767x63: .byte 0xf3,0xc3 #endif -.def __smulq_383x63; .scl 3; .type 32; .endef +.def __smulq_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_383x63: +__smulq_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -785,6 +832,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -800,16 +848,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -831,10 +883,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -842,13 +895,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -864,16 +919,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -895,17 +954,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -913,6 +974,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -924,9 +986,9 @@ __smulq_383x63: .byte 0xf3,0xc3 #endif -.def __smulq_383_n_shift_by_62; .scl 3; .type 32; .endef +.def __smulq_384_n_shift_by_62; .scl 3; .type 32; .endef .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1236,28 +1303,28 @@ __inner_loop_62: .section .pdata .p2align 2 -.rva .LSEH_begin_ct_inverse_mod_383 -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_prologue +.rva .LSEH_begin_ct_inverse_mod_384 +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_prologue -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_body +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_body -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_end_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_end_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ct_inverse_mod_383_prologue: +.LSEH_info_ct_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ct_inverse_mod_383_body: +.LSEH_info_ct_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1270,7 +1337,7 @@ __inner_loop_62: .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ct_inverse_mod_383_epilogue: +.LSEH_info_ct_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s b/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s index ed3002a4..060940a5 100644 --- a/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s @@ -1,22 +1,22 @@ .text -.globl ctx_inverse_mod_383 +.globl ctx_inverse_mod_384 -.def ctx_inverse_mod_383; .scl 2; .type 32; .endef +.def ctx_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ctx_inverse_mod_383: +.LSEH_begin_ctx_inverse_mod_384: movq %rcx,%rdi movq %rdx,%rsi movq %r8,%rdx movq %r9,%rcx -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp pushq %rbx @@ -31,7 +31,7 @@ ct_inverse_mod_383$1: subq $1112,%rsp -.LSEH_body_ctx_inverse_mod_383: +.LSEH_body_ctx_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -81,7 +81,7 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -89,10 +89,10 @@ ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -105,19 +105,19 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -134,6 +134,7 @@ ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -144,13 +145,14 @@ ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -161,14 +163,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -176,12 +178,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -192,14 +194,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -207,12 +209,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -223,14 +225,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -238,12 +240,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -254,14 +256,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -269,12 +271,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -285,14 +287,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -300,12 +302,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -316,14 +318,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -331,12 +333,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -347,14 +349,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -362,12 +364,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -378,14 +380,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -393,12 +395,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -409,14 +411,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -424,12 +426,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -440,14 +442,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -455,19 +457,17 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -478,14 +478,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -493,12 +493,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -509,14 +509,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -524,12 +524,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -540,14 +540,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -555,12 +555,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -571,14 +571,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -586,12 +586,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -602,14 +602,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -617,12 +617,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -633,14 +633,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -648,12 +648,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -664,14 +664,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -679,12 +679,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -710,12 +710,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -741,12 +741,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -772,12 +772,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -803,21 +803,21 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -834,40 +834,80 @@ ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -884,7 +924,7 @@ ct_inverse_mod_383$1: leaq 48(%r8),%rsp -.LSEH_epilogue_ctx_inverse_mod_383: +.LSEH_epilogue_ctx_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -898,10 +938,10 @@ ct_inverse_mod_383$1: .byte 0xf3,0xc3 #endif -.LSEH_end_ctx_inverse_mod_383: -.def __smulx_767x63; .scl 3; .type 32; .endef +.LSEH_end_ctx_inverse_mod_384: +.def __smulx_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_767x63: +__smulx_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -910,6 +950,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -918,7 +959,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -928,37 +969,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -993,7 +1038,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1005,62 +1050,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1072,9 +1119,9 @@ __smulx_767x63: .byte 0xf3,0xc3 #endif -.def __smulx_383x63; .scl 3; .type 32; .endef +.def __smulx_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_383x63: +__smulx_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0+0(%rsi),%r8 @@ -1083,6 +1130,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1098,12 +1146,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1117,19 +1170,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1145,12 +1201,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1163,13 +1224,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1177,6 +1240,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1188,13 +1252,12 @@ __smulx_383x63: .byte 0xf3,0xc3 #endif -.def __smulx_383_n_shift_by_31; .scl 3; .type 32; .endef +.def __smulx_384_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1578,9 +1646,9 @@ __inner_loop_31: #endif -.def __tail_loop_53; .scl 3; .type 32; .endef +.def __tail_loop_55; .scl 3; .type 32; .endef .p2align 5 -__tail_loop_53: +__tail_loop_55: .byte 0xf3,0x0f,0x1e,0xfa movq $1,%rdx @@ -1588,7 +1656,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1615,7 +1683,7 @@ __tail_loop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1629,28 +1697,28 @@ __tail_loop_53: .section .pdata .p2align 2 -.rva .LSEH_begin_ctx_inverse_mod_383 -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_prologue +.rva .LSEH_begin_ctx_inverse_mod_384 +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_prologue -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_body +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_body -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_end_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_end_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ctx_inverse_mod_383_prologue: +.LSEH_info_ctx_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ctx_inverse_mod_383_body: +.LSEH_info_ctx_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1663,7 +1731,7 @@ __tail_loop_53: .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ctx_inverse_mod_383_epilogue: +.LSEH_info_ctx_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/blst_src/build/coff/div3w-armv8.S b/blst_src/build/coff/div3w-armv8.S index fd694935..eaa741f4 100644 --- a/blst_src/build/coff/div3w-armv8.S +++ b/blst_src/build/coff/div3w-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -43,6 +44,7 @@ div_3_limbs: .endef .p2align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -82,6 +84,7 @@ quot_rem_128: .endef .p2align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/blst_src/build/coff/mul_mont_256-armv8.S b/blst_src/build/coff/mul_mont_256-armv8.S index 1b256a8a..ce49bae3 100644 --- a/blst_src/build/coff/mul_mont_256-armv8.S +++ b/blst_src/build/coff/mul_mont_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -200,7 +201,7 @@ mul_mont_sparse_256: .endef .p2align 5 sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -301,7 +302,7 @@ sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl from_mont_256 @@ -311,7 +312,7 @@ sqr_mont_sparse_256: .endef .p2align 5 from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -336,7 +337,7 @@ from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -347,7 +348,7 @@ from_mont_256: .endef .p2align 5 redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -382,7 +383,7 @@ redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/mul_mont_384-armv8.S b/blst_src/build/coff/mul_mont_384-armv8.S index a5e1cfef..79896c2b 100644 --- a/blst_src/build/coff/mul_mont_384-armv8.S +++ b/blst_src/build/coff/mul_mont_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -93,7 +93,7 @@ __add_mod_384x384: .endef .p2align 5 sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -111,7 +111,7 @@ sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -255,7 +255,7 @@ __sub_mod_384: .endef .p2align 5 mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -328,7 +328,7 @@ mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -339,7 +339,7 @@ mul_mont_384x: .endef .p2align 5 sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -415,7 +415,7 @@ sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -426,7 +426,7 @@ sqr_mont_384x: .endef .p2align 5 mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -458,7 +458,7 @@ mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -843,7 +843,7 @@ __mul_mont_384: .endef .p2align 5 sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -880,7 +880,7 @@ sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -891,7 +891,7 @@ sqr_mont_384: .endef .p2align 5 sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -947,7 +947,7 @@ sqr_n_mul_mont_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .def __sqr_384; @@ -1072,7 +1072,7 @@ __sqr_384: .endef .p2align 5 sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1094,7 +1094,7 @@ sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1105,7 +1105,7 @@ sqr_384: .endef .p2align 5 redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1129,7 +1129,7 @@ redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1140,7 +1140,7 @@ redc_mont_384: .endef .p2align 5 from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1181,7 +1181,7 @@ from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1405,7 +1405,7 @@ __redc_tail_mont_384: .endef .p2align 5 mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1423,7 +1423,7 @@ mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1613,7 +1613,7 @@ __mul_384: .endef .p2align 5 mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1695,7 +1695,7 @@ mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1706,7 +1706,7 @@ mul_382x: .endef .p2align 5 sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1797,475 +1797,7 @@ sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.globl sqr_mont_382x - -.def sqr_mont_382x; -.type 32; -.endef -.p2align 5 -sqr_mont_382x: -.long 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.def __mul_mont_383_nonred; -.type 32; -.endef -.p2align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret @@ -2276,7 +1808,7 @@ __mul_mont_383_nonred: .endef .p2align 5 sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2321,7 +1853,7 @@ sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2332,7 +1864,7 @@ sgn0_pty_mont_384: .endef .p2align 5 sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2421,6 +1953,6 @@ sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/mulq_mont_384-x86_64.s b/blst_src/build/coff/mulq_mont_384-x86_64.s index 6434f46d..c214a906 100644 --- a/blst_src/build/coff/mulq_mont_384-x86_64.s +++ b/blst_src/build/coff/mulq_mont_384-x86_64.s @@ -3158,771 +3158,6 @@ sqr_n_mul_mont_383: #endif .LSEH_end_sqr_n_mul_mont_383: -.def __mulq_mont_383_nonred; .scl 3; .type 32; .endef -.p2align 5 -__mulq_mont_383_nonred: - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.globl sqr_mont_382x - -.def sqr_mont_382x; .scl 2; .type 32; .endef -.p2align 5 -sqr_mont_382x: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_sqr_mont_382x: - - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx -#ifdef __BLST_PORTABLE__ - testl $1,__blst_platform_cap(%rip) - jnz sqr_mont_382x$1 -#endif - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $136,%rsp - -.LSEH_body_sqr_mont_382x: - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 - - movq 8(%r8),%r14 - - movq 16(%r8),%r13 - - movq 24(%r8),%r12 - - movq 32(%r8),%rbx - - movq 40(%r8),%rbp - - leaq 48(%r8),%rsp - -.LSEH_epilogue_sqr_mont_382x: - mov 8(%rsp),%rdi - mov 16(%rsp),%rsi - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.LSEH_end_sqr_mont_382x: .section .pdata .p2align 2 .rva .LSEH_begin_mul_mont_384x @@ -4093,18 +3328,6 @@ sqr_mont_382x: .rva .LSEH_end_sqr_n_mul_mont_383 .rva .LSEH_info_sqr_n_mul_mont_383_epilogue -.rva .LSEH_begin_sqr_mont_382x -.rva .LSEH_body_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_prologue - -.rva .LSEH_body_sqr_mont_382x -.rva .LSEH_epilogue_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_body - -.rva .LSEH_epilogue_sqr_mont_382x -.rva .LSEH_end_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_epilogue - .section .xdata .p2align 3 .LSEH_info_mul_mont_384x_prologue: @@ -4467,29 +3690,3 @@ sqr_mont_382x: .byte 0x00,0x64,0x02,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqr_mont_382x_prologue: -.byte 1,0,5,0x0b -.byte 0,0x74,1,0 -.byte 0,0x64,2,0 -.byte 0,0xb3 -.byte 0,0 -.long 0,0 -.LSEH_info_sqr_mont_382x_body: -.byte 1,0,18,0 -.byte 0x00,0xf4,0x11,0x00 -.byte 0x00,0xe4,0x12,0x00 -.byte 0x00,0xd4,0x13,0x00 -.byte 0x00,0xc4,0x14,0x00 -.byte 0x00,0x34,0x15,0x00 -.byte 0x00,0x54,0x16,0x00 -.byte 0x00,0x74,0x18,0x00 -.byte 0x00,0x64,0x19,0x00 -.byte 0x00,0x01,0x17,0x00 -.byte 0x00,0x00,0x00,0x00 -.byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqr_mont_382x_epilogue: -.byte 1,0,4,0 -.byte 0x00,0x74,0x01,0x00 -.byte 0x00,0x64,0x02,0x00 -.byte 0x00,0x00,0x00,0x00 - diff --git a/blst_src/build/coff/mulx_mont_256-x86_64.s b/blst_src/build/coff/mulx_mont_256-x86_64.s index 307d9185..1ecc9a73 100644 --- a/blst_src/build/coff/mulx_mont_256-x86_64.s +++ b/blst_src/build/coff/mulx_mont_256-x86_64.s @@ -212,17 +212,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -257,17 +254,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -302,16 +296,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -328,11 +319,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/coff/mulx_mont_384-x86_64.s b/blst_src/build/coff/mulx_mont_384-x86_64.s index 69fcbd73..0e1c2a4d 100644 --- a/blst_src/build/coff/mulx_mont_384-x86_64.s +++ b/blst_src/build/coff/mulx_mont_384-x86_64.s @@ -2064,8 +2064,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2093,11 +2093,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2127,8 +2125,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2156,11 +2154,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2190,8 +2186,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2219,11 +2215,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2253,8 +2247,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2282,11 +2276,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2316,8 +2308,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2345,11 +2337,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2380,10 +2370,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2789,9 +2778,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2849,9 +2837,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2909,9 +2896,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2969,9 +2955,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -3029,9 +3014,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3085,214 +3069,6 @@ __mulx_mont_383_nonred: #endif -.globl sqrx_mont_382x - -.def sqrx_mont_382x; .scl 2; .type 32; .endef -.p2align 5 -sqrx_mont_382x: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_sqrx_mont_382x: - - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx -sqr_mont_382x$1: - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $136,%rsp - -.LSEH_body_sqrx_mont_382x: - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 - - movq 8(%r8),%r14 - - movq 16(%r8),%r13 - - movq 24(%r8),%r12 - - movq 32(%r8),%rbx - - movq 40(%r8),%rbp - - leaq 48(%r8),%rsp - -.LSEH_epilogue_sqrx_mont_382x: - mov 8(%rsp),%rdi - mov 16(%rsp),%rsi - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.LSEH_end_sqrx_mont_382x: .section .pdata .p2align 2 .rva .LSEH_begin_mulx_mont_384x @@ -3463,18 +3239,6 @@ sqr_mont_382x$1: .rva .LSEH_end_sqrx_n_mul_mont_383 .rva .LSEH_info_sqrx_n_mul_mont_383_epilogue -.rva .LSEH_begin_sqrx_mont_382x -.rva .LSEH_body_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_prologue - -.rva .LSEH_body_sqrx_mont_382x -.rva .LSEH_epilogue_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_body - -.rva .LSEH_epilogue_sqrx_mont_382x -.rva .LSEH_end_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_epilogue - .section .xdata .p2align 3 .LSEH_info_mulx_mont_384x_prologue: @@ -3841,29 +3605,3 @@ sqr_mont_382x$1: .byte 0x00,0x64,0x02,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqrx_mont_382x_prologue: -.byte 1,0,5,0x0b -.byte 0,0x74,1,0 -.byte 0,0x64,2,0 -.byte 0,0xb3 -.byte 0,0 -.long 0,0 -.LSEH_info_sqrx_mont_382x_body: -.byte 1,0,18,0 -.byte 0x00,0xf4,0x11,0x00 -.byte 0x00,0xe4,0x12,0x00 -.byte 0x00,0xd4,0x13,0x00 -.byte 0x00,0xc4,0x14,0x00 -.byte 0x00,0x34,0x15,0x00 -.byte 0x00,0x54,0x16,0x00 -.byte 0x00,0x74,0x18,0x00 -.byte 0x00,0x64,0x19,0x00 -.byte 0x00,0x01,0x17,0x00 -.byte 0x00,0x00,0x00,0x00 -.byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqrx_mont_382x_epilogue: -.byte 1,0,4,0 -.byte 0x00,0x74,0x01,0x00 -.byte 0x00,0x64,0x02,0x00 -.byte 0x00,0x00,0x00,0x00 - diff --git a/blst_src/build/coff/sha256-armv8.S b/blst_src/build/coff/sha256-armv8.S index dcf1b5e7..ee5af87d 100644 --- a/blst_src/build/coff/sha256-armv8.S +++ b/blst_src/build/coff/sha256-armv8.S @@ -49,6 +49,7 @@ .endef .p2align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -191,6 +192,7 @@ blst_sha256_block_armv8: .endef .p2align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1042,6 +1044,7 @@ blst_sha256_block_data_order: .endef .p2align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1072,6 +1075,7 @@ blst_sha256_emit: .endef .p2align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1087,6 +1091,7 @@ blst_sha256_bcopy: .endef .p2align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/build/elf/add_mod_256-armv8.S b/blst_src/build/elf/add_mod_256-armv8.S index 57476aaa..4d1d791b 100644 --- a/blst_src/build/elf/add_mod_256-armv8.S +++ b/blst_src/build/elf/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +48,7 @@ add_mod_256: .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +98,7 @@ mul_by_3_mod_256: .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +137,7 @@ lshift_mod_256: .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -377,3 +389,13 @@ sub_n_check_mod_256: ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/add_mod_384-armv8.S b/blst_src/build/elf/add_mod_384-armv8.S index f5c0ead0..da6ee40a 100644 --- a/blst_src/build/elf/add_mod_384-armv8.S +++ b/blst_src/build/elf/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ __add_mod_384_ab_are_loaded: .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ add_mod_384x: .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ __rshift_mod_384: .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ div_by_2_mod_384: .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ __lshift_mod_384: .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ mul_by_3_mod_384: .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ mul_by_8_mod_384: .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ mul_by_3_mod_384x: .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ mul_by_8_mod_384x: .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ cneg_mod_384: .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ __sub_mod_384: .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ sub_mod_384x: .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ mul_by_1_plus_i_mod_384x: .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +681,7 @@ sgn0_pty_mod_384: .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +763,14 @@ sgn0_pty_mod_384x: .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ vec_select_32: .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ vec_select_48: .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ vec_select_96: .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ vec_select_192: .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ vec_select_144: .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ vec_select_288: .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ vec_prefetch: .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ vec_is_zero_16x: .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ vec_is_equal_16x: csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_inverse_mod_256-armv8.S b/blst_src/build/elf/ct_inverse_mod_256-armv8.S index 7d4d2eb5..32e3d271 100644 --- a/blst_src/build/elf/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/elf/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -20,6 +28,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +48,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -48,18 +57,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +80,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +113,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +138,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +163,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +188,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +213,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +238,24 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +270,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +298,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +326,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +354,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +382,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +410,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +438,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +510,7 @@ ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +548,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +594,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +847,13 @@ __inner_loop_62_256: ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_inverse_mod_384-armv8.S b/blst_src/build/elf/ct_inverse_mod_384-armv8.S index daebd370..f507c54e 100644 --- a/blst_src/build/elf/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/elf/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +30,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +54,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +105,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +377,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -375,20 +387,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +413,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,15 +480,15 @@ ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +496,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +511,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +552,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +582,41 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +624,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +637,15 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +764,7 @@ __smul_383_n_shift_by_62: sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +857,13 @@ __inner_loop_62: ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_is_square_mod_384-armv8.S b/blst_src/build/elf/ct_is_square_mod_384-armv8.S index ecac2453..950d49a0 100644 --- a/blst_src/build/elf/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/elf/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ct_is_square_mod_384: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ __inner_loop_48: ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s b/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s index ce59e2c3..049532e4 100644 --- a/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383,@function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384,@function .align 32 -ct_inverse_mod_383: +ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -549,10 +589,10 @@ ct_inverse_mod_383: .byte 0xf3,0xc3 #endif .cfi_endproc -.size ct_inverse_mod_383,.-ct_inverse_mod_383 -.type __smulq_767x63,@function +.size ct_inverse_mod_384,.-ct_inverse_mod_384 +.type __smulq_768x63,@function .align 32 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -770,10 +817,10 @@ __smulq_767x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_767x63,.-__smulq_767x63 -.type __smulq_383x63,@function +.size __smulq_768x63,.-__smulq_768x63 +.type __smulq_384x63,@function .align 32 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -922,10 +984,10 @@ __smulq_383x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383x63,.-__smulq_383x63 -.type __smulq_383_n_shift_by_62,@function +.size __smulq_384x63,.-__smulq_384x63 +.type __smulq_384_n_shift_by_62,@function .align 32 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1109,7 +1176,7 @@ __smulq_383_n_shift_by_62: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383_n_shift_by_62,.-__smulq_383_n_shift_by_62 +.size __smulq_384_n_shift_by_62,.-__smulq_384_n_shift_by_62 .type __ab_approximation_62,@function .align 32 __ab_approximation_62: diff --git a/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s b/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s index a7bc5a5a..96e129d5 100644 --- a/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl ctx_inverse_mod_383 -.hidden ctx_inverse_mod_383 -.type ctx_inverse_mod_383,@function +.globl ctx_inverse_mod_384 +.hidden ctx_inverse_mod_384 +.type ctx_inverse_mod_384,@function .align 32 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -893,10 +933,10 @@ ct_inverse_mod_383$1: .byte 0xf3,0xc3 #endif .cfi_endproc -.size ctx_inverse_mod_383,.-ctx_inverse_mod_383 -.type __smulx_767x63,@function +.size ctx_inverse_mod_384,.-ctx_inverse_mod_384 +.type __smulx_768x63,@function .align 32 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,62 +1046,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1068,10 +1115,10 @@ __smulx_767x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_767x63,.-__smulx_767x63 -.type __smulx_383x63,@function +.size __smulx_768x63,.-__smulx_768x63 +.type __smulx_384x63,@function .align 32 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,12 +1144,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1115,19 +1168,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,12 +1199,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1161,13 +1222,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1186,15 +1250,14 @@ __smulx_383x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383x63,.-__smulx_383x63 -.type __smulx_383_n_shift_by_31,@function +.size __smulx_384x63,.-__smulx_384x63 +.type __smulx_384_n_shift_by_31,@function .align 32 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1342,7 +1410,7 @@ __smulx_383_n_shift_by_31: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383_n_shift_by_31,.-__smulx_383_n_shift_by_31 +.size __smulx_384_n_shift_by_31,.-__smulx_384_n_shift_by_31 .type __smulx_191_n_shift_by_31,@function .align 32 __smulx_191_n_shift_by_31: @@ -1585,9 +1653,9 @@ __inner_loop_31: .cfi_endproc .size __inner_loop_31,.-__inner_loop_31 -.type __tail_loop_53,@function +.type __tail_loop_55,@function .align 32 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ __tail_loop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1635,7 +1703,7 @@ __tail_loop_53: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __tail_loop_53,.-__tail_loop_53 +.size __tail_loop_55,.-__tail_loop_55 .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/div3w-armv8.S b/blst_src/build/elf/div3w-armv8.S index 864773bf..48bbcb7d 100644 --- a/blst_src/build/elf/div3w-armv8.S +++ b/blst_src/build/elf/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ div_3_limbs: .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ quot_rem_128: .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ quot_rem_64: ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mul_mont_256-armv8.S b/blst_src/build/elf/mul_mont_256-armv8.S index b4ea3a6e..b8c39313 100644 --- a/blst_src/build/elf/mul_mont_256-armv8.S +++ b/blst_src/build/elf/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ mul_mont_sparse_256: .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ sqr_mont_sparse_256: .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +339,7 @@ from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ from_mont_256: .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +383,7 @@ redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ __mul_by_1_mont_256: ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mul_mont_384-armv8.S b/blst_src/build/elf/mul_mont_384-armv8.S index ea120827..13f45234 100644 --- a/blst_src/build/elf/mul_mont_384-armv8.S +++ b/blst_src/build/elf/mul_mont_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 @@ -5,7 +13,7 @@ .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -23,7 +31,7 @@ add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -87,7 +95,7 @@ __add_mod_384x384: .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +113,7 @@ sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -241,7 +249,7 @@ __sub_mod_384: .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -314,7 +322,7 @@ mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -323,7 +331,7 @@ mul_mont_384x: .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -399,7 +407,7 @@ sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -408,7 +416,7 @@ sqr_mont_384x: .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -440,7 +448,7 @@ mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -821,7 +829,7 @@ __mul_mont_384: .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -858,7 +866,7 @@ sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -867,7 +875,7 @@ sqr_mont_384: .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -923,7 +931,7 @@ sqr_n_mul_mont_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1044,7 +1052,7 @@ __sqr_384: .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1074,7 @@ sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1075,7 +1083,7 @@ sqr_384: .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1107,7 @@ redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1108,7 +1116,7 @@ redc_mont_384: .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1157,7 @@ from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1367,7 +1375,7 @@ __redc_tail_mont_384: .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1393,7 @@ mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1571,7 +1579,7 @@ __mul_384: .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1661,7 @@ mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1662,7 +1670,7 @@ mul_382x: .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1753,480 +1761,16 @@ sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,%function -.align 5 -sqr_mont_382x: - .inst 0xd503233f - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf - ret -.size sqr_mont_382x,.-sqr_mont_382x - -.type __mul_mont_383_nonred,%function -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - - ret -.size __mul_mont_383_nonred,.-__mul_mont_383_nonred - .globl sgn0_pty_mont_384 .hidden sgn0_pty_mont_384 .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1815,7 @@ sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2280,7 +1824,7 @@ sgn0_pty_mont_384: .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1913,16 @@ sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mulq_mont_384-x86_64.s b/blst_src/build/elf/mulq_mont_384-x86_64.s index b5653128..377fe596 100644 --- a/blst_src/build/elf/mulq_mont_384-x86_64.s +++ b/blst_src/build/elf/mulq_mont_384-x86_64.s @@ -3101,768 +3101,6 @@ sqr_n_mul_mont_383: #endif .cfi_endproc .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 -.type __mulq_mont_383_nonred,@function -.align 32 -__mulq_mont_383_nonred: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size __mulq_mont_383_nonred,.-__mulq_mont_383_nonred -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,@function -.align 32 -sqr_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -#ifdef __BLST_PORTABLE__ - testl $1,__blst_platform_cap(%rip) - jnz sqr_mont_382x$1 -#endif - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size sqr_mont_382x,.-sqr_mont_382x .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/mulx_mont_256-x86_64.s b/blst_src/build/elf/mulx_mont_256-x86_64.s index 788c78df..989d68c5 100644 --- a/blst_src/build/elf/mulx_mont_256-x86_64.s +++ b/blst_src/build/elf/mulx_mont_256-x86_64.s @@ -202,17 +202,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -247,17 +244,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -292,16 +286,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -318,11 +309,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/elf/mulx_mont_384-x86_64.s b/blst_src/build/elf/mulx_mont_384-x86_64.s index 8d299432..8c4a6623 100644 --- a/blst_src/build/elf/mulx_mont_384-x86_64.s +++ b/blst_src/build/elf/mulx_mont_384-x86_64.s @@ -2028,8 +2028,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2057,11 +2057,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2091,8 +2089,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2120,11 +2118,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2154,8 +2150,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2183,11 +2179,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2217,8 +2211,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2246,11 +2240,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2280,8 +2272,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2309,11 +2301,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2344,10 +2334,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2735,9 +2724,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2795,9 +2783,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2855,9 +2842,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2915,9 +2901,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -2975,9 +2960,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3031,209 +3015,6 @@ __mulx_mont_383_nonred: #endif .cfi_endproc .size __mulx_mont_383_nonred,.-__mulx_mont_383_nonred -.globl sqrx_mont_382x -.hidden sqrx_mont_382x -.type sqrx_mont_382x,@function -.align 32 -sqrx_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -sqr_mont_382x$1: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size sqrx_mont_382x,.-sqrx_mont_382x .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/sha256-armv8.S b/blst_src/build/elf/sha256-armv8.S index 965f60eb..3766a3a7 100644 --- a/blst_src/build/elf/sha256-armv8.S +++ b/blst_src/build/elf/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -47,6 +55,7 @@ .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -187,6 +196,7 @@ blst_sha256_block_armv8: .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1036,6 +1046,7 @@ blst_sha256_block_data_order: .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1075,7 @@ blst_sha256_emit: .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1077,9 +1089,20 @@ blst_sha256_bcopy: .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] stp x6,x7,[x0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/mach-o/add_mod_256-armv8.S b/blst_src/build/mach-o/add_mod_256-armv8.S index 198d65ae..1a6e63f5 100644 --- a/blst_src/build/mach-o/add_mod_256-armv8.S +++ b/blst_src/build/mach-o/add_mod_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +40,7 @@ _add_mod_256: .align 5 _mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +90,7 @@ _mul_by_3_mod_256: .align 5 _lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +129,7 @@ Loop_lshift_mod_256: .align 5 _rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/mach-o/add_mod_384-armv8.S b/blst_src/build/mach-o/add_mod_384-armv8.S index dedfa62f..c046b44a 100644 --- a/blst_src/build/mach-o/add_mod_384-armv8.S +++ b/blst_src/build/mach-o/add_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ _add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -71,7 +71,7 @@ __add_mod_384_ab_are_loaded: .align 5 _add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +99,7 @@ _add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -108,7 +108,7 @@ _add_mod_384x: .align 5 _rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +135,7 @@ Loop_rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -170,7 +170,7 @@ __rshift_mod_384: .align 5 _div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +194,7 @@ _div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -203,7 +203,7 @@ _div_by_2_mod_384: .align 5 _lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +230,7 @@ Loop_lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -268,7 +268,7 @@ __lshift_mod_384: .align 5 _mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +298,7 @@ _mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -307,7 +307,7 @@ _mul_by_3_mod_384: .align 5 _mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +333,7 @@ _mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -342,7 +342,7 @@ _mul_by_8_mod_384: .align 5 _mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +387,7 @@ _mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -396,7 +396,7 @@ _mul_by_3_mod_384x: .align 5 _mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +433,7 @@ _mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -442,7 +442,7 @@ _mul_by_8_mod_384x: .align 5 _cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +484,7 @@ _cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -493,7 +493,7 @@ _cneg_mod_384: .align 5 _sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +513,7 @@ _sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -556,7 +556,7 @@ __sub_mod_384: .align 5 _sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +584,7 @@ _sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -593,7 +593,7 @@ _sub_mod_384x: .align 5 _mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +626,7 @@ _mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -635,6 +635,7 @@ _mul_by_1_plus_i_mod_384x: .align 5 _sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +673,7 @@ _sgn0_pty_mod_384: .align 5 _sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +755,14 @@ _sgn0_pty_mod_384x: .align 5 _vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl _vec_select_48 @@ -768,6 +770,7 @@ _vec_select_32: .align 5 _vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +786,7 @@ _vec_select_48: .align 5 _vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +808,7 @@ _vec_select_96: .align 5 _vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +842,7 @@ _vec_select_192: .align 5 _vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +870,7 @@ _vec_select_144: .align 5 _vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +916,7 @@ _vec_select_288: .align 5 _vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +957,7 @@ _vec_prefetch: .align 5 _vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +983,7 @@ Loop_is_zero_done: .align 5 _vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S b/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S index 48cebab2..54a93e77 100644 --- a/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -20,6 +20,7 @@ _ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +40,7 @@ _ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -48,18 +49,18 @@ _ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +72,27 @@ _ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +105,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +130,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +155,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +180,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +205,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +230,24 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +262,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +290,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +318,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +346,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +374,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +402,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +430,15 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +502,7 @@ _ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -579,11 +540,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +586,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S b/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S index f1bb996f..07bdb673 100644 --- a/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S @@ -1,11 +1,11 @@ .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .align 5 -_ct_inverse_mod_383: -.long 3573752639 +_ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ _ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +46,40 @@ _ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +97,269 @@ _ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +369,7 @@ _ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -375,20 +379,22 @@ _ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +405,60 @@ _ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,7 +472,7 @@ _ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -439,7 +480,7 @@ _ct_inverse_mod_383: // see corresponding commentary in ctx_inverse_mod_384-x86_64... .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +488,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +503,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +544,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +574,41 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +616,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +629,15 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +657,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +697,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S b/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S index fe9224af..1e61ea46 100644 --- a/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +84,7 @@ Loop_is_square: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s b/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s index 0a37c731..c977ae13 100644 --- a/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm ___blst_platform_cap,4 .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .p2align 5 -_ct_inverse_mod_383: +_ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz L$ct_inverse_mod_383$1 + jnz L$ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ _ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ _ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ _ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ _ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ _ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -552,7 +592,7 @@ _ct_inverse_mod_383: .p2align 5 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -773,7 +820,7 @@ __smulq_767x63: .p2align 5 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -925,7 +987,7 @@ __smulq_383x63: .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 diff --git a/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s b/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s index e494e44b..aad4e1a4 100644 --- a/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl _ctx_inverse_mod_383 -.private_extern _ctx_inverse_mod_383 +.globl _ctx_inverse_mod_384 +.private_extern _ctx_inverse_mod_384 .p2align 5 -_ctx_inverse_mod_383: +_ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -L$ct_inverse_mod_383$1: +L$ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ L$ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ L$ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ L$ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ L$ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -896,7 +936,7 @@ L$ct_inverse_mod_383$1: .p2align 5 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,62 +1046,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1071,7 +1118,7 @@ __smulx_767x63: .p2align 5 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,12 +1144,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1115,19 +1168,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,12 +1199,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1161,13 +1222,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1189,12 +1253,11 @@ __smulx_383x63: .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1587,7 +1655,7 @@ L$oop_31: .p2align 5 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -L$oop_53: +L$oop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ L$oop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz L$oop_53 + jnz L$oop_55 #ifdef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/mach-o/div3w-armv8.S b/blst_src/build/mach-o/div3w-armv8.S index 55bec60a..6530fb8f 100644 --- a/blst_src/build/mach-o/div3w-armv8.S +++ b/blst_src/build/mach-o/div3w-armv8.S @@ -5,6 +5,7 @@ .align 5 _div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +40,7 @@ Loop: .align 5 _quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +78,7 @@ _quot_rem_128: .align 5 _quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/blst_src/build/mach-o/mul_mont_256-armv8.S b/blst_src/build/mach-o/mul_mont_256-armv8.S index 3fac56fd..e1b9d454 100644 --- a/blst_src/build/mach-o/mul_mont_256-armv8.S +++ b/blst_src/build/mach-o/mul_mont_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +197,7 @@ _mul_mont_sparse_256: .align 5 _sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +298,7 @@ _sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl _from_mont_256 @@ -305,7 +306,7 @@ _sqr_mont_sparse_256: .align 5 _from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +331,7 @@ _from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -339,7 +340,7 @@ _from_mont_256: .align 5 _redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +375,7 @@ _redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/mul_mont_384-armv8.S b/blst_src/build/mach-o/mul_mont_384-armv8.S index 3c069ed7..77ca24d7 100644 --- a/blst_src/build/mach-o/mul_mont_384-armv8.S +++ b/blst_src/build/mach-o/mul_mont_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -23,7 +23,7 @@ _add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -87,7 +87,7 @@ __add_mod_384x384: .align 5 _sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +105,7 @@ _sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -241,7 +241,7 @@ __sub_mod_384: .align 5 _mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -314,7 +314,7 @@ _mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -323,7 +323,7 @@ _mul_mont_384x: .align 5 _sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -399,7 +399,7 @@ _sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -408,7 +408,7 @@ _sqr_mont_384x: .align 5 _mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -440,7 +440,7 @@ _mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -821,7 +821,7 @@ __mul_mont_384: .align 5 _sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -858,7 +858,7 @@ _sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -867,7 +867,7 @@ _sqr_mont_384: .align 5 _sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -923,7 +923,7 @@ Loop_sqr_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1044,7 +1044,7 @@ __sqr_384: .align 5 _sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1066,7 @@ _sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1075,7 +1075,7 @@ _sqr_384: .align 5 _redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1099,7 @@ _redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1108,7 +1108,7 @@ _redc_mont_384: .align 5 _from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1149,7 @@ _from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1367,7 +1367,7 @@ __redc_tail_mont_384: .align 5 _mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1385,7 @@ _mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1571,7 +1571,7 @@ __mul_384: .align 5 _mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1653,7 @@ _mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1662,7 +1662,7 @@ _mul_382x: .align 5 _sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1753,471 +1753,7 @@ _sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.globl _sqr_mont_382x -.private_extern _sqr_mont_382x - -.align 5 -_sqr_mont_382x: -.long 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // _mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // _mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - - -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret @@ -2226,7 +1762,7 @@ __mul_mont_383_nonred: .align 5 _sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1807,7 @@ _sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2280,7 +1816,7 @@ _sgn0_pty_mont_384: .align 5 _sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1905,6 @@ _sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/mulq_mont_384-x86_64.s b/blst_src/build/mach-o/mulq_mont_384-x86_64.s index 3cab12b1..b7c1ad8e 100644 --- a/blst_src/build/mach-o/mulq_mont_384-x86_64.s +++ b/blst_src/build/mach-o/mulq_mont_384-x86_64.s @@ -3101,765 +3101,3 @@ L$oop_sqr_383: #endif .cfi_endproc - -.p2align 5 -__mulq_mont_383_nonred: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - -.globl _sqr_mont_382x -.private_extern _sqr_mont_382x - -.p2align 5 -_sqr_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -#ifdef __BLST_PORTABLE__ - testl $1,___blst_platform_cap(%rip) - jnz L$sqr_mont_382x$1 -#endif - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - diff --git a/blst_src/build/mach-o/mulx_mont_256-x86_64.s b/blst_src/build/mach-o/mulx_mont_256-x86_64.s index 9f0c6a2b..df73cc42 100644 --- a/blst_src/build/mach-o/mulx_mont_256-x86_64.s +++ b/blst_src/build/mach-o/mulx_mont_256-x86_64.s @@ -202,17 +202,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -247,17 +244,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -292,16 +286,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -318,11 +309,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/mach-o/mulx_mont_384-x86_64.s b/blst_src/build/mach-o/mulx_mont_384-x86_64.s index 45b3600e..e32c8834 100644 --- a/blst_src/build/mach-o/mulx_mont_384-x86_64.s +++ b/blst_src/build/mach-o/mulx_mont_384-x86_64.s @@ -2028,8 +2028,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2057,11 +2057,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2091,8 +2089,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2120,11 +2118,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2154,8 +2150,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2183,11 +2179,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2217,8 +2211,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2246,11 +2240,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2280,8 +2272,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2309,11 +2301,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2344,10 +2334,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2735,9 +2724,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2795,9 +2783,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2855,9 +2842,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2915,9 +2901,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -2975,9 +2960,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3031,206 +3015,3 @@ __mulx_mont_383_nonred: #endif .cfi_endproc -.globl _sqrx_mont_382x -.private_extern _sqrx_mont_382x - -.p2align 5 -_sqrx_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -L$sqr_mont_382x$1: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - diff --git a/blst_src/build/mach-o/sha256-armv8.S b/blst_src/build/mach-o/sha256-armv8.S index cf0f80d1..c7befef9 100644 --- a/blst_src/build/mach-o/sha256-armv8.S +++ b/blst_src/build/mach-o/sha256-armv8.S @@ -47,6 +47,7 @@ LK256: .align 6 _blst_sha256_block_armv8: + hint #34 Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -187,6 +188,7 @@ Loop_hw: .align 4 _blst_sha256_block_data_order: + hint #34 adrp x16,___blst_platform_cap@PAGE ldr w16,[x16,___blst_platform_cap@PAGEOFF] tst w16,#1 @@ -1036,6 +1038,7 @@ L_00_48: .align 4 _blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1067,7 @@ _blst_sha256_emit: .align 4 _blst_sha256_bcopy: + hint #34 Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1077,6 +1081,7 @@ Loop_bcopy: .align 4 _blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/build/win64/add_mod_256-armv8.asm b/blst_src/build/win64/add_mod_256-armv8.asm index 91b7b5a7..ea64176f 100644 --- a/blst_src/build/win64/add_mod_256-armv8.asm +++ b/blst_src/build/win64/add_mod_256-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_256|[FUNC] ALIGN 32 |add_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -41,6 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_256|[FUNC] ALIGN 32 |mul_by_3_mod_256| PROC + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -90,6 +92,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |lshift_mod_256|[FUNC] ALIGN 32 |lshift_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -128,6 +131,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |rshift_mod_256|[FUNC] ALIGN 32 |rshift_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/win64/add_mod_384-armv8.asm b/blst_src/build/win64/add_mod_384-armv8.asm index 5b2989ce..dfb0a747 100644 --- a/blst_src/build/win64/add_mod_384-armv8.asm +++ b/blst_src/build/win64/add_mod_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384|[FUNC] ALIGN 32 |add_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -27,7 +27,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -73,7 +73,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384x|[FUNC] ALIGN 32 |add_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -101,7 +101,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -110,7 +110,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |rshift_mod_384|[FUNC] ALIGN 32 |rshift_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -137,7 +137,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -172,7 +172,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |div_by_2_mod_384|[FUNC] ALIGN 32 |div_by_2_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +196,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -205,7 +205,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |lshift_mod_384|[FUNC] ALIGN 32 |lshift_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -232,7 +232,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -270,7 +270,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_384|[FUNC] ALIGN 32 |mul_by_3_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -300,7 +300,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -309,7 +309,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_8_mod_384|[FUNC] ALIGN 32 |mul_by_8_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -335,7 +335,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -344,7 +344,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_384x|[FUNC] ALIGN 32 |mul_by_3_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -389,7 +389,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -398,7 +398,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_8_mod_384x|[FUNC] ALIGN 32 |mul_by_8_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -435,7 +435,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -444,7 +444,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |cneg_mod_384|[FUNC] ALIGN 32 |cneg_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -486,7 +486,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -495,7 +495,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384|[FUNC] ALIGN 32 |sub_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -515,7 +515,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -558,7 +558,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384x|[FUNC] ALIGN 32 |sub_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -586,7 +586,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -595,7 +595,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_1_plus_i_mod_384x|[FUNC] ALIGN 32 |mul_by_1_plus_i_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -628,7 +628,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -637,6 +637,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mod_384|[FUNC] ALIGN 32 |sgn0_pty_mod_384| PROC + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -674,6 +675,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mod_384x|[FUNC] ALIGN 32 |sgn0_pty_mod_384x| PROC + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -755,14 +757,14 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_32|[FUNC] ALIGN 32 |vec_select_32| PROC + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret ENDP @@ -770,6 +772,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_48|[FUNC] ALIGN 32 |vec_select_48| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -785,6 +788,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_96|[FUNC] ALIGN 32 |vec_select_96| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -806,6 +810,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_192|[FUNC] ALIGN 32 |vec_select_192| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -839,6 +844,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_144|[FUNC] ALIGN 32 |vec_select_144| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -866,6 +872,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_288|[FUNC] ALIGN 32 |vec_select_288| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -911,6 +918,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_prefetch|[FUNC] ALIGN 32 |vec_prefetch| PROC + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -951,6 +959,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_is_zero_16x|[FUNC] ALIGN 32 |vec_is_zero_16x| PROC + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -976,6 +985,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_is_equal_16x|[FUNC] ALIGN 32 |vec_is_equal_16x| PROC + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/win64/blst.def b/blst_src/build/win64/blst.def index dda95336..5bc8602e 100644 --- a/blst_src/build/win64/blst.def +++ b/blst_src/build/win64/blst.def @@ -217,5 +217,7 @@ EXPORTS blst_p2_sizeof blst_p2_affine_sizeof blst_fp12_sizeof + blst_fp_from_le_bytes + blst_fp_from_be_bytes blst_sha256 diff --git a/blst_src/build/win64/ct_inverse_mod_256-armv8.asm b/blst_src/build/win64/ct_inverse_mod_256-armv8.asm index fde83e0e..b6be3bc2 100644 --- a/blst_src/build/win64/ct_inverse_mod_256-armv8.asm +++ b/blst_src/build/win64/ct_inverse_mod_256-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |ct_inverse_mod_256|[FUNC] ALIGN 32 |ct_inverse_mod_256| PROC - DCDU 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ __SIZEOF_POINTER__ SETA 64/8 if :def: __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 else add x1, sp, #16+511 and x1, x1, #-512 @@ -41,7 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] @@ -50,18 +51,18 @@ __SIZEOF_POINTER__ SETA 64/8 mov x13, x15 add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] + str x12, [x0,#8*10] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -73,29 +74,27 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] - ldr x9, [x1,#8*13] + ldr x9, [x1,#8*14] madd x4, x16, x8, xzr madd x4, x17, x9, x4 - str x4, [x0,#8*4] asr x5, x4, #63 - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr madd x4, x13, x9, x4 - str x4, [x0,#8*9] asr x5, x4, #63 - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -108,25 +107,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -139,25 +132,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -170,25 +157,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -201,25 +182,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -232,25 +207,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -263,25 +232,24 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -296,21 +264,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -325,21 +292,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -354,21 +320,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -383,21 +348,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -412,21 +376,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -441,21 +404,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -470,16 +432,15 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #47 @@ -543,7 +504,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -581,11 +542,11 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] + ldp x8, x9, [x1,#8*0+112] asr x14, x17, #63 - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 sub x17, x17, x14 @@ -627,9 +588,9 @@ __SIZEOF_POINTER__ SETA 64/8 ALIGN 32 |__smul_512x63_tail| PROC umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] + ldr x5, [x1,#8*19] adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 diff --git a/blst_src/build/win64/ct_inverse_mod_384-armv8.asm b/blst_src/build/win64/ct_inverse_mod_384-armv8.asm index 36fe37b0..be929b12 100644 --- a/blst_src/build/win64/ct_inverse_mod_384-armv8.asm +++ b/blst_src/build/win64/ct_inverse_mod_384-armv8.asm @@ -4,10 +4,10 @@ __SIZEOF_POINTER__ SETA 64/8 - EXPORT |ct_inverse_mod_383|[FUNC] + EXPORT |ct_inverse_mod_384|[FUNC] ALIGN 32 -|ct_inverse_mod_383| PROC - DCDU 3573752639 +|ct_inverse_mod_384| PROC + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -24,6 +24,7 @@ __SIZEOF_POINTER__ SETA 64/8 if :def: __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 else add x1, sp, #32+511 and x1, x1, #-512 @@ -47,40 +48,40 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] - ldr x8, [x1,#8*18] + ldr x8, [x1,#8*20] mul x3, x20, x7 smulh x4, x20, x7 mul x5, x21, x8 @@ -98,266 +99,269 @@ __SIZEOF_POINTER__ SETA 64/8 smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 @@ -367,7 +371,7 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -377,20 +381,22 @@ __SIZEOF_POINTER__ SETA 64/8 mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif - mov x2, #22 + mov x2, #24 ldr x3, [x1,#8*0] eor x8, x8, x8 @@ -401,25 +407,60 @@ __SIZEOF_POINTER__ SETA 64/8 mov x20, x17 mov x21, x19 ldp x0, x15, [sp] - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 + smulh x23, x8, x21 + adc x26, x26, x28 ldp x9, x10, [x15,#8*0] + add x23, x23, x26 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 ldp x13, x14, [x15,#8*4] - and x9, x9, x22 - and x10, x10, x22 + and x26, x9, x22 + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr + + neg x22, x23 + orr x23, x23, x22 + asr x22, x22, #63 + + and x9, x9, x23 + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + adds x3, x3, x9 - and x11, x11, x22 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -433,7 +474,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -441,7 +482,7 @@ __SIZEOF_POINTER__ SETA 64/8 ALIGN 32 -|__smul_383x63| PROC +|__smul_384x63| PROC ldp x3, x4, [x1,#8*0+96] asr x17, x20, #63 ldp x5, x6, [x1,#8*2+96] @@ -449,6 +490,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -463,28 +505,33 @@ __SIZEOF_POINTER__ SETA 64/8 umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + cselne x25,x25,xzr adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] + ldp x9, x10, [x1,#8*0+160] asr x17, x21, #63 - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -499,21 +546,25 @@ __SIZEOF_POINTER__ SETA 64/8 umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr + cmp x21, #0 mul x11, x11, x21 + cselne x26,x26,xzr adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -525,41 +576,41 @@ __SIZEOF_POINTER__ SETA 64/8 stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr ret ENDP ALIGN 32 -|__smul_767x63_tail| PROC - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] +|__smul_768x63_tail| PROC + umulh x27, x8, x20 + ldr x4, [x1,#8*27] + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 - eor x3, x3, x17 + umulh x14, x14, x21 eor x4, x4, x17 eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -567,10 +618,11 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr adds x3, x3, x27 adcs x4, x4, x28 @@ -579,15 +631,15 @@ __SIZEOF_POINTER__ SETA 64/8 stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 + stp x7, x25, [x0,#8*10] ret ENDP ALIGN 32 -|__smul_383_n_shift_by_62| PROC +|__smul_384_n_shift_by_62| PROC ldp x3, x4, [x1,#8*0+0] asr x28, x15, #63 ldp x5, x6, [x1,#8*2+0] @@ -607,25 +659,27 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] asr x28, x16, #63 ldp x11, x12, [x1,#8*2+48] @@ -645,25 +699,27 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/win64/ct_is_square_mod_384-armv8.asm b/blst_src/build/win64/ct_is_square_mod_384-armv8.asm index 5d3ca14e..cb8682cd 100644 --- a/blst_src/build/win64/ct_is_square_mod_384-armv8.asm +++ b/blst_src/build/win64/ct_is_square_mod_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |ct_is_square_mod_384|[FUNC] ALIGN 32 |ct_is_square_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -86,7 +86,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP diff --git a/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm b/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm index 30c038cb..13aa9430 100644 --- a/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm +++ b/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm @@ -1,20 +1,22 @@ OPTION DOTNAME -EXTERN ct_inverse_mod_383$1:NEAR +ifdef __BLST_PORTABLE__ +EXTERN ct_inverse_mod_384$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS .text$ SEGMENT ALIGN(256) 'CODE' -PUBLIC ct_inverse_mod_383 +PUBLIC ct_inverse_mod_384 ALIGN 32 -ct_inverse_mod_383 PROC PUBLIC +ct_inverse_mod_384 PROC PUBLIC DB 243,15,30,250 mov QWORD PTR[8+rsp],rdi ;WIN64 prologue mov QWORD PTR[16+rsp],rsi mov r11,rsp -$L$SEH_begin_ct_inverse_mod_383:: +$L$SEH_begin_ct_inverse_mod_384:: mov rdi,rcx @@ -23,7 +25,7 @@ $L$SEH_begin_ct_inverse_mod_383:: mov rcx,r9 ifdef __BLST_PORTABLE__ test DWORD PTR[__blst_platform_cap],1 - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 endif push rbp @@ -39,7 +41,7 @@ endif sub rsp,1112 -$L$SEH_body_ct_inverse_mod_383:: +$L$SEH_body_ct_inverse_mod_384:: lea rax,QWORD PTR[((88+511))+rsp] @@ -86,7 +88,7 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[96+rdi],rdx @@ -94,10 +96,10 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - mov QWORD PTR[96+rdi],rdx + mov QWORD PTR[104+rdi],rdx xor rsi,256 @@ -110,19 +112,19 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov rax,QWORD PTR[96+rsi] - mov r11,QWORD PTR[144+rsi] + mov r11,QWORD PTR[152+rsi] mov rbx,rdx mov r10,rax imul QWORD PTR[56+rsp] @@ -139,6 +141,7 @@ $L$SEH_body_ct_inverse_mod_383:: mov QWORD PTR[72+rdi],r9 mov QWORD PTR[80+rdi],r9 mov QWORD PTR[88+rdi],r9 + mov QWORD PTR[96+rdi],r9 lea rsi,QWORD PTR[96+rsi] mov rax,r10 @@ -149,13 +152,14 @@ $L$SEH_body_ct_inverse_mod_383:: imul rcx add r8,rax adc r9,rdx - mov QWORD PTR[96+rdi],r8 - mov QWORD PTR[104+rdi],r9 - sar r9,63 + mov QWORD PTR[104+rdi],r8 mov QWORD PTR[112+rdi],r9 + sar r9,63 mov QWORD PTR[120+rdi],r9 mov QWORD PTR[128+rdi],r9 mov QWORD PTR[136+rdi],r9 + mov QWORD PTR[144+rdi],r9 + mov QWORD PTR[152+rdi],r9 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -166,14 +170,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -181,12 +185,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -197,14 +201,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -212,12 +216,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -228,14 +232,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -243,12 +247,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -259,14 +263,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -274,19 +278,17 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 - sar r13,63 - mov QWORD PTR[48+rdi],r13 - mov QWORD PTR[56+rdi],r13 - mov QWORD PTR[64+rdi],r13 - mov QWORD PTR[72+rdi],r13 - mov QWORD PTR[80+rdi],r13 - mov QWORD PTR[88+rdi],r13 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 + mov QWORD PTR[56+rdi],r14 + mov QWORD PTR[64+rdi],r14 + mov QWORD PTR[72+rdi],r14 + mov QWORD PTR[80+rdi],r14 + mov QWORD PTR[88+rdi],r14 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -297,14 +299,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -312,12 +314,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -328,14 +330,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -343,12 +345,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -359,14 +361,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -374,12 +376,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -390,14 +392,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -405,12 +407,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -421,14 +423,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -436,12 +438,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 @@ -465,16 +467,16 @@ $L$SEH_body_ct_inverse_mod_383:: lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[96+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 - mov edi,22 + mov edi,24 mov r8,QWORD PTR[rsi] xor r9,r9 @@ -497,37 +499,77 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdx,r12 mov rcx,r13 mov rdi,QWORD PTR[32+rsp] - call __smulq_767x63 + call __smulq_768x63 mov rsi,QWORD PTR[40+rsp] - mov rdx,rax - sar rax,63 + mov r13,rdx + sar r13,63 - mov r8,rax - mov r9,rax - mov r10,rax + mov r8,r13 + mov r9,r13 + mov r10,r13 + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,r13 + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,r13 + and r12,QWORD PTR[32+rsi] + and r13,QWORD PTR[40+rsi] + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rax,r13 + adc rdx,0 + + mov r13,rdx + neg rdx + or r13,rdx + sar rdx,63 + + mov r8,r13 + mov r9,r13 + mov r10,r13 and r8,QWORD PTR[rsi] and r9,QWORD PTR[8+rsi] - mov r11,rax + mov r11,r13 and r10,QWORD PTR[16+rsi] and r11,QWORD PTR[24+rsi] - mov r12,rax + mov r12,r13 and r12,QWORD PTR[32+rsi] - and rax,QWORD PTR[40+rsi] + and r13,QWORD PTR[40+rsi] + + xor r8,rdx + xor rsi,rsi + xor r9,rdx + sub rsi,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add r8,rsi + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 add r14,r8 adc r15,r9 adc rbx,r10 adc rbp,r11 adc rcx,r12 - adc rdx,rax + adc rax,r13 mov QWORD PTR[48+rdi],r14 mov QWORD PTR[56+rdi],r15 mov QWORD PTR[64+rdi],rbx mov QWORD PTR[72+rdi],rbp mov QWORD PTR[80+rdi],rcx - mov QWORD PTR[88+rdi],rdx + mov QWORD PTR[88+rdi],rax lea r8,QWORD PTR[1112+rsp] mov r15,QWORD PTR[r8] @@ -544,7 +586,7 @@ $L$SEH_body_ct_inverse_mod_383:: lea rsp,QWORD PTR[48+r8] -$L$SEH_epilogue_ct_inverse_mod_383:: +$L$SEH_epilogue_ct_inverse_mod_384:: mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue mov rsi,QWORD PTR[16+rsp] @@ -558,11 +600,11 @@ else DB 0F3h,0C3h endif -$L$SEH_end_ct_inverse_mod_383:: -ct_inverse_mod_383 ENDP +$L$SEH_end_ct_inverse_mod_384:: +ct_inverse_mod_384 ENDP ALIGN 32 -__smulq_767x63 PROC PRIVATE +__smulq_768x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -571,6 +613,7 @@ __smulq_767x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -579,7 +622,7 @@ __smulq_767x63 PROC PRIVATE mov QWORD PTR[8+rsp],rdi mov QWORD PTR[16+rsp],rsi - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] xor rbp,rdx add rbp,rax @@ -590,16 +633,20 @@ __smulq_767x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov QWORD PTR[rdi],rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -625,14 +672,14 @@ __smulq_767x63 PROC PRIVATE adc rdx,0 mov r13,rdx mov QWORD PTR[32+rdi],r12 - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r14,rdx mov QWORD PTR[40+rdi],r13 - mov QWORD PTR[48+rdi],rdx - sar rdx,63 - mov QWORD PTR[56+rdi],rdx + mov QWORD PTR[48+rdi],r14 + sar r14,63 + mov QWORD PTR[56+rdi],r14 mov rdx,rcx mov r8,QWORD PTR[rsi] @@ -735,39 +782,41 @@ __smulq_767x63 PROC PRIVATE mov rax,rdi adc rdx,0 mov rdi,rdx - mov rdx,QWORD PTR[8+rsp] - imul rax,rsi - mov rsi,QWORD PTR[16+rsp] + imul rsi + mov rsi,QWORD PTR[8+rsp] add rax,rdi + adc rdx,0 - add r8,QWORD PTR[rdx] - adc r9,QWORD PTR[8+rdx] - adc r10,QWORD PTR[16+rdx] - adc r11,QWORD PTR[24+rdx] - adc r12,QWORD PTR[32+rdx] - adc r13,QWORD PTR[40+rdx] - adc r14,QWORD PTR[48+rdx] - mov rdi,QWORD PTR[56+rdx] + add r8,QWORD PTR[rsi] + adc r9,QWORD PTR[8+rsi] + adc r10,QWORD PTR[16+rsi] + adc r11,QWORD PTR[24+rsi] + adc r12,QWORD PTR[32+rsi] + adc r13,QWORD PTR[40+rsi] + adc r14,QWORD PTR[48+rsi] + mov rdi,QWORD PTR[56+rsi] adc r15,rdi adc rbx,rdi adc rbp,rdi adc rcx,rdi adc rax,rdi + adc rdx,rdi - mov rdi,rdx + lea rdi,QWORD PTR[rsi] + mov rsi,QWORD PTR[16+rsp] - mov QWORD PTR[rdx],r8 - mov QWORD PTR[8+rdx],r9 - mov QWORD PTR[16+rdx],r10 - mov QWORD PTR[24+rdx],r11 - mov QWORD PTR[32+rdx],r12 - mov QWORD PTR[40+rdx],r13 - mov QWORD PTR[48+rdx],r14 - mov QWORD PTR[56+rdx],r15 - mov QWORD PTR[64+rdx],rbx - mov QWORD PTR[72+rdx],rbp - mov QWORD PTR[80+rdx],rcx - mov QWORD PTR[88+rdx],rax + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rax ifdef __SGX_LVI_HARDENING__ @@ -778,10 +827,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_767x63 ENDP +__smulq_768x63 ENDP ALIGN 32 -__smulq_383x63 PROC PRIVATE +__smulq_384x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -790,6 +839,7 @@ __smulq_383x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -805,16 +855,20 @@ __smulq_383x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -836,10 +890,11 @@ __smulq_383x63 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rax,rbp + mul rbp add r13,rax + adc r14,rdx - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] mov rdx,rcx mov QWORD PTR[rdi],r8 @@ -847,13 +902,15 @@ __smulq_383x63 PROC PRIVATE mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],r13 + mov r15,r13 + mov rbx,r14 mov r8,QWORD PTR[rsi] mov r9,QWORD PTR[8+rsi] mov r10,QWORD PTR[16+rsi] mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -869,16 +926,20 @@ __smulq_383x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -900,17 +961,19 @@ __smulq_383x63 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rax,rbp + mul rbp add r13,rax + adc r14,rdx - lea rsi,QWORD PTR[((-48))+rsi] + lea rsi,QWORD PTR[((-56))+rsi] add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc r13,QWORD PTR[40+rdi] + adc r13,r15 + adc r14,rbx mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 @@ -918,6 +981,7 @@ __smulq_383x63 PROC PRIVATE mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 ifdef __SGX_LVI_HARDENING__ @@ -928,10 +992,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_383x63 ENDP +__smulq_384x63 ENDP ALIGN 32 -__smulq_383_n_shift_by_62 PROC PRIVATE +__smulq_384_n_shift_by_62 PROC PRIVATE DB 243,15,30,250 mov rbx,rdx @@ -956,6 +1020,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + mov r14,rdx add rax,r8 adc r9,0 adc r10,0 @@ -966,6 +1031,8 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -987,12 +1054,11 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r14,rdx lea rsi,QWORD PTR[48+rsi] - mov r14,rdx mov rdx,rcx mov QWORD PTR[rdi],r8 @@ -1022,6 +1088,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + mov r15,rdx add rax,r8 adc r9,0 adc r10,0 @@ -1032,6 +1099,8 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mul rbp mov r8,rax mov rax,r9 + and r15,rbp + neg r15 mov r9,rdx mul rbp add r9,rax @@ -1053,11 +1122,12 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r15,rdx lea rsi,QWORD PTR[((-48))+rsi] + mov rdx,rbx add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] @@ -1065,8 +1135,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] adc r13,QWORD PTR[40+rdi] - adc r14,rdx - mov rdx,rbx + adc r14,r15 shrd r8,r9,62 shrd r9,r10,62 @@ -1113,7 +1182,7 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_383_n_shift_by_62 ENDP +__smulq_384_n_shift_by_62 ENDP ALIGN 32 __ab_approximation_62 PROC PRIVATE @@ -1242,29 +1311,29 @@ __inner_loop_62 ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 - DD imagerel $L$SEH_begin_ct_inverse_mod_383 - DD imagerel $L$SEH_body_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_prologue + DD imagerel $L$SEH_begin_ct_inverse_mod_384 + DD imagerel $L$SEH_body_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_prologue - DD imagerel $L$SEH_body_ct_inverse_mod_383 - DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_body + DD imagerel $L$SEH_body_ct_inverse_mod_384 + DD imagerel $L$SEH_epilogue_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_body - DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 - DD imagerel $L$SEH_end_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_epilogue + DD imagerel $L$SEH_epilogue_ct_inverse_mod_384 + DD imagerel $L$SEH_end_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_epilogue .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 -$L$SEH_info_ct_inverse_mod_383_prologue:: +$L$SEH_info_ct_inverse_mod_384_prologue:: DB 1,0,5,00bh DB 0,074h,1,0 DB 0,064h,2,0 DB 0,0b3h DB 0,0 DD 0,0 -$L$SEH_info_ct_inverse_mod_383_body:: +$L$SEH_info_ct_inverse_mod_384_body:: DB 1,0,18,0 DB 000h,0f4h,08bh,000h DB 000h,0e4h,08ch,000h @@ -1277,7 +1346,7 @@ DB 000h,064h,093h,000h DB 000h,001h,091h,000h DB 000h,000h,000h,000h DB 000h,000h,000h,000h -$L$SEH_info_ct_inverse_mod_383_epilogue:: +$L$SEH_info_ct_inverse_mod_384_epilogue:: DB 1,0,4,0 DB 000h,074h,001h,000h DB 000h,064h,002h,000h diff --git a/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm b/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm index cfc2141c..90d23389 100644 --- a/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm +++ b/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm @@ -1,24 +1,26 @@ OPTION DOTNAME -PUBLIC ct_inverse_mod_383$1 +ifdef __BLST_PORTABLE__ +PUBLIC ct_inverse_mod_384$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' -PUBLIC ctx_inverse_mod_383 +PUBLIC ctx_inverse_mod_384 ALIGN 32 -ctx_inverse_mod_383 PROC PUBLIC +ctx_inverse_mod_384 PROC PUBLIC DB 243,15,30,250 mov QWORD PTR[8+rsp],rdi ;WIN64 prologue mov QWORD PTR[16+rsp],rsi mov r11,rsp -$L$SEH_begin_ctx_inverse_mod_383:: +$L$SEH_begin_ctx_inverse_mod_384:: mov rdi,rcx mov rsi,rdx mov rdx,r8 mov rcx,r9 -ct_inverse_mod_383$1:: +ct_inverse_mod_384$1:: push rbp push rbx @@ -33,7 +35,7 @@ ct_inverse_mod_383$1:: sub rsp,1112 -$L$SEH_body_ctx_inverse_mod_383:: +$L$SEH_body_ctx_inverse_mod_384:: lea rax,QWORD PTR[((88+511))+rsp] @@ -83,7 +85,7 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[96+rdi],rdx @@ -91,10 +93,10 @@ endif mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - mov QWORD PTR[96+rdi],rdx + mov QWORD PTR[104+rdi],rdx xor rsi,256 @@ -107,19 +109,19 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov rax,QWORD PTR[96+rsi] - mov r11,QWORD PTR[144+rsi] + mov r11,QWORD PTR[152+rsi] mov rbx,rdx mov r10,rax imul QWORD PTR[56+rsp] @@ -136,6 +138,7 @@ endif mov QWORD PTR[72+rdi],r9 mov QWORD PTR[80+rdi],r9 mov QWORD PTR[88+rdi],r9 + mov QWORD PTR[96+rdi],r9 lea rsi,QWORD PTR[96+rsi] mov rax,r10 @@ -146,13 +149,14 @@ endif imul rcx add r8,rax adc r9,rdx - mov QWORD PTR[96+rdi],r8 - mov QWORD PTR[104+rdi],r9 - sar r9,63 + mov QWORD PTR[104+rdi],r8 mov QWORD PTR[112+rdi],r9 + sar r9,63 mov QWORD PTR[120+rdi],r9 mov QWORD PTR[128+rdi],r9 mov QWORD PTR[136+rdi],r9 + mov QWORD PTR[144+rdi],r9 + mov QWORD PTR[152+rdi],r9 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -163,14 +167,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -178,12 +182,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -194,14 +198,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -209,12 +213,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -225,14 +229,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -240,12 +244,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -256,14 +260,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -271,12 +275,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -287,14 +291,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -302,12 +306,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -318,14 +322,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -333,12 +337,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -349,14 +353,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -364,12 +368,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -380,14 +384,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -395,12 +399,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -411,14 +415,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -426,12 +430,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -442,14 +446,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -457,19 +461,17 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 - sar r13,63 - mov QWORD PTR[48+rdi],r13 - mov QWORD PTR[56+rdi],r13 - mov QWORD PTR[64+rdi],r13 - mov QWORD PTR[72+rdi],r13 - mov QWORD PTR[80+rdi],r13 - mov QWORD PTR[88+rdi],r13 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 + mov QWORD PTR[56+rdi],r14 + mov QWORD PTR[64+rdi],r14 + mov QWORD PTR[72+rdi],r14 + mov QWORD PTR[80+rdi],r14 + mov QWORD PTR[88+rdi],r14 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -480,14 +482,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -495,12 +497,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -511,14 +513,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -526,12 +528,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -542,14 +544,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -557,12 +559,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -573,14 +575,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -588,12 +590,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -604,14 +606,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -619,12 +621,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -635,14 +637,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -650,12 +652,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -666,14 +668,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -681,12 +683,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -712,12 +714,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -743,12 +745,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -774,12 +776,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -805,21 +807,21 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 - mov edi,53 + mov edi,55 mov r8,QWORD PTR[rsi] mov r10,QWORD PTR[48+rsi] - call __tail_loop_53 + call __tail_loop_55 @@ -836,40 +838,80 @@ endif mov rdx,r12 mov rcx,r13 mov rdi,QWORD PTR[32+rsp] - call __smulx_767x63 + call __smulx_768x63 mov rsi,QWORD PTR[40+rsp] - mov rdx,rax - sar rax,63 + mov r13,rdx + sar r13,63 - mov r8,rax - mov r9,rax - mov r10,rax + mov r8,r13 + mov r9,r13 + mov r10,r13 ifdef __SGX_LVI_HARDENING__ lfence endif and r8,QWORD PTR[rsi] and r9,QWORD PTR[8+rsi] - mov r11,rax + mov r11,r13 and r10,QWORD PTR[16+rsi] and r11,QWORD PTR[24+rsi] - mov r12,rax + mov r12,r13 and r12,QWORD PTR[32+rsi] - and rax,QWORD PTR[40+rsi] + and r13,QWORD PTR[40+rsi] add r14,r8 adc r15,r9 adc rbx,r10 adc rbp,r11 adc rcx,r12 - adc rdx,rax + adc rax,r13 + adc rdx,0 + + mov r13,rdx + neg rdx + or r13,rdx + sar rdx,63 + + mov r8,r13 + mov r9,r13 + mov r10,r13 + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,r13 + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,r13 + and r12,QWORD PTR[32+rsi] + and r13,QWORD PTR[40+rsi] + + xor r8,rdx + xor rsi,rsi + xor r9,rdx + sub rsi,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add r8,rsi + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rax,r13 mov QWORD PTR[48+rdi],r14 mov QWORD PTR[56+rdi],r15 mov QWORD PTR[64+rdi],rbx mov QWORD PTR[72+rdi],rbp mov QWORD PTR[80+rdi],rcx - mov QWORD PTR[88+rdi],rdx + mov QWORD PTR[88+rdi],rax lea r8,QWORD PTR[1112+rsp] mov r15,QWORD PTR[r8] @@ -886,7 +928,7 @@ endif lea rsp,QWORD PTR[48+r8] -$L$SEH_epilogue_ctx_inverse_mod_383:: +$L$SEH_epilogue_ctx_inverse_mod_384:: mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue mov rsi,QWORD PTR[16+rsp] @@ -900,11 +942,11 @@ else DB 0F3h,0C3h endif -$L$SEH_end_ctx_inverse_mod_383:: -ctx_inverse_mod_383 ENDP +$L$SEH_end_ctx_inverse_mod_384:: +ctx_inverse_mod_384 ENDP ALIGN 32 -__smulx_767x63 PROC PRIVATE +__smulx_768x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -913,6 +955,7 @@ __smulx_767x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rax,rdx sar rax,63 @@ -921,7 +964,7 @@ __smulx_767x63 PROC PRIVATE mov QWORD PTR[8+rsp],rdi mov QWORD PTR[16+rsp],rsi - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] xor rdx,rax add rdx,rbp @@ -931,37 +974,41 @@ __smulx_767x63 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax + xor r14,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx rax,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,rax + mulx rax,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc rdx,0 + adc r12,rax + mulx rax,r13,r13 + adc r13,rbp + adc r14,rax mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax - mov QWORD PTR[48+rdi],rdx - sar rdx,63 - mov QWORD PTR[56+rdi],rdx + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + sar r14,63 + mov QWORD PTR[56+rdi],r14 mov rdx,rcx mov rax,rcx @@ -996,7 +1043,7 @@ __smulx_767x63 PROC PRIVATE xor rbx,rax xor rbp,rax xor rcx,rax - xor rdi,rax + xor rax,rdi add r8,rsi adc r9,0 adc r10,0 @@ -1008,62 +1055,64 @@ __smulx_767x63 PROC PRIVATE adc rbx,0 adc rbp,0 adc rcx,0 - adc rdi,0 - - mulx rax,r8,r8 - mulx rsi,r9,r9 - add r9,rax - mulx rax,r10,r10 - adc r10,rsi - mulx rsi,r11,r11 - adc r11,rax - mulx rax,r12,r12 - adc r12,rsi - mulx rsi,r13,r13 - adc r13,rax - mulx rax,r14,r14 - adc r14,rsi - mulx rsi,r15,r15 - adc r15,rax - mulx rax,rbx,rbx + adc rax,0 + + mulx rsi,r8,r8 + mulx rdi,r9,r9 + add r9,rsi + mulx rsi,r10,r10 + adc r10,rdi + mulx rdi,r11,r11 + adc r11,rsi + mulx rsi,r12,r12 + adc r12,rdi + mulx rdi,r13,r13 + adc r13,rsi + mulx rsi,r14,r14 + adc r14,rdi + mulx rdi,r15,r15 + adc r15,rsi + mulx rsi,rbx,rbx + adc rbx,rdi + mulx rdi,rbp,rbp + adc rbp,rsi + mulx rsi,rcx,rcx + adc rcx,rdi + mov rdi,QWORD PTR[8+rsp] + adc rsi,0 + imul rdx + add rax,rsi + adc rdx,0 + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + adc r14,QWORD PTR[48+rdi] + mov rsi,QWORD PTR[56+rdi] + adc r15,rsi adc rbx,rsi - mulx rsi,rbp,rbp - adc rbp,rax - mulx rax,rcx,rcx + adc rbp,rsi adc rcx,rsi - mulx rsi,rdi,rdi - mov rdx,QWORD PTR[8+rsp] + adc rax,rsi + adc rdx,rsi + mov rsi,QWORD PTR[16+rsp] - adc rax,rdi - - add r8,QWORD PTR[rdx] - adc r9,QWORD PTR[8+rdx] - adc r10,QWORD PTR[16+rdx] - adc r11,QWORD PTR[24+rdx] - adc r12,QWORD PTR[32+rdx] - adc r13,QWORD PTR[40+rdx] - adc r14,QWORD PTR[48+rdx] - mov rdi,QWORD PTR[56+rdx] - adc r15,rdi - adc rbx,rdi - adc rbp,rdi - adc rcx,rdi - adc rax,rdi - - mov rdi,rdx - - mov QWORD PTR[rdx],r8 - mov QWORD PTR[8+rdx],r9 - mov QWORD PTR[16+rdx],r10 - mov QWORD PTR[24+rdx],r11 - mov QWORD PTR[32+rdx],r12 - mov QWORD PTR[40+rdx],r13 - mov QWORD PTR[48+rdx],r14 - mov QWORD PTR[56+rdx],r15 - mov QWORD PTR[64+rdx],rbx - mov QWORD PTR[72+rdx],rbp - mov QWORD PTR[80+rdx],rcx - mov QWORD PTR[88+rdx],rax + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rax ifdef __SGX_LVI_HARDENING__ @@ -1074,10 +1123,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_767x63 ENDP +__smulx_768x63 ENDP ALIGN 32 -__smulx_383x63 PROC PRIVATE +__smulx_384x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[((0+0))+rsi] @@ -1086,6 +1135,7 @@ __smulx_383x63 PROC PRIVATE mov r11,QWORD PTR[((0+24))+rsi] mov r12,QWORD PTR[((0+32))+rsi] mov r13,QWORD PTR[((0+40))+rsi] + mov r14,QWORD PTR[((0+48))+rsi] mov rbp,rdx sar rbp,63 @@ -1101,12 +1151,17 @@ __smulx_383x63 PROC PRIVATE xor r11,rbp xor r12,rbp xor r13,rbp + xor r14,rbp add r8,rax adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 mulx rax,r9,r9 @@ -1120,19 +1175,22 @@ __smulx_383x63 PROC PRIVATE mulx rax,r13,r13 mov rdx,rcx adc r13,rbp + adc r14,rax mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],r13 - mov r8,QWORD PTR[((48+0))+rsi] - mov r9,QWORD PTR[((48+8))+rsi] - mov r10,QWORD PTR[((48+16))+rsi] - mov r11,QWORD PTR[((48+24))+rsi] - mov r12,QWORD PTR[((48+32))+rsi] - mov r13,QWORD PTR[((48+40))+rsi] + mov r15,r13 + mov rbx,r14 + mov r8,QWORD PTR[((56+0))+rsi] + mov r9,QWORD PTR[((56+8))+rsi] + mov r10,QWORD PTR[((56+16))+rsi] + mov r11,QWORD PTR[((56+24))+rsi] + mov r12,QWORD PTR[((56+32))+rsi] + mov r13,QWORD PTR[((56+40))+rsi] + mov r14,QWORD PTR[((56+48))+rsi] mov rbp,rdx sar rbp,63 @@ -1148,12 +1206,17 @@ __smulx_383x63 PROC PRIVATE xor r11,rbp xor r12,rbp xor r13,rbp + xor r14,rbp add r8,rax adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 mulx rax,r9,r9 @@ -1166,13 +1229,15 @@ __smulx_383x63 PROC PRIVATE adc r12,rax mulx rax,r13,r13 adc r13,rbp + adc r14,rax add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc r13,QWORD PTR[40+rdi] + adc r13,r15 + adc r14,rbx mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 @@ -1180,6 +1245,7 @@ __smulx_383x63 PROC PRIVATE mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 ifdef __SGX_LVI_HARDENING__ @@ -1190,14 +1256,13 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_383x63 ENDP +__smulx_384x63 ENDP ALIGN 32 -__smulx_383_n_shift_by_31 PROC PRIVATE +__smulx_384_n_shift_by_31 PROC PRIVATE DB 243,15,30,250 mov rbx,rdx - xor r14,r14 mov r8,QWORD PTR[((0+0))+rsi] mov r9,QWORD PTR[((0+8))+rsi] mov r10,QWORD PTR[((0+16))+rsi] @@ -1218,27 +1283,29 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + + and rax,rdx + neg rax mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx r14,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,r14 + mulx r14,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc r14,rdx + adc r12,r14 + mulx r14,r13,r13 + adc r13,rbp + adc r14,rax mov rdx,rcx @@ -1247,7 +1314,8 @@ __smulx_383_n_shift_by_31 PROC PRIVATE mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax + mov QWORD PTR[40+rdi],r13 + mov r15,r14 mov r8,QWORD PTR[((48+0))+rsi] mov r9,QWORD PTR[((48+8))+rsi] mov r10,QWORD PTR[((48+16))+rsi] @@ -1268,43 +1336,45 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + + and rax,rdx + neg rax mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx r14,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,r14 + mulx r14,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc rdx,0 + adc r12,r14 + mulx r14,r13,r13 + adc r13,rbp + adc r14,rax add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc rax,QWORD PTR[40+rdi] - adc r14,rdx + adc r13,QWORD PTR[40+rdi] + adc r14,r15 mov rdx,rbx shrd r8,r9,31 shrd r9,r10,31 shrd r10,r11,31 shrd r11,r12,31 - shrd r12,rax,31 - shrd rax,r14,31 + shrd r12,r13,31 + shrd r13,r14,31 sar r14,63 xor rbp,rbp @@ -1315,20 +1385,20 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,r14 xor r11,r14 xor r12,r14 - xor rax,r14 + xor r13,r14 add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax + mov QWORD PTR[40+rdi],r13 xor rdx,r14 xor rcx,r14 @@ -1344,7 +1414,7 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_383_n_shift_by_31 ENDP +__smulx_384_n_shift_by_31 ENDP ALIGN 32 __smulx_191_n_shift_by_31 PROC PRIVATE @@ -1583,7 +1653,7 @@ __inner_loop_31 ENDP ALIGN 32 -__tail_loop_53 PROC PRIVATE +__tail_loop_55 PROC PRIVATE DB 243,15,30,250 mov rdx,1 @@ -1591,7 +1661,7 @@ __tail_loop_53 PROC PRIVATE xor r12,r12 mov r13,1 -$L$oop_53:: +$L$oop_55:: xor rax,rax test r8,1 mov rbx,r10 @@ -1618,7 +1688,7 @@ $L$oop_53:: sub rdx,rax sub rcx,rbx sub edi,1 - jnz $L$oop_53 + jnz $L$oop_55 ifdef __SGX_LVI_HARDENING__ @@ -1629,33 +1699,33 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__tail_loop_53 ENDP +__tail_loop_55 ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 - DD imagerel $L$SEH_begin_ctx_inverse_mod_383 - DD imagerel $L$SEH_body_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_prologue + DD imagerel $L$SEH_begin_ctx_inverse_mod_384 + DD imagerel $L$SEH_body_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_prologue - DD imagerel $L$SEH_body_ctx_inverse_mod_383 - DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_body + DD imagerel $L$SEH_body_ctx_inverse_mod_384 + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_body - DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 - DD imagerel $L$SEH_end_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_epilogue + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_384 + DD imagerel $L$SEH_end_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_epilogue .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 -$L$SEH_info_ctx_inverse_mod_383_prologue:: +$L$SEH_info_ctx_inverse_mod_384_prologue:: DB 1,0,5,00bh DB 0,074h,1,0 DB 0,064h,2,0 DB 0,0b3h DB 0,0 DD 0,0 -$L$SEH_info_ctx_inverse_mod_383_body:: +$L$SEH_info_ctx_inverse_mod_384_body:: DB 1,0,18,0 DB 000h,0f4h,08bh,000h DB 000h,0e4h,08ch,000h @@ -1668,7 +1738,7 @@ DB 000h,064h,093h,000h DB 000h,001h,091h,000h DB 000h,000h,000h,000h DB 000h,000h,000h,000h -$L$SEH_info_ctx_inverse_mod_383_epilogue:: +$L$SEH_info_ctx_inverse_mod_384_epilogue:: DB 1,0,4,0 DB 000h,074h,001h,000h DB 000h,064h,002h,000h diff --git a/blst_src/build/win64/div3w-armv8.asm b/blst_src/build/win64/div3w-armv8.asm index 25659de2..27c37e2b 100644 --- a/blst_src/build/win64/div3w-armv8.asm +++ b/blst_src/build/win64/div3w-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |div_3_limbs|[FUNC] ALIGN 32 |div_3_limbs| PROC + hint #34 ldp x4,x5,[x0] eor x0,x0,x0 mov x3,#64 @@ -41,6 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |quot_rem_128|[FUNC] ALIGN 32 |quot_rem_128| PROC + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 @@ -78,6 +80,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |quot_rem_64|[FUNC] ALIGN 32 |quot_rem_64| PROC + hint #34 ldr x3,[x1] ldr x8,[x0] diff --git a/blst_src/build/win64/mul_mont_256-armv8.asm b/blst_src/build/win64/mul_mont_256-armv8.asm index 9bb3987c..54387a09 100644 --- a/blst_src/build/win64/mul_mont_256-armv8.asm +++ b/blst_src/build/win64/mul_mont_256-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_sparse_256|[FUNC] ALIGN 32 |mul_mont_sparse_256| PROC + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -198,7 +199,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_sparse_256|[FUNC] ALIGN 32 |sqr_mont_sparse_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -299,7 +300,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -307,7 +308,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |from_mont_256|[FUNC] ALIGN 32 |from_mont_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -332,7 +333,7 @@ __SIZEOF_POINTER__ SETA 64/8 stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -341,7 +342,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |redc_mont_256|[FUNC] ALIGN 32 |redc_mont_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -376,7 +377,7 @@ __SIZEOF_POINTER__ SETA 64/8 stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP diff --git a/blst_src/build/win64/mul_mont_384-armv8.asm b/blst_src/build/win64/mul_mont_384-armv8.asm index d1213a90..26032557 100644 --- a/blst_src/build/win64/mul_mont_384-armv8.asm +++ b/blst_src/build/win64/mul_mont_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384x384|[FUNC] ALIGN 32 |add_mod_384x384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -89,7 +89,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384x384|[FUNC] ALIGN 32 |sub_mod_384x384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -107,7 +107,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -243,7 +243,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_384x|[FUNC] ALIGN 32 |mul_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -316,7 +316,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -325,7 +325,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_384x|[FUNC] ALIGN 32 |sqr_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -401,7 +401,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -410,7 +410,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_384|[FUNC] ALIGN 32 |mul_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -442,7 +442,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -823,7 +823,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_384|[FUNC] ALIGN 32 |sqr_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -860,7 +860,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -869,7 +869,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_n_mul_mont_383|[FUNC] ALIGN 32 |sqr_n_mul_mont_383| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -925,7 +925,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1046,7 +1046,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_384|[FUNC] ALIGN 32 |sqr_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1068,7 +1068,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1077,7 +1077,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |redc_mont_384|[FUNC] ALIGN 32 |redc_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1101,7 +1101,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1110,7 +1110,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |from_mont_384|[FUNC] ALIGN 32 |from_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1151,7 +1151,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1369,7 +1369,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_384|[FUNC] ALIGN 32 |mul_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1387,7 +1387,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1573,7 +1573,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_382x|[FUNC] ALIGN 32 |mul_382x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1655,7 +1655,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1664,7 +1664,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_382x|[FUNC] ALIGN 32 |sqr_382x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1755,471 +1755,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 - ret - ENDP - - - - EXPORT |sqr_mont_382x|[FUNC] - ALIGN 32 -|sqr_mont_382x| PROC - DCDU 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] - sub sp,sp,#112 - mov x4,x3 - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred - - adds x19,x11,x11 - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 - ret - ENDP - - - ALIGN 32 -|__mul_mont_383_nonred| PROC - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret ENDP @@ -2228,7 +1764,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mont_384|[FUNC] ALIGN 32 |sgn0_pty_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2273,7 +1809,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -2282,7 +1818,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mont_384x|[FUNC] ALIGN 32 |sgn0_pty_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2371,7 +1907,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP END diff --git a/blst_src/build/win64/mulq_mont_256-x86_64.asm b/blst_src/build/win64/mulq_mont_256-x86_64.asm index 70fa3c4b..9f30eb9b 100644 --- a/blst_src/build/win64/mulq_mont_256-x86_64.asm +++ b/blst_src/build/win64/mulq_mont_256-x86_64.asm @@ -1,8 +1,10 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ EXTERN mul_mont_sparse_256$1:NEAR EXTERN sqr_mont_sparse_256$1:NEAR EXTERN from_mont_256$1:NEAR EXTERN redc_mont_256$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS diff --git a/blst_src/build/win64/mulq_mont_384-x86_64.asm b/blst_src/build/win64/mulq_mont_384-x86_64.asm index 1c06947d..7e52c08a 100644 --- a/blst_src/build/win64/mulq_mont_384-x86_64.asm +++ b/blst_src/build/win64/mulq_mont_384-x86_64.asm @@ -1,4 +1,5 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ EXTERN mul_mont_384x$1:NEAR EXTERN sqr_mont_384x$1:NEAR EXTERN mul_382x$1:NEAR @@ -13,7 +14,7 @@ EXTERN mul_mont_384$1:NEAR EXTERN sqr_mont_384$1:NEAR EXTERN sqr_n_mul_mont_384$1:NEAR EXTERN sqr_n_mul_mont_383$1:NEAR -EXTERN sqr_mont_382x$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS @@ -3190,890 +3191,124 @@ endif $L$SEH_end_sqr_n_mul_mont_383:: sqr_n_mul_mont_383 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mul_mont_384x + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_prologue -ALIGN 32 -__mulq_mont_383_nonred PROC PRIVATE - DB 243,15,30,250 - - mov rbp,rax - mul r14 - mov r8,rax - mov rax,rbp - mov r9,rdx + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_body - mul r15 - add r9,rax - mov rax,rbp - adc rdx,0 - mov r10,rdx + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_end_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_epilogue - mul r12 - add r10,rax - mov rax,rbp - adc rdx,0 - mov r11,rdx + DD imagerel $L$SEH_begin_sqr_mont_384x + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_prologue - mov r15,r8 - imul r8,QWORD PTR[8+rsp] + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_body - mul r13 - add r11,rax - mov rax,rbp - adc rdx,0 - mov r12,rdx + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_end_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_epilogue - mul QWORD PTR[32+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - mov r13,rdx + DD imagerel $L$SEH_begin_mul_382x + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_info_mul_382x_prologue - mul QWORD PTR[40+rsi] - add r13,rax - mov rax,r8 - adc rdx,0 - mov r14,rdx + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_info_mul_382x_body - mul QWORD PTR[rcx] - add r15,rax - mov rax,r8 - adc r15,rdx + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_end_mul_382x + DD imagerel $L$SEH_info_mul_382x_epilogue - mul QWORD PTR[8+rcx] - add r9,rax - mov rax,r8 - adc rdx,0 - add r9,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_382x + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_prologue - mul QWORD PTR[16+rcx] - add r10,rax - mov rax,r8 - adc rdx,0 - add r10,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_body - mul QWORD PTR[24+rcx] - add r11,r15 - adc rdx,0 - add r11,rax - mov rax,r8 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_end_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_epilogue - mul QWORD PTR[32+rcx] - add r12,rax - mov rax,r8 - adc rdx,0 - add r12,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_mul_384 + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_info_mul_384_prologue - mul QWORD PTR[40+rcx] - add r13,rax - mov rax,QWORD PTR[8+rbx] - adc rdx,0 - add r13,r15 - adc r14,rdx + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_info_mul_384_body - mov rbp,rax - mul QWORD PTR[rsi] - add r9,rax - mov rax,rbp - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_end_mul_384 + DD imagerel $L$SEH_info_mul_384_epilogue - mul QWORD PTR[8+rsi] - add r10,rax - mov rax,rbp - adc rdx,0 - add r10,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_384 + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_info_sqr_384_prologue - mul QWORD PTR[16+rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - add r11,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_info_sqr_384_body - mov r8,r9 - imul r9,QWORD PTR[8+rsp] + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_end_sqr_384 + DD imagerel $L$SEH_info_sqr_384_epilogue - mul QWORD PTR[24+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_mont_384 + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_prologue - mul QWORD PTR[32+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_body - mul QWORD PTR[40+rsi] - add r14,r15 - adc rdx,0 - add r14,rax - mov rax,r9 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_end_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_epilogue - mul QWORD PTR[rcx] - add r8,rax - mov rax,r9 - adc r8,rdx + DD imagerel $L$SEH_begin_redc_mont_384 + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_prologue - mul QWORD PTR[8+rcx] - add r10,rax - mov rax,r9 - adc rdx,0 - add r10,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_body - mul QWORD PTR[16+rcx] - add r11,rax - mov rax,r9 - adc rdx,0 - add r11,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_end_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_epilogue - mul QWORD PTR[24+rcx] - add r12,r8 - adc rdx,0 - add r12,rax - mov rax,r9 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_begin_from_mont_384 + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_prologue - mul QWORD PTR[32+rcx] - add r13,rax - mov rax,r9 - adc rdx,0 - add r13,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_body - mul QWORD PTR[40+rcx] - add r14,rax - mov rax,QWORD PTR[16+rbx] - adc rdx,0 - add r14,r8 - adc r15,rdx + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_end_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_epilogue - mov rbp,rax - mul QWORD PTR[rsi] - add r10,rax - mov rax,rbp - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_begin_sgn0_pty_mont_384 + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_prologue - mul QWORD PTR[8+rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - add r11,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[16+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r8 - adc rdx,0 - mov r8,rdx - - mov r9,r10 - imul r10,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[32+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[40+rsi] - add r15,r8 - adc rdx,0 - add r15,rax - mov rax,r10 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[rcx] - add r9,rax - mov rax,r10 - adc r9,rdx - - mul QWORD PTR[8+rcx] - add r11,rax - mov rax,r10 - adc rdx,0 - add r11,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[16+rcx] - add r12,rax - mov rax,r10 - adc rdx,0 - add r12,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[24+rcx] - add r13,r9 - adc rdx,0 - add r13,rax - mov rax,r10 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[32+rcx] - add r14,rax - mov rax,r10 - adc rdx,0 - add r14,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[40+rcx] - add r15,rax - mov rax,QWORD PTR[24+rbx] - adc rdx,0 - add r15,r9 - adc r8,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[8+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[16+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r9 - adc rdx,0 - mov r9,rdx - - mov r10,r11 - imul r11,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[32+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[40+rsi] - add r8,r9 - adc rdx,0 - add r8,rax - mov rax,r11 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[rcx] - add r10,rax - mov rax,r11 - adc r10,rdx - - mul QWORD PTR[8+rcx] - add r12,rax - mov rax,r11 - adc rdx,0 - add r12,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[16+rcx] - add r13,rax - mov rax,r11 - adc rdx,0 - add r13,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[24+rcx] - add r14,r10 - adc rdx,0 - add r14,rax - mov rax,r11 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[32+rcx] - add r15,rax - mov rax,r11 - adc rdx,0 - add r15,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[40+rcx] - add r8,rax - mov rax,QWORD PTR[32+rbx] - adc rdx,0 - add r8,r10 - adc r9,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[8+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[16+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r10 - adc rdx,0 - mov r10,rdx - - mov r11,r12 - imul r12,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[32+rsi] - add r8,rax - mov rax,rbp - adc rdx,0 - add r8,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[40+rsi] - add r9,r10 - adc rdx,0 - add r9,rax - mov rax,r12 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[rcx] - add r11,rax - mov rax,r12 - adc r11,rdx - - mul QWORD PTR[8+rcx] - add r13,rax - mov rax,r12 - adc rdx,0 - add r13,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[16+rcx] - add r14,rax - mov rax,r12 - adc rdx,0 - add r14,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[24+rcx] - add r15,r11 - adc rdx,0 - add r15,rax - mov rax,r12 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[32+rcx] - add r8,rax - mov rax,r12 - adc rdx,0 - add r8,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[40+rcx] - add r9,rax - mov rax,QWORD PTR[40+rbx] - adc rdx,0 - add r9,r11 - adc r10,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[8+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[16+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r11 - adc rdx,0 - mov r11,rdx - - mov r12,r13 - imul r13,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r8,rax - mov rax,rbp - adc rdx,0 - add r8,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[32+rsi] - add r9,rax - mov rax,rbp - adc rdx,0 - add r9,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[40+rsi] - add r10,r11 - adc rdx,0 - add r10,rax - mov rax,r13 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[rcx] - add r12,rax - mov rax,r13 - adc r12,rdx - - mul QWORD PTR[8+rcx] - add r14,rax - mov rax,r13 - adc rdx,0 - add r14,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[16+rcx] - add r15,rax - mov rax,r13 - adc rdx,0 - add r15,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[24+rcx] - add r8,r12 - adc rdx,0 - add r8,rax - mov rax,r13 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[32+rcx] - add r9,rax - mov rax,r13 - adc rdx,0 - add r9,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[40+rcx] - add r10,rax - mov rax,r14 - adc rdx,0 - add r10,r12 - adc r11,rdx - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif -__mulq_mont_383_nonred ENDP -PUBLIC sqr_mont_382x - - -ALIGN 32 -sqr_mont_382x PROC PUBLIC - DB 243,15,30,250 - mov QWORD PTR[8+rsp],rdi ;WIN64 prologue - mov QWORD PTR[16+rsp],rsi - mov r11,rsp -$L$SEH_begin_sqr_mont_382x:: - - - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 -ifdef __BLST_PORTABLE__ - test DWORD PTR[__blst_platform_cap],1 - jnz sqr_mont_382x$1 -endif - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,136 - -$L$SEH_body_sqr_mont_382x:: - - - mov QWORD PTR[rsp],rcx - mov rcx,rdx - mov QWORD PTR[16+rsp],rsi - mov QWORD PTR[24+rsp],rdi - - - mov r8,QWORD PTR[rsi] - mov r9,QWORD PTR[8+rsi] - mov r10,QWORD PTR[16+rsi] - mov r11,QWORD PTR[24+rsi] - mov r12,QWORD PTR[32+rsi] - mov r13,QWORD PTR[40+rsi] - - mov r14,r8 - add r8,QWORD PTR[48+rsi] - mov r15,r9 - adc r9,QWORD PTR[56+rsi] - mov rax,r10 - adc r10,QWORD PTR[64+rsi] - mov rdx,r11 - adc r11,QWORD PTR[72+rsi] - mov rbx,r12 - adc r12,QWORD PTR[80+rsi] - mov rbp,r13 - adc r13,QWORD PTR[88+rsi] - - sub r14,QWORD PTR[48+rsi] - sbb r15,QWORD PTR[56+rsi] - sbb rax,QWORD PTR[64+rsi] - sbb rdx,QWORD PTR[72+rsi] - sbb rbx,QWORD PTR[80+rsi] - sbb rbp,QWORD PTR[88+rsi] - sbb rdi,rdi - - mov QWORD PTR[((32+0))+rsp],r8 - mov QWORD PTR[((32+8))+rsp],r9 - mov QWORD PTR[((32+16))+rsp],r10 - mov QWORD PTR[((32+24))+rsp],r11 - mov QWORD PTR[((32+32))+rsp],r12 - mov QWORD PTR[((32+40))+rsp],r13 - - mov QWORD PTR[((32+48))+rsp],r14 - mov QWORD PTR[((32+56))+rsp],r15 - mov QWORD PTR[((32+64))+rsp],rax - mov QWORD PTR[((32+72))+rsp],rdx - mov QWORD PTR[((32+80))+rsp],rbx - mov QWORD PTR[((32+88))+rsp],rbp - mov QWORD PTR[((32+96))+rsp],rdi - - - - lea rbx,QWORD PTR[48+rsi] - - mov rax,QWORD PTR[48+rsi] - mov r14,QWORD PTR[rsi] - mov r15,QWORD PTR[8+rsi] - mov r12,QWORD PTR[16+rsi] - mov r13,QWORD PTR[24+rsi] - - mov rdi,QWORD PTR[24+rsp] - call __mulq_mont_383_nonred - add r14,r14 - adc r15,r15 - adc r8,r8 - adc r9,r9 - adc r10,r10 - adc r11,r11 - - mov QWORD PTR[48+rdi],r14 - mov QWORD PTR[56+rdi],r15 - mov QWORD PTR[64+rdi],r8 - mov QWORD PTR[72+rdi],r9 - mov QWORD PTR[80+rdi],r10 - mov QWORD PTR[88+rdi],r11 - - lea rsi,QWORD PTR[32+rsp] - lea rbx,QWORD PTR[((32+48))+rsp] - - mov rax,QWORD PTR[((32+48))+rsp] - mov r14,QWORD PTR[((32+0))+rsp] - mov r15,QWORD PTR[((32+8))+rsp] - mov r12,QWORD PTR[((32+16))+rsp] - mov r13,QWORD PTR[((32+24))+rsp] - - call __mulq_mont_383_nonred - mov rsi,QWORD PTR[((32+96))+rsp] - mov r12,QWORD PTR[((32+0))+rsp] - mov r13,QWORD PTR[((32+8))+rsp] - and r12,rsi - mov rax,QWORD PTR[((32+16))+rsp] - and r13,rsi - mov rbx,QWORD PTR[((32+24))+rsp] - and rax,rsi - mov rbp,QWORD PTR[((32+32))+rsp] - and rbx,rsi - and rbp,rsi - and rsi,QWORD PTR[((32+40))+rsp] - - sub r14,r12 - mov r12,QWORD PTR[rcx] - sbb r15,r13 - mov r13,QWORD PTR[8+rcx] - sbb r8,rax - mov rax,QWORD PTR[16+rcx] - sbb r9,rbx - mov rbx,QWORD PTR[24+rcx] - sbb r10,rbp - mov rbp,QWORD PTR[32+rcx] - sbb r11,rsi - sbb rsi,rsi - - and r12,rsi - and r13,rsi - and rax,rsi - and rbx,rsi - and rbp,rsi - and rsi,QWORD PTR[40+rcx] - - add r14,r12 - adc r15,r13 - adc r8,rax - adc r9,rbx - adc r10,rbp - adc r11,rsi - - mov QWORD PTR[rdi],r14 - mov QWORD PTR[8+rdi],r15 - mov QWORD PTR[16+rdi],r8 - mov QWORD PTR[24+rdi],r9 - mov QWORD PTR[32+rdi],r10 - mov QWORD PTR[40+rdi],r11 - lea r8,QWORD PTR[136+rsp] - mov r15,QWORD PTR[r8] - - mov r14,QWORD PTR[8+r8] - - mov r13,QWORD PTR[16+r8] - - mov r12,QWORD PTR[24+r8] - - mov rbx,QWORD PTR[32+r8] - - mov rbp,QWORD PTR[40+r8] - - lea rsp,QWORD PTR[48+r8] - -$L$SEH_epilogue_sqr_mont_382x:: - mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue - mov rsi,QWORD PTR[16+rsp] - - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif - -$L$SEH_end_sqr_mont_382x:: -sqr_mont_382x ENDP -.text$ ENDS -.pdata SEGMENT READONLY ALIGN(4) -ALIGN 4 - DD imagerel $L$SEH_begin_mul_mont_384x - DD imagerel $L$SEH_body_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_prologue - - DD imagerel $L$SEH_body_mul_mont_384x - DD imagerel $L$SEH_epilogue_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_body - - DD imagerel $L$SEH_epilogue_mul_mont_384x - DD imagerel $L$SEH_end_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_epilogue - - DD imagerel $L$SEH_begin_sqr_mont_384x - DD imagerel $L$SEH_body_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_prologue - - DD imagerel $L$SEH_body_sqr_mont_384x - DD imagerel $L$SEH_epilogue_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_body - - DD imagerel $L$SEH_epilogue_sqr_mont_384x - DD imagerel $L$SEH_end_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_epilogue - - DD imagerel $L$SEH_begin_mul_382x - DD imagerel $L$SEH_body_mul_382x - DD imagerel $L$SEH_info_mul_382x_prologue - - DD imagerel $L$SEH_body_mul_382x - DD imagerel $L$SEH_epilogue_mul_382x - DD imagerel $L$SEH_info_mul_382x_body - - DD imagerel $L$SEH_epilogue_mul_382x - DD imagerel $L$SEH_end_mul_382x - DD imagerel $L$SEH_info_mul_382x_epilogue - - DD imagerel $L$SEH_begin_sqr_382x - DD imagerel $L$SEH_body_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_prologue - - DD imagerel $L$SEH_body_sqr_382x - DD imagerel $L$SEH_epilogue_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_body - - DD imagerel $L$SEH_epilogue_sqr_382x - DD imagerel $L$SEH_end_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_epilogue - - DD imagerel $L$SEH_begin_mul_384 - DD imagerel $L$SEH_body_mul_384 - DD imagerel $L$SEH_info_mul_384_prologue - - DD imagerel $L$SEH_body_mul_384 - DD imagerel $L$SEH_epilogue_mul_384 - DD imagerel $L$SEH_info_mul_384_body - - DD imagerel $L$SEH_epilogue_mul_384 - DD imagerel $L$SEH_end_mul_384 - DD imagerel $L$SEH_info_mul_384_epilogue - - DD imagerel $L$SEH_begin_sqr_384 - DD imagerel $L$SEH_body_sqr_384 - DD imagerel $L$SEH_info_sqr_384_prologue - - DD imagerel $L$SEH_body_sqr_384 - DD imagerel $L$SEH_epilogue_sqr_384 - DD imagerel $L$SEH_info_sqr_384_body - - DD imagerel $L$SEH_epilogue_sqr_384 - DD imagerel $L$SEH_end_sqr_384 - DD imagerel $L$SEH_info_sqr_384_epilogue - - DD imagerel $L$SEH_begin_sqr_mont_384 - DD imagerel $L$SEH_body_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_prologue - - DD imagerel $L$SEH_body_sqr_mont_384 - DD imagerel $L$SEH_epilogue_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_body - - DD imagerel $L$SEH_epilogue_sqr_mont_384 - DD imagerel $L$SEH_end_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_epilogue - - DD imagerel $L$SEH_begin_redc_mont_384 - DD imagerel $L$SEH_body_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_prologue - - DD imagerel $L$SEH_body_redc_mont_384 - DD imagerel $L$SEH_epilogue_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_body - - DD imagerel $L$SEH_epilogue_redc_mont_384 - DD imagerel $L$SEH_end_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_epilogue - - DD imagerel $L$SEH_begin_from_mont_384 - DD imagerel $L$SEH_body_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_prologue - - DD imagerel $L$SEH_body_from_mont_384 - DD imagerel $L$SEH_epilogue_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_body - - DD imagerel $L$SEH_epilogue_from_mont_384 - DD imagerel $L$SEH_end_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_epilogue - - DD imagerel $L$SEH_begin_sgn0_pty_mont_384 - DD imagerel $L$SEH_body_sgn0_pty_mont_384 - DD imagerel $L$SEH_info_sgn0_pty_mont_384_prologue - - DD imagerel $L$SEH_body_sgn0_pty_mont_384 - DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 - DD imagerel $L$SEH_info_sgn0_pty_mont_384_body + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_body DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 DD imagerel $L$SEH_end_sgn0_pty_mont_384 @@ -4127,18 +3362,6 @@ ALIGN 4 DD imagerel $L$SEH_end_sqr_n_mul_mont_383 DD imagerel $L$SEH_info_sqr_n_mul_mont_383_epilogue - DD imagerel $L$SEH_begin_sqr_mont_382x - DD imagerel $L$SEH_body_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_prologue - - DD imagerel $L$SEH_body_sqr_mont_382x - DD imagerel $L$SEH_epilogue_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_body - - DD imagerel $L$SEH_epilogue_sqr_mont_382x - DD imagerel $L$SEH_end_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_epilogue - .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 @@ -4502,32 +3725,6 @@ DB 000h,074h,001h,000h DB 000h,064h,002h,000h DB 000h,000h,000h,000h -$L$SEH_info_sqr_mont_382x_prologue:: -DB 1,0,5,00bh -DB 0,074h,1,0 -DB 0,064h,2,0 -DB 0,0b3h -DB 0,0 - DD 0,0 -$L$SEH_info_sqr_mont_382x_body:: -DB 1,0,18,0 -DB 000h,0f4h,011h,000h -DB 000h,0e4h,012h,000h -DB 000h,0d4h,013h,000h -DB 000h,0c4h,014h,000h -DB 000h,034h,015h,000h -DB 000h,054h,016h,000h -DB 000h,074h,018h,000h -DB 000h,064h,019h,000h -DB 000h,001h,017h,000h -DB 000h,000h,000h,000h -DB 000h,000h,000h,000h -$L$SEH_info_sqr_mont_382x_epilogue:: -DB 1,0,4,0 -DB 000h,074h,001h,000h -DB 000h,064h,002h,000h -DB 000h,000h,000h,000h - .xdata ENDS END diff --git a/blst_src/build/win64/mulx_mont_256-x86_64.asm b/blst_src/build/win64/mulx_mont_256-x86_64.asm index 19946a28..f1dd4e46 100644 --- a/blst_src/build/win64/mulx_mont_256-x86_64.asm +++ b/blst_src/build/win64/mulx_mont_256-x86_64.asm @@ -1,8 +1,10 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ PUBLIC mul_mont_sparse_256$1 PUBLIC sqr_mont_sparse_256$1 PUBLIC from_mont_256$1 PUBLIC redc_mont_256$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' PUBLIC mulx_mont_sparse_256 @@ -219,17 +221,14 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rbp - adox r14,r9 - adcx r14,r10 - adox r15,r10 + adox r9,r10 + adcx r14,r9 adcx r15,r10 - adox r10,r10 - adc r10,0 mov r11,rax imul rax,r8 - xor rbp,rbp + xor r10,r10 mulx r9,rbp,QWORD PTR[((0+128))+rsi] adox r12,rbp adcx r13,r9 @@ -264,17 +263,14 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rbp - adox r15,r9 - adcx r15,r11 - adox r10,r11 + adox r9,r11 + adcx r15,r9 adcx r10,r11 - adox r11,r11 - adc r11,0 mov r12,rax imul rax,r8 - xor rbp,rbp + xor r11,r11 mulx r9,rbp,QWORD PTR[((0+128))+rsi] adox r13,rbp adcx r14,r9 @@ -309,16 +305,13 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,rax adcx r15,rbp - adox r10,r9 - adcx r10,r12 - adox r11,r12 + adox r9,r12 + adcx r10,r9 adcx r11,r12 - adox r12,r12 - adc r12,0 imul rdx,r8 - xor rbp,rbp + xor r12,r12 mulx r9,r13,QWORD PTR[((0+128))+rcx] adcx r13,rax adox r14,r9 @@ -335,11 +328,10 @@ __mulx_mont_sparse_256 PROC PRIVATE mov rdx,r14 lea rcx,QWORD PTR[128+rcx] adcx r10,rbp - adox r11,r9 + adox r9,r13 mov rax,r15 - adcx r11,r13 - adox r12,r13 - adc r12,0 + adcx r11,r9 + adcx r12,r13 diff --git a/blst_src/build/win64/mulx_mont_384-x86_64.asm b/blst_src/build/win64/mulx_mont_384-x86_64.asm index 8a6752f2..02b820ad 100644 --- a/blst_src/build/win64/mulx_mont_384-x86_64.asm +++ b/blst_src/build/win64/mulx_mont_384-x86_64.asm @@ -1,4 +1,5 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ PUBLIC mul_mont_384x$1 PUBLIC sqr_mont_384x$1 PUBLIC mul_382x$1 @@ -13,7 +14,7 @@ PUBLIC mul_mont_384$1 PUBLIC sqr_mont_384$1 PUBLIC sqr_n_mul_mont_384$1 PUBLIC sqr_n_mul_mont_383$1 -PUBLIC sqr_mont_382x$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' @@ -2091,8 +2092,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r8 adox r14,rdi - adcx r15,rbp - adox r15,rax + adcx rbp,rax + adox r15,rbp adox rax,rax @@ -2120,11 +2121,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rdi - adox r14,rbp - adcx r14,r8 - adox r15,r8 + adox rbp,r8 + adcx r14,rbp adcx r15,r8 - adox rax,r8 adcx rax,r8 mov QWORD PTR[16+rsp],r9 imul r9,QWORD PTR[8+rsp] @@ -2154,8 +2153,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r9 adox r15,rdi - adcx rax,rbp - adox rax,r8 + adcx rbp,r8 + adox rax,rbp adox r8,r8 @@ -2183,11 +2182,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rdi - adox r15,rbp - adcx r15,r9 - adox rax,r9 + adox rbp,r9 + adcx r15,rbp adcx rax,r9 - adox r8,r9 adcx r8,r9 mov QWORD PTR[16+rsp],r10 imul r10,QWORD PTR[8+rsp] @@ -2217,8 +2214,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r10 adox rax,rdi - adcx r8,rbp - adox r8,r9 + adcx rbp,r9 + adox r8,rbp adox r9,r9 @@ -2246,11 +2243,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[32+rbx] adcx r15,rdi - adox rax,rbp - adcx rax,r10 - adox r8,r10 + adox rbp,r10 + adcx rax,rbp adcx r8,r10 - adox r9,r10 adcx r9,r10 mov QWORD PTR[16+rsp],r11 imul r11,QWORD PTR[8+rsp] @@ -2280,8 +2275,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r11 adox r8,rdi - adcx r9,rbp - adox r9,r10 + adcx rbp,r10 + adox r9,rbp adox r10,r10 @@ -2309,11 +2304,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[40+rbx] adcx rax,rdi - adox r8,rbp - adcx r8,r11 - adox r9,r11 + adox rbp,r11 + adcx r8,rbp adcx r9,r11 - adox r10,r11 adcx r10,r11 mov QWORD PTR[16+rsp],r12 imul r12,QWORD PTR[8+rsp] @@ -2343,8 +2336,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r12 adox r9,rdi - adcx r10,rbp - adox r10,r11 + adcx rbp,r11 + adox r10,rbp adox r11,r11 @@ -2372,11 +2365,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,r13 adcx r8,rdi - adox r9,rbp - adcx r9,r12 - adox r10,r12 + adox rbp,r12 + adcx r9,rbp adcx r10,r12 - adox r11,r12 adcx r11,r12 imul rdx,QWORD PTR[8+rsp] mov rbx,QWORD PTR[24+rsp] @@ -2407,10 +2398,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] adcx r9,rdi - adox r10,rbp + adox rbp,r12 mov rdx,r14 - adcx r10,r12 - adox r11,r12 + adcx r10,rbp lea rcx,QWORD PTR[128+rcx] mov r12,r8 adc r11,0 @@ -2819,9 +2809,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rdi - adox r14,rbp - adcx r14,rax - adox r15,rax + adox rbp,rax + adcx r14,rbp adcx r15,rax mov r8,r9 imul r9,QWORD PTR[8+rsp] @@ -2879,9 +2868,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rdi - adox r15,rbp - adcx r15,r8 - adox rax,r8 + adox rbp,r8 + adcx r15,rbp adcx rax,r8 mov r9,r10 imul r10,QWORD PTR[8+rsp] @@ -2939,9 +2927,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[32+rbx] adcx r15,rdi - adox rax,rbp - adcx rax,r9 - adox r8,r9 + adox rbp,r9 + adcx rax,rbp adcx r8,r9 mov r10,r11 imul r11,QWORD PTR[8+rsp] @@ -2999,9 +2986,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[40+rbx] adcx rax,rdi - adox r8,rbp - adcx r8,r10 - adox r9,r10 + adox rbp,r10 + adcx r8,rbp adcx r9,r10 mov r11,r12 imul r12,QWORD PTR[8+rsp] @@ -3059,9 +3045,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,r13 adcx r8,rdi - adox r9,rbp - adcx r9,r11 - adox r10,r11 + adox rbp,r11 + adcx r9,rbp adcx r10,r11 imul rdx,QWORD PTR[8+rsp] mov rbx,QWORD PTR[24+rsp] @@ -3115,215 +3100,6 @@ else endif __mulx_mont_383_nonred ENDP -PUBLIC sqrx_mont_382x - - -ALIGN 32 -sqrx_mont_382x PROC PUBLIC - DB 243,15,30,250 - mov QWORD PTR[8+rsp],rdi ;WIN64 prologue - mov QWORD PTR[16+rsp],rsi - mov r11,rsp -$L$SEH_begin_sqrx_mont_382x:: - - - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 -sqr_mont_382x$1:: - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,136 - -$L$SEH_body_sqrx_mont_382x:: - - - mov QWORD PTR[rsp],rcx - mov rcx,rdx - mov QWORD PTR[16+rsp],rdi - mov QWORD PTR[24+rsp],rsi - - -ifdef __SGX_LVI_HARDENING__ - lfence -endif - mov r8,QWORD PTR[rsi] - mov r9,QWORD PTR[8+rsi] - mov r10,QWORD PTR[16+rsi] - mov r11,QWORD PTR[24+rsi] - mov r12,QWORD PTR[32+rsi] - mov r13,QWORD PTR[40+rsi] - - mov r14,r8 - add r8,QWORD PTR[48+rsi] - mov r15,r9 - adc r9,QWORD PTR[56+rsi] - mov rax,r10 - adc r10,QWORD PTR[64+rsi] - mov rdx,r11 - adc r11,QWORD PTR[72+rsi] - mov rbx,r12 - adc r12,QWORD PTR[80+rsi] - mov rbp,r13 - adc r13,QWORD PTR[88+rsi] - - sub r14,QWORD PTR[48+rsi] - sbb r15,QWORD PTR[56+rsi] - sbb rax,QWORD PTR[64+rsi] - sbb rdx,QWORD PTR[72+rsi] - sbb rbx,QWORD PTR[80+rsi] - sbb rbp,QWORD PTR[88+rsi] - sbb rdi,rdi - - mov QWORD PTR[((32+0))+rsp],r8 - mov QWORD PTR[((32+8))+rsp],r9 - mov QWORD PTR[((32+16))+rsp],r10 - mov QWORD PTR[((32+24))+rsp],r11 - mov QWORD PTR[((32+32))+rsp],r12 - mov QWORD PTR[((32+40))+rsp],r13 - - mov QWORD PTR[((32+48))+rsp],r14 - mov QWORD PTR[((32+56))+rsp],r15 - mov QWORD PTR[((32+64))+rsp],rax - mov QWORD PTR[((32+72))+rsp],rdx - mov QWORD PTR[((32+80))+rsp],rbx - mov QWORD PTR[((32+88))+rsp],rbp - mov QWORD PTR[((32+96))+rsp],rdi - - - - lea rbx,QWORD PTR[48+rsi] - - mov rdx,QWORD PTR[48+rsi] - mov r14,QWORD PTR[rsi] - mov r15,QWORD PTR[8+rsi] - mov rax,QWORD PTR[16+rsi] - mov r12,QWORD PTR[24+rsi] - mov rdi,QWORD PTR[32+rsi] - mov rbp,QWORD PTR[40+rsi] - lea rsi,QWORD PTR[((-128))+rsi] - lea rcx,QWORD PTR[((-128))+rcx] - - mulx r9,r8,r14 - call __mulx_mont_383_nonred - add rdx,rdx - adc r15,r15 - adc rax,rax - adc r12,r12 - adc rdi,rdi - adc rbp,rbp - - mov QWORD PTR[48+rbx],rdx - mov QWORD PTR[56+rbx],r15 - mov QWORD PTR[64+rbx],rax - mov QWORD PTR[72+rbx],r12 - mov QWORD PTR[80+rbx],rdi - mov QWORD PTR[88+rbx],rbp - - lea rsi,QWORD PTR[((32-128))+rsp] - lea rbx,QWORD PTR[((32+48))+rsp] - - mov rdx,QWORD PTR[((32+48))+rsp] - mov r14,QWORD PTR[((32+0))+rsp] - mov r15,QWORD PTR[((32+8))+rsp] - mov rax,QWORD PTR[((32+16))+rsp] - mov r12,QWORD PTR[((32+24))+rsp] - mov rdi,QWORD PTR[((32+32))+rsp] - mov rbp,QWORD PTR[((32+40))+rsp] - - - - mulx r9,r8,r14 - call __mulx_mont_383_nonred - mov r14,QWORD PTR[((32+96))+rsp] - lea rcx,QWORD PTR[128+rcx] - mov r8,QWORD PTR[((32+0))+rsp] - and r8,r14 - mov r9,QWORD PTR[((32+8))+rsp] - and r9,r14 - mov r10,QWORD PTR[((32+16))+rsp] - and r10,r14 - mov r11,QWORD PTR[((32+24))+rsp] - and r11,r14 - mov r13,QWORD PTR[((32+32))+rsp] - and r13,r14 - and r14,QWORD PTR[((32+40))+rsp] - - sub rdx,r8 - mov r8,QWORD PTR[rcx] - sbb r15,r9 - mov r9,QWORD PTR[8+rcx] - sbb rax,r10 - mov r10,QWORD PTR[16+rcx] - sbb r12,r11 - mov r11,QWORD PTR[24+rcx] - sbb rdi,r13 - mov r13,QWORD PTR[32+rcx] - sbb rbp,r14 - sbb r14,r14 - - and r8,r14 - and r9,r14 - and r10,r14 - and r11,r14 - and r13,r14 - and r14,QWORD PTR[40+rcx] - - add rdx,r8 - adc r15,r9 - adc rax,r10 - adc r12,r11 - adc rdi,r13 - adc rbp,r14 - - mov QWORD PTR[rbx],rdx - mov QWORD PTR[8+rbx],r15 - mov QWORD PTR[16+rbx],rax - mov QWORD PTR[24+rbx],r12 - mov QWORD PTR[32+rbx],rdi - mov QWORD PTR[40+rbx],rbp - lea r8,QWORD PTR[136+rsp] - mov r15,QWORD PTR[r8] - - mov r14,QWORD PTR[8+r8] - - mov r13,QWORD PTR[16+r8] - - mov r12,QWORD PTR[24+r8] - - mov rbx,QWORD PTR[32+r8] - - mov rbp,QWORD PTR[40+r8] - - lea rsp,QWORD PTR[48+r8] - -$L$SEH_epilogue_sqrx_mont_382x:: - mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue - mov rsi,QWORD PTR[16+rsp] - - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif - -$L$SEH_end_sqrx_mont_382x:: -sqrx_mont_382x ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 @@ -3495,18 +3271,6 @@ ALIGN 4 DD imagerel $L$SEH_end_sqrx_n_mul_mont_383 DD imagerel $L$SEH_info_sqrx_n_mul_mont_383_epilogue - DD imagerel $L$SEH_begin_sqrx_mont_382x - DD imagerel $L$SEH_body_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_prologue - - DD imagerel $L$SEH_body_sqrx_mont_382x - DD imagerel $L$SEH_epilogue_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_body - - DD imagerel $L$SEH_epilogue_sqrx_mont_382x - DD imagerel $L$SEH_end_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_epilogue - .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 @@ -3874,32 +3638,6 @@ DB 000h,074h,001h,000h DB 000h,064h,002h,000h DB 000h,000h,000h,000h -$L$SEH_info_sqrx_mont_382x_prologue:: -DB 1,0,5,00bh -DB 0,074h,1,0 -DB 0,064h,2,0 -DB 0,0b3h -DB 0,0 - DD 0,0 -$L$SEH_info_sqrx_mont_382x_body:: -DB 1,0,18,0 -DB 000h,0f4h,011h,000h -DB 000h,0e4h,012h,000h -DB 000h,0d4h,013h,000h -DB 000h,0c4h,014h,000h -DB 000h,034h,015h,000h -DB 000h,054h,016h,000h -DB 000h,074h,018h,000h -DB 000h,064h,019h,000h -DB 000h,001h,017h,000h -DB 000h,000h,000h,000h -DB 000h,000h,000h,000h -$L$SEH_info_sqrx_mont_382x_epilogue:: -DB 1,0,4,0 -DB 000h,074h,001h,000h -DB 000h,064h,002h,000h -DB 000h,000h,000h,000h - .xdata ENDS END diff --git a/blst_src/build/win64/sha256-armv8.asm b/blst_src/build/win64/sha256-armv8.asm index 46e821e4..83e4e99c 100644 --- a/blst_src/build/win64/sha256-armv8.asm +++ b/blst_src/build/win64/sha256-armv8.asm @@ -49,6 +49,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_block_armv8|[FUNC] ALIGN 64 |blst_sha256_block_armv8| PROC + hint #34 |$Lv8_entry| stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -189,6 +190,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_block_data_order|[FUNC] ALIGN 16 |blst_sha256_block_data_order| PROC + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,__blst_platform_cap] tst w16,#1 @@ -1038,6 +1040,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_emit|[FUNC] ALIGN 16 |blst_sha256_emit| PROC + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] if :lnot::def: __AARCH64EB__ @@ -1066,6 +1069,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_bcopy|[FUNC] ALIGN 16 |blst_sha256_bcopy| PROC + hint #34 |$Loop_bcopy| ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1079,6 +1083,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_hcopy|[FUNC] ALIGN 16 |blst_sha256_hcopy| PROC + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/cpuid.c b/blst_src/cpuid.c index 82317043..a1a01d4a 100644 --- a/blst_src/cpuid.c +++ b/blst_src/cpuid.c @@ -47,14 +47,14 @@ static int __blst_cpuid(void) return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # elif defined(__SUNPRO_C) # pragma init(__blst_cpuid) # endif -#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) || defined(_M_ARM64EC) # if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) extern unsigned long getauxval(unsigned long type) __attribute__ ((weak)); @@ -105,10 +105,56 @@ static int __blst_cpuid(void) return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # endif # endif #endif + +#if defined(_WIN64) && defined(__BLST_DLL_MAIN__) +# define IsProcessorFeaturePresent mask_IsProcessorFeaturePresent +# define WIN32_LEAN_AND_MEAN +# include + +BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD dwReason, LPVOID lpvReserved) +{ + if (dwReason == DLL_PROCESS_ATTACH) { + DisableThreadLibraryCalls(hinstDLL); + __blst_cpuid(); + } + + return TRUE; + + (void)lpvReserved; +} + +# if defined(_MSC_VER) +/* + * Even though we don't have memcpy/memset anywhere, MSVC compiler + * generates calls to them as it recognizes corresponding patterns. + */ +#pragma function(memcpy) +void *memcpy(unsigned char *dst, const unsigned char *src, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = *src++; + + return ret; +} + +#pragma function(memset) +void *memset(unsigned char *dst, int c, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = (unsigned char)c; + + return ret; +} +# endif +#endif diff --git a/blst_src/ec_mult.h b/blst_src/ec_mult.h index 24c151ba..9182c7bf 100644 --- a/blst_src/ec_mult.h +++ b/blst_src/ec_mult.h @@ -210,8 +210,7 @@ static void ptype##_mult_w##SZ(ptype *ret, const ptype *point, \ wval &= wmask; \ wval = booth_encode(wval, SZ); \ row_is_inf = ptype##_gather_booth_w##SZ(row, table, wval); \ - if (bits > 0) ptype##_add(sum, ret, row); \ - else ptype##_dadd(sum, ret, row, NULL); \ + ptype##_dadd(sum, ret, row, NULL); \ ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ ret_is_inf |= sum_is_inf; \ diff --git a/blst_src/exports.c b/blst_src/exports.c index 1ca4d475..359ed3d1 100644 --- a/blst_src/exports.c +++ b/blst_src/exports.c @@ -64,7 +64,7 @@ void blst_fr_cneg(vec256 ret, const vec256 a, int flag) { cneg_mod_256(ret, a, is_zero(flag) ^ 1, BLS12_381_r); } void blst_fr_to(vec256 ret, const vec256 a) -{ mul_mont_sparse_256(ret, a, BLS12_381_rRR, BLS12_381_r, r0); } +{ mul_mont_sparse_256(ret, BLS12_381_rRR, a, BLS12_381_r, r0); } void blst_fr_from(vec256 ret, const vec256 a) { from_mont_256(ret, a, BLS12_381_r, r0); } @@ -82,7 +82,7 @@ void blst_fr_from_scalar(vec256 ret, const pow256 a) } else { vec256 out; limbs_from_le_bytes(out, a, 32); - mul_mont_sparse_256(ret, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, out, BLS12_381_r, r0); vec_zero(out, sizeof(out)); } } @@ -151,14 +151,14 @@ void blst_sk_inverse(pow256 ret, const pow256 a) if (((size_t)a|(size_t)ret)%sizeof(limb_t) == 0 && is_endian.little) { limb_t *out = (limb_t *)ret; - mul_mont_sparse_256(out, (const limb_t *)a, BLS12_381_rRR, - BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, (const limb_t *)a, + BLS12_381_r, r0); reciprocal_fr(out, out); from_mont_256(out, out, BLS12_381_r, r0); } else { vec256 out; limbs_from_le_bytes(out, a, 32); - mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, out, BLS12_381_r, r0); reciprocal_fr(out, out); from_mont_256(out, out, BLS12_381_r, r0); le_bytes_from_limbs(ret, out, 32); @@ -472,7 +472,7 @@ void blst_fr_from_uint64(vec256 ret, const unsigned long long a[4]) } a = (const unsigned long long *)ret; } - mul_mont_sparse_256(ret, (const limb_t *)a, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, (const limb_t *)a, BLS12_381_r, r0); } void blst_uint64_from_fr(unsigned long long ret[4], const vec256 a) @@ -497,7 +497,7 @@ void blst_uint64_from_fr(unsigned long long ret[4], const vec256 a) int blst_scalar_from_le_bytes(pow256 out, const unsigned char *bytes, size_t n) { - size_t rem = (n - 1) % 32 + 1; + size_t rem = n ? ((n - 1) % 32 + 1) : 0; struct { vec256 out, digit; } t; limb_t ret; @@ -525,7 +525,7 @@ int blst_scalar_from_le_bytes(pow256 out, const unsigned char *bytes, size_t n) int blst_scalar_from_be_bytes(pow256 out, const unsigned char *bytes, size_t n) { - size_t rem = (n - 1) % 32 + 1; + size_t rem = n ? ((n - 1) % 32 + 1) : 0; struct { vec256 out, digit; } t; limb_t ret; @@ -550,6 +550,43 @@ int blst_scalar_from_be_bytes(pow256 out, const unsigned char *bytes, size_t n) return (int)(ret^1); } +void blst_fp_from_le_bytes(vec384 out, const unsigned char *bytes, size_t n) +{ + size_t rem = n ? ((n - 1) % 48 + 1) : 0; + vec384 digit; + + vec_zero(out, sizeof(vec384)); + + n -= rem; + limbs_from_le_bytes(out, bytes += n, rem); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + + while (n) { + limbs_from_le_bytes(digit, bytes -= 48, 48); + add_mod_384(out, out, digit, BLS12_381_P); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + n -= 48; + } +} + +void blst_fp_from_be_bytes(vec384 out, const unsigned char *bytes, size_t n) +{ + size_t rem = n ? ((n - 1) % 48 + 1) : 0; + vec384 digit; + + vec_zero(out, sizeof(vec384)); + + limbs_from_be_bytes(out, bytes, rem); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + + while (n -= rem) { + limbs_from_be_bytes(digit, bytes += rem, 48); + add_mod_384(out, out, digit, BLS12_381_P); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + rem = 48; + } +} + /* * Single-short SHA-256 hash function. */ @@ -573,7 +610,7 @@ void blst_scalar_from_hexascii(pow256 ret, const char *hex) void blst_fr_from_hexascii(vec256 ret, const char *hex) { limbs_from_hexascii(ret, sizeof(vec256), hex); - mul_mont_sparse_256(ret, ret, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, ret, BLS12_381_r, r0); } void blst_fp_from_hexascii(vec384 ret, const char *hex) diff --git a/blst_src/keygen.c b/blst_src/keygen.c index 9b62f16b..f904331d 100644 --- a/blst_src/keygen.c +++ b/blst_src/keygen.c @@ -33,7 +33,7 @@ static void HMAC_init(HMAC_SHA256_CTX *ctx, const void *K, size_t K_len) sha256_init(&ctx->ctx); sha256_update(&ctx->ctx, K, K_len); sha256_final(ctx->tail.c, &ctx->ctx); - } else { + } else if (K_len != 0) { sha256_bcopy(ctx->tail.c, K, K_len); } diff --git a/blst_src/multi_scalar.c b/blst_src/multi_scalar.c index 55ab8227..ce3f5184 100644 --- a/blst_src/multi_scalar.c +++ b/blst_src/multi_scalar.c @@ -7,9 +7,6 @@ #include "fields.h" #include "point.h" -/* - * Infinite point among inputs would be devastating. Shall we change it? - */ #define POINTS_TO_AFFINE_IMPL(prefix, ptype, bits, field) \ static void ptype##s_to_affine(ptype##_affine dst[], \ const ptype *const points[], size_t npoints) \ @@ -25,27 +22,36 @@ static void ptype##s_to_affine(ptype##_affine dst[], \ \ point = *points ? *points++ : point+1; \ acc = (vec##bits *)dst; \ - vec_copy(acc++, point->Z, sizeof(vec##bits)); \ - for (i = 1; i < delta; i++, acc++) \ - point = *points ? *points++ : point+1, \ - mul_##field(acc[0], acc[-1], point->Z); \ + vec_select(acc++, BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + for (i = 1; i < delta; i++, acc++) { \ + point = *points ? *points++ : point+1; \ + vec_select(acc[0], BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + mul_##field(acc[0], acc[0], acc[-1]); \ + } \ \ --acc; reciprocal_##field(acc[0], acc[0]); \ \ walkback = points-1, p = point, --delta, dst += delta; \ for (i = 0; i < delta; i++, acc--, dst--) { \ + bool_t is_inf = vec_is_zero(p->Z, sizeof(p->Z)); \ mul_##field(acc[-1], acc[-1], acc[0]); /* 1/Z */\ sqr_##field(ZZ, acc[-1]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[-1]); /* 1/Z^3 */\ - mul_##field(acc[-1], p->Z, acc[0]); \ + vec_select(acc[-1], BLS12_381_Rx.p, p->Z, sizeof(vec##bits), \ + is_inf); \ + mul_##field(acc[-1], acc[-1], acc[0]); \ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), is_inf); \ p = (p == *walkback) ? *--walkback : p-1; \ } \ sqr_##field(ZZ, acc[0]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[0]); /* 1/Z^3 */\ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), vec_is_zero(p->Z, sizeof(p->Z))); \ ++delta, dst += delta, npoints -= delta; \ } \ } \ @@ -74,18 +80,23 @@ POINTS_TO_AFFINE_IMPL(blst_p2, POINTonE2, 384x, fp2) #define SCRATCH_SZ(ptype) (sizeof(ptype)==sizeof(POINTonE1) ? 8192 : 4096) +/* The intermediate infinity points are encoded as [0, 0, 1]. */ + #define PRECOMPUTE_WBITS_IMPL(prefix, ptype, bits, field, one) \ -static void ptype##_precompute_row_wbits(ptype row[], size_t wbits, \ - const ptype##_affine *point) \ +static void ptype##_precompute_row(ptype row[], size_t n, \ + const ptype##_affine *point) \ { \ - size_t i, j, n = (size_t)1 << (wbits-1); \ + size_t i, j; \ + bool_t inf = vec_is_zero(point, sizeof(*point)); \ /* row[-1] is implicit infinity */\ vec_copy(&row[0], point, sizeof(*point)); /* row[0]=p*1 */\ vec_copy(&row[0].Z, one, sizeof(row[0].Z)); \ ptype##_double(&row[1], &row[0]); /* row[1]=p*(1+1) */\ + vec_select(&row[1].Z, one, &row[1].Z, sizeof(row[1].Z), inf); \ for (i = 2, j = 1; i < n; i += 2, j++) \ ptype##_add_affine(&row[i], &row[i-1], point), /* row[2]=p*(2+1) */\ - ptype##_double(&row[i+1], &row[j]); /* row[3]=p*(2+2) */\ + ptype##_double(&row[i+1], &row[j]), /* row[3]=p*(2+2) */\ + vec_select(&row[i+1].Z, one, &row[i+1].Z, sizeof(row[i+1].Z), inf); \ } /* row[4] ... */\ \ static void ptype##s_to_affine_row_wbits(ptype##_affine dst[], ptype src[], \ @@ -142,16 +153,44 @@ static void ptype##s_precompute_wbits(ptype##_affine table[], size_t wbits, \ rows = row = (ptype *)(&table[top]); \ for (i = 0; i < stride; i++, row += nwin) \ point = *points ? *points++ : point+1, \ - ptype##_precompute_row_wbits(row, wbits, point); \ + ptype##_precompute_row(row, nwin, point); \ ptype##s_to_affine_row_wbits(&table[top], rows, wbits, stride); \ top += stride << (wbits-1); \ npoints -= stride; \ } \ - rows = row = alloca(2*sizeof(ptype##_affine) * npoints * nwin); \ - for (i = 0; i < npoints; i++, row += nwin) \ - point = *points ? *points++ : point+1, \ - ptype##_precompute_row_wbits(row, wbits, point); \ - ptype##s_to_affine_row_wbits(&table[top], rows, wbits, npoints); \ + if ((i = 2*sizeof(ptype##_affine)*npoints*nwin) <= SCRATCH_LIMIT) { \ + rows = row = alloca(i); \ + for (i = 0; i < npoints; i++, row += nwin) \ + point = *points ? *points++ : point+1, \ + ptype##_precompute_row(row, nwin, point); \ + ptype##s_to_affine_row_wbits(&table[top], rows, wbits, npoints); \ + } else { \ + const ptype *pp[2]; \ +\ + stride = SCRATCH_LIMIT / sizeof(ptype); \ + stride -= stride % 2; \ + if (stride > nwin) stride = nwin; \ +\ + pp[0] = row = alloca(stride * sizeof(ptype)); \ + pp[1] = NULL; \ + for (i = 0; i < npoints; i++, top += nwin) { \ + size_t j, k, n; \ +\ + point = *points ? *points++ : point+1; \ + ptype##_precompute_row(row, stride, point); \ + ptype##s_to_affine(&table[top], pp, stride); \ + for (j = stride; j < nwin; j += stride) { \ + n = (j+stride) <= nwin ? stride : nwin-j; \ + for (k = 0; k < n-1; k++) \ + ptype##_add_affine(&row[k], &row[stride-1], &table[top+k]); \ + if (j == stride) \ + ptype##_double(&row[k], &row[stride-1]); \ + else \ + ptype##_add_affine(&row[k], &row[stride-1], &table[top+k]); \ + ptype##s_to_affine(&table[top+j], pp, n); \ + } \ + } \ + } \ } \ \ size_t prefix##s_mult_wbits_precompute_sizeof(size_t wbits, size_t npoints) \ @@ -202,7 +241,7 @@ static void ptype##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ \ nbits -= window; \ z = is_zero(nbits); \ - wval = (get_wval_limb(scalar, nbits - (z^1), wbits + (z^1)) << z) & wmask; \ + wval = (get_wval_limb(scalar, nbits - (z^1), window + (z^1)) << z) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[0], row, wbits, wval); \ row += nwin; \ @@ -232,7 +271,7 @@ static void ptype##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ if (j == scratch_sz) \ ptype##s_accumulate(ret, scratch, j), j = 0; \ scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ - wval = (get_wval_limb(scalar, 0, wbits) << 1) & wmask; \ + wval = (get_wval_limb(scalar, 0, window) << 1) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[j], row, wbits, wval); \ } \ @@ -267,7 +306,14 @@ static size_t pippenger_window_size(size_t npoints) for (wbits=0; npoints>>=1; wbits++) ; - return wbits>12 ? wbits-3 : (wbits>4 ? wbits-2 : (wbits ? 2 : 1)); + if (wbits > 12) + return wbits - 3; + else if (wbits > 8) + return wbits - 2; + else if (wbits > 4) + return wbits - 1; + + return wbits ? 2 : 1; } #define DECLARE_PRIVATE_POINTXYZZ(ptype, bits) \ @@ -402,10 +448,11 @@ void prefix##s_mult_pippenger(ptype *ret, \ { \ if (npoints == 1) { \ prefix##_from_affine(ret, points[0]); \ - prefix##_mult(ret, ret, scalars[0], nbits); \ + ptype##_mult_w5(ret, ret, scalars[0], nbits); \ return; \ } \ - if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT) { \ + if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT && \ + npoints < 32) { \ ptype##_affine *table = alloca(npoints * sizeof(ptype##_affine) * 8); \ ptype##s_precompute_wbits(table, 4, points, npoints); \ ptype##s_mult_wbits(ret, table, 4, npoints, scalars, nbits, NULL); \ diff --git a/blst_src/no_asm.h b/blst_src/no_asm.h index be7bf47e..80bbba57 100644 --- a/blst_src/no_asm.h +++ b/blst_src/no_asm.h @@ -635,58 +635,6 @@ void sqr_n_mul_mont_383(vec384 ret, const vec384 a, size_t count, mul_mont_n(ret, ret, b, p, n0, NLIMBS(384)); } -void sqr_mont_382x(vec384x ret, const vec384x a, - const vec384 p, limb_t n0) -{ - llimb_t limbx; - limb_t mask, carry, borrow; - size_t i; - vec384 t0, t1; - - /* "add_mod_n(t0, a[0], a[1], p, NLIMBS(384));" */ - for (carry=0, i=0; i> LIMB_T_BITS); - } - - /* "sub_mod_n(t1, a[0], a[1], p, NLIMBS(384));" */ - for (borrow=0, i=0; i> LIMB_T_BITS) & 1; - } - mask = 0 - borrow; - launder(mask); - - /* "mul_mont_n(ret[1], a[0], a[1], p, n0, NLIMBS(384));" */ - mul_mont_nonred_n(ret[1], a[0], a[1], p, n0, NLIMBS(384)); - - /* "add_mod_n(ret[1], ret[1], ret[1], p, NLIMBS(384));" */ - for (carry=0, i=0; i>(LIMB_T_BITS-1); - } - - /* "mul_mont_n(ret[0], t0, t1, p, n0, NLIMBS(384));" */ - mul_mont_nonred_n(ret[0], t0, t1, p, n0, NLIMBS(384)); - - /* account for t1's sign... */ - for (borrow=0, i=0; i> LIMB_T_BITS) & 1; - } - mask = 0 - borrow; - launder(mask); - for (carry=0, i=0; i> LIMB_T_BITS); - } -} - #if defined(__GNUC__) || defined(__clang__) # define MSB(x) ({ limb_t ret = (x) >> (LIMB_T_BITS-1); launder(ret); ret; }) #else diff --git a/blst_src/recip.c b/blst_src/recip.c index e0c70063..85f00228 100644 --- a/blst_src/recip.c +++ b/blst_src/recip.c @@ -62,20 +62,11 @@ static void reciprocal_fp(vec384 out, const vec384 inp) TO_LIMB_T(0x39869507b587b120), TO_LIMB_T(0x23ba5c279c2895fb), TO_LIMB_T(0x58dd3db21a5d66bb), TO_LIMB_T(0xd0088f51cbff34d2) }; -#ifdef __BLST_NO_ASM__ -# define RRx4 BLS12_381_RR -#else - static const vec384 RRx4 = { /* (4<<768)%P */ - TO_LIMB_T(0x5f7e7cd070d107c2), TO_LIMB_T(0xec839a9ac49c13c8), - TO_LIMB_T(0x6933786f44f4ef0b), TO_LIMB_T(0xd6bf8b9c676be983), - TO_LIMB_T(0xd3adaaaa4dcefb06), TO_LIMB_T(0x12601bc1d82bc175) - }; -#endif union { vec768 x; vec384 r[2]; } temp; - ct_inverse_mod_383(temp.x, inp, BLS12_381_P, Px8); + ct_inverse_mod_384(temp.x, inp, BLS12_381_P, Px8); redc_mont_384(temp.r[0], temp.x, BLS12_381_P, p0); - mul_mont_384(temp.r[0], temp.r[0], RRx4, BLS12_381_P, p0); + mul_mont_384(temp.r[0], temp.r[0], BLS12_381_RR, BLS12_381_P, p0); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* sign goes straight to flt_reciprocal */ @@ -88,7 +79,6 @@ static void reciprocal_fp(vec384 out, const vec384 inp) #else vec_copy(out, temp.r[0], sizeof(vec384)); #endif -#undef RRx4 } void blst_fp_inverse(vec384 out, const vec384 inp) @@ -129,7 +119,7 @@ static void reciprocal_fr(vec256 out, const vec256 inp) ct_inverse_mod_256(temp, inp, BLS12_381_r, rx2); redc_mont_256(out, temp, BLS12_381_r, r0); - mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, out, BLS12_381_r, r0); } void blst_fr_inverse(vec256 out, const vec256 inp) diff --git a/blst_src/vect.h b/blst_src/vect.h index 19640b11..6b48d633 100644 --- a/blst_src/vect.h +++ b/blst_src/vect.h @@ -85,9 +85,7 @@ typedef limb_t bool_t; # define from_mont_384 fromx_mont_384 # define sgn0_pty_mont_384 sgn0x_pty_mont_384 # define sgn0_pty_mont_384x sgn0x_pty_mont_384x -# define ct_inverse_mod_383 ctx_inverse_mod_383 -#elif defined(__BLST_NO_ASM__) -# define ct_inverse_mod_383 ct_inverse_mod_384 +# define ct_inverse_mod_384 ctx_inverse_mod_384 #endif void mul_mont_sparse_256(vec256 ret, const vec256 a, const vec256 b, @@ -137,7 +135,7 @@ void cneg_mod_384(vec384 ret, const vec384 a, bool_t flag, const vec384 p); void lshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void rshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void div_by_2_mod_384(vec384 ret, const vec384 a, const vec384 p); -void ct_inverse_mod_383(vec768 ret, const vec384 inp, const vec384 mod, +void ct_inverse_mod_384(vec768 ret, const vec384 inp, const vec384 mod, const vec384 modx); void ct_inverse_mod_256(vec512 ret, const vec256 inp, const vec256 mod, const vec256 modx); @@ -146,7 +144,6 @@ bool_t ct_is_square_mod_384(const vec384 inp, const vec384 mod); #if defined(__ADX__) /* e.g. -march=broadwell */ && !defined(__BLST_PORTABLE__) # define mul_mont_384x mulx_mont_384x # define sqr_mont_384x sqrx_mont_384x -# define sqr_mont_382x sqrx_mont_382x # define mul_382x mulx_382x # define sqr_382x sqrx_382x #endif @@ -154,7 +151,6 @@ bool_t ct_is_square_mod_384(const vec384 inp, const vec384 mod); void mul_mont_384x(vec384x ret, const vec384x a, const vec384x b, const vec384 p, limb_t n0); void sqr_mont_384x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); -void sqr_mont_382x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); void mul_382x(vec768 ret[2], const vec384x a, const vec384x b, const vec384 p); void sqr_382x(vec768 ret[2], const vec384x a, const vec384 p); @@ -407,7 +403,7 @@ static inline void vec_czero(void *ret, size_t num, bool_t cbit) #if defined(__INTEL_COMPILER) # pragma warning(disable:167) # pragma warning(disable:556) -#elif defined(__GNUC__) && !defined(__clang__) +#elif defined(__GNUC__) && !defined(__clang__) && (__STDC_VERSION__-0) < 202311 # pragma GCC diagnostic ignored "-Wpedantic" #elif defined(_MSC_VER) # pragma warning(disable: 4127 4189) diff --git a/internal/blst/blst.go b/internal/blst/blst.go index 6fe6b318..a40fdbd6 100644 --- a/internal/blst/blst.go +++ b/internal/blst/blst.go @@ -301,8 +301,12 @@ func KeyGenV5(ikm []byte, salt []byte, optional ...[]byte) *SecretKey { if len(optional) > 0 { info = optional[0] } + saltLen := len(salt) + if saltLen == 0 { + salt = []byte{0} + } C.blst_keygen_v5(&sk.cgo, (*C.byte)(&ikm[0]), C.size_t(len(ikm)), - (*C.byte)(&salt[0]), C.size_t(len(salt)), + (*C.byte)(&salt[0]), C.size_t(saltLen), ptrOrNil(info), C.size_t(len(info))) // Postponing secret key zeroing till garbage collection can be too // late to be effective, but every little bit helps... @@ -370,14 +374,6 @@ func PairingAsFp12(ctx Pairing) *Fp12 { return &pt } -func (pt *Fp12) asPtr() *C.blst_fp12 { - if pt != nil { - return &pt.cgo - } - - return nil -} - func Fp12One() Fp12 { return cgo_fp12One } @@ -472,6 +468,14 @@ func (pt1 *Fp12) Equals(pt2 *Fp12) bool { return *pt1 == *pt2 } +func (pt *Fp12) asPtr() *C.blst_fp12 { + if pt != nil { + return &pt.cgo + } + + return nil +} + func ptrOrNil(bytes []byte) *C.byte { var ptr *C.byte if len(bytes) > 0 { @@ -488,14 +492,6 @@ func ptrOrNil(bytes []byte) *C.byte { // PublicKey // -func (pt *P1Affine) asPtr() *C.blst_p1_affine { - if pt != nil { - return &pt.cgo - } - - return nil -} - func (pk *P1Affine) From(s *Scalar) *P1Affine { C.blst_sk_to_pk2_in_g1(nil, &pk.cgo, &s.cgo) return pk @@ -604,7 +600,7 @@ func (sig *P2Affine) AggregateVerify(sigGroupcheck bool, // Aggregate verify with compressed signature and public keys // Uses a dummy signature to get the correct type -func (_ *P2Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, +func (*P2Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, pks [][]byte, pksVerify bool, msgs []Message, dst []byte, optional ...bool) bool { // useHash bool, usePksAsAugs bool @@ -810,7 +806,7 @@ func (sig *P2Affine) FastAggregateVerify(sigGroupcheck bool, return sig.Verify(sigGroupcheck, pkAff, false, msg, dst, optional...) } -func (_ *P2Affine) MultipleAggregateVerify(sigs []*P2Affine, +func (*P2Affine) MultipleAggregateVerify(sigs []*P2Affine, sigsGroupcheck bool, pks []*P1Affine, pksVerify bool, msgs []Message, dst []byte, randFn func(*Scalar), randBits int, optional ...interface{}) bool { // useHash @@ -1096,14 +1092,6 @@ func (agg *P2Aggregate) coreAggregate(getter aggGetterP2, groupcheck bool, // PublicKey // -func (pt *P2Affine) asPtr() *C.blst_p2_affine { - if pt != nil { - return &pt.cgo - } - - return nil -} - func (pk *P2Affine) From(s *Scalar) *P2Affine { C.blst_sk_to_pk2_in_g2(nil, &pk.cgo, &s.cgo) return pk @@ -1212,7 +1200,7 @@ func (sig *P1Affine) AggregateVerify(sigGroupcheck bool, // Aggregate verify with compressed signature and public keys // Uses a dummy signature to get the correct type -func (_ *P1Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, +func (*P1Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, pks [][]byte, pksVerify bool, msgs []Message, dst []byte, optional ...bool) bool { // useHash bool, usePksAsAugs bool @@ -1418,7 +1406,7 @@ func (sig *P1Affine) FastAggregateVerify(sigGroupcheck bool, return sig.Verify(sigGroupcheck, pkAff, false, msg, dst, optional...) } -func (_ *P1Affine) MultipleAggregateVerify(sigs []*P1Affine, +func (*P1Affine) MultipleAggregateVerify(sigs []*P1Affine, sigsGroupcheck bool, pks []*P2Affine, pksVerify bool, msgs []Message, dst []byte, randFn func(*Scalar), randBits int, optional ...interface{}) bool { // useHash @@ -1721,6 +1709,10 @@ func PairingMulNAggregatePkInG1(ctx Pairing, PK *P1Affine, pkValidate bool, aug = optional[0] } + if randBits > 256 { + panic("scalar length mismatch") + } + r := C.blst_pairing_chk_n_mul_n_aggr_pk_in_g1(&ctx[0], PK.asPtr(), C.bool(pkValidate), sig.asPtr(), C.bool(sigGroupcheck), @@ -1771,7 +1763,7 @@ func (p1 *P1Affine) InG1() bool { return bool(C.blst_p1_affine_in_g1(&p1.cgo)) } -func (_ *P1Affine) BatchUncompress(in [][]byte) []*P1Affine { +func (*P1Affine) BatchUncompress(in [][]byte) []*P1Affine { // Allocate space for all of the resulting points. Later we'll save pointers // and return those so that the result could be used in other functions, // such as MultipleAggregateVerify. @@ -2172,6 +2164,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { case P1Affines: pointsBySlice[0] = &val[0].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } scalarsBySlice := [2]*C.byte{nil, nil} @@ -2191,6 +2184,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: p_scalars = &scalars[0] + default: // type is already vetted } var ret P1 @@ -2206,18 +2200,19 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { return &ret } - if npoints < 32 { + if npoints < 32 || npoints < numThreads { if numThreads > npoints { numThreads = npoints } + acc := make([]P1, numThreads) + curItem := uint32(0) - msgs := make(chan P1, numThreads) + var wg sync.WaitGroup + wg.Add(numThreads) for tid := 0; tid < numThreads; tid++ { - go func() { - var acc P1 - + go func(acc *P1) { for { workItem := int(atomic.AddUint32(&curItem, 1) - 1) if workItem >= npoints { @@ -2232,6 +2227,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { point = &val[workItem] case P1Affines: point = &val[workItem] + default: // type is already vetted } var scalar *C.byte @@ -2248,20 +2244,22 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: scalar = scalars[workItem] + default: // type is already vetted } C.go_p1_mult_n_acc(&acc.cgo, &point.cgo.x, true, scalar, C.size_t(nbits)) } - msgs <- acc - }() + wg.Done() + }(&acc[tid]) } - ret := <-msgs + wg.Wait() + + ret := acc[0] for tid := 1; tid < numThreads; tid++ { - point := <-msgs - C.blst_p1_add_or_double(&ret.cgo, &ret.cgo, &point.cgo) + C.blst_p1_add_or_double(&ret.cgo, &ret.cgo, &acc[tid].cgo) } for i := range scalars { @@ -2338,6 +2336,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { case P1Affines: pointsBySlice[0] = &val[x].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } var p_scalars **C.byte @@ -2356,6 +2355,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: p_scalars = &scalars[x] + default: // type is already vetted } C.blst_p1s_tile_pippenger(&grid[workItem].point.cgo, @@ -2528,6 +2528,10 @@ func PairingMulNAggregatePkInG2(ctx Pairing, PK *P2Affine, pkValidate bool, aug = optional[0] } + if randBits > 256 { + panic("scalar length mismatch") + } + r := C.blst_pairing_chk_n_mul_n_aggr_pk_in_g2(&ctx[0], PK.asPtr(), C.bool(pkValidate), sig.asPtr(), C.bool(sigGroupcheck), @@ -2578,7 +2582,7 @@ func (p2 *P2Affine) InG2() bool { return bool(C.blst_p2_affine_in_g2(&p2.cgo)) } -func (_ *P2Affine) BatchUncompress(in [][]byte) []*P2Affine { +func (*P2Affine) BatchUncompress(in [][]byte) []*P2Affine { // Allocate space for all of the resulting points. Later we'll save pointers // and return those so that the result could be used in other functions, // such as MultipleAggregateVerify. @@ -2979,6 +2983,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { case P2Affines: pointsBySlice[0] = &val[0].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } scalarsBySlice := [2]*C.byte{nil, nil} @@ -2998,6 +3003,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: p_scalars = &scalars[0] + default: // type is already vetted } var ret P2 @@ -3013,18 +3019,19 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { return &ret } - if npoints < 32 { + if npoints < 32 || npoints < numThreads { if numThreads > npoints { numThreads = npoints } + acc := make([]P2, numThreads) + curItem := uint32(0) - msgs := make(chan P2, numThreads) + var wg sync.WaitGroup + wg.Add(numThreads) for tid := 0; tid < numThreads; tid++ { - go func() { - var acc P2 - + go func(acc *P2) { for { workItem := int(atomic.AddUint32(&curItem, 1) - 1) if workItem >= npoints { @@ -3039,6 +3046,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { point = &val[workItem] case P2Affines: point = &val[workItem] + default: // type is already vetted } var scalar *C.byte @@ -3055,20 +3063,22 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: scalar = scalars[workItem] + default: // type is already vetted } C.go_p2_mult_n_acc(&acc.cgo, &point.cgo.x, true, scalar, C.size_t(nbits)) } - msgs <- acc - }() + wg.Done() + }(&acc[tid]) } - ret := <-msgs + wg.Wait() + + ret := acc[0] for tid := 1; tid < numThreads; tid++ { - point := <-msgs - C.blst_p2_add_or_double(&ret.cgo, &ret.cgo, &point.cgo) + C.blst_p2_add_or_double(&ret.cgo, &ret.cgo, &acc[tid].cgo) } for i := range scalars { @@ -3145,6 +3155,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { case P2Affines: pointsBySlice[0] = &val[x].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } var p_scalars **C.byte @@ -3163,6 +3174,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: p_scalars = &scalars[x] + default: // type is already vetted } C.blst_p2s_tile_pippenger(&grid[workItem].point.cgo, @@ -3547,6 +3559,14 @@ func (e1 *P1Affine) Equals(e2 *P1Affine) bool { return bool(C.blst_p1_affine_is_equal(&e1.cgo, &e2.cgo)) } +func (pt *P1Affine) asPtr() *C.blst_p1_affine { + if pt != nil { + return &pt.cgo + } + + return nil +} + func (e1 *P1) Equals(e2 *P1) bool { return bool(C.blst_p1_is_equal(&e1.cgo, &e2.cgo)) } @@ -3555,6 +3575,14 @@ func (e1 *P2Affine) Equals(e2 *P2Affine) bool { return bool(C.blst_p2_affine_is_equal(&e1.cgo, &e2.cgo)) } +func (pt *P2Affine) asPtr() *C.blst_p2_affine { + if pt != nil { + return &pt.cgo + } + + return nil +} + func (e1 *P2) Equals(e2 *P2) bool { return bool(C.blst_p2_is_equal(&e1.cgo, &e2.cgo)) } @@ -3585,15 +3613,18 @@ func breakdown(nbits, window, ncpus int) (nx int, ny int, wnd int) { wnd = window } } - } else { + } else if window > 3 { nx = 2 wnd = window - 2 - for (nbits/wnd+1)*nx < ncpus { + for wnd > 1 && (nbits/wnd+1)*nx < ncpus { nx += 1 wnd = window - bits.Len(3*uint(nx)/2) } nx -= 1 wnd = window - bits.Len(3*uint(nx)/2) + } else { + nx = 1 + wnd = window } ny = nbits/wnd + 1 wnd = nbits/ny + 1 diff --git a/internal/blst/blst.h b/internal/blst/blst.h index 8d90abf0..e234b395 100644 --- a/internal/blst/blst.h +++ b/internal/blst/blst.h @@ -26,12 +26,12 @@ extern "C" { #elif !defined(__STDC_VERSION__) || __STDC_VERSION__<202311 # if defined(__BLST_CGO__) typedef _Bool bool; /* it's assumed that cgo calls modern enough compiler */ +# elif defined(__BLST_RUST_BINDGEN__) || defined(__BLST_ZIG__) +# define bool _Bool +# elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 +# include # elif !defined(bool) -# if defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 -# define bool _Bool -# else -# define bool int -# endif +# define bool int # define __blst_h_bool__ # endif #endif diff --git a/internal/blst/blst_aux.h b/internal/blst/blst_aux.h index 3de0850e..394a94dd 100644 --- a/internal/blst/blst_aux.h +++ b/internal/blst/blst_aux.h @@ -110,6 +110,9 @@ size_t blst_p2_sizeof(void); size_t blst_p2_affine_sizeof(void); size_t blst_fp12_sizeof(void); +void blst_fp_from_le_bytes(blst_fp *ret, const byte *in, size_t len); +void blst_fp_from_be_bytes(blst_fp *ret, const byte *in, size_t len); + /* * Single-shot SHA-256 hash function. */