From 95e55d860d1f84705bc35b45a1bf1d1e9a2f7227 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 06:26:47 +0000 Subject: [PATCH] ML-DSA x86-64 sign: each row of w with one fused dot-inverse call Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015ajQWygy8vKC2iCbkRdW6i --- .../Artifacts/MlDsaArith/X86_64.lean | 47 + .../Impl/MlDsa/X86_64/Arith/Avx2.lean | 2 +- .../Impl/MlDsa/X86_64/Arith/Backend.lean | 14 +- .../Impl/MlDsa/X86_64/Arith/Dot.lean | 63 + .../Impl/MlDsa/X86_64/Arith/Montgomery.lean | 28 +- .../Impl/MlDsa/X86_64/Arith/Ntt.lean | 5 +- .../Impl/MlDsa/X86_64/Sign/Frag.lean | 17 +- .../Impl/MlDsa/X86_64/Sign/Sign.lean | 4 +- .../Proof/MlDsa/Arith/DotSum.lean | 54 + .../Proof/MlDsa/X86_64/Arith/Avx2Lit.lean | 1 + .../Proof/MlDsa/X86_64/Arith/Backend.lean | 12 +- .../Proof/MlDsa/X86_64/Arith/BackendAvx2.lean | 8 +- .../Proof/MlDsa/X86_64/Arith/Dot.lean | 372 ++ .../Proof/MlDsa/X86_64/Arith/DotCore.lean | 211 + .../Proof/MlDsa/X86_64/Arith/NttInv.lean | 55 +- .../Proof/MlDsa/X86_64/Arith/YDot.lean | 369 ++ .../MlDsa/X86_64/Arith/YMontgomeryInv.lean | 63 +- .../Proof/MlDsa/X86_64/KeyGen/Inst.lean | 1 + .../Proof/MlDsa/X86_64/Message/SignFn.lean | 11 +- .../Proof/MlDsa/X86_64/Sign/Inst.lean | 9 +- .../Proof/MlDsa/X86_64/Sign/PhaseC.lean | 73 +- .../Proof/MlDsa/X86_64/Sign/PhaseCCT.lean | 37 +- .../Proof/MlDsa/X86_64/Sign/Prims.lean | 154 +- .../Proof/MlDsa/X86_64/Sign/Verified.lean | 28 +- src/asm/x86_64/mldsa.rs | 4602 +++++++++++++++++ src/asm/x86_64/mldsa44.rs | 252 +- src/asm/x86_64/mldsa65.rs | 456 +- src/asm/x86_64/mldsa87.rs | 1328 ++--- 28 files changed, 6422 insertions(+), 1854 deletions(-) create mode 100644 lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Dot.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/Arith/DotSum.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Dot.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/DotCore.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YDot.lean diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean index b197c0207c..33b726d67e 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaArith/X86_64.lean @@ -9,6 +9,8 @@ public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMontgomery public import VerifiedGarbage.Proof.Framework.Registration +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Dot +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YDot /-! # ML-DSA (FIPS 204) on x86-64: the arithmetic of polynomials -/ @@ -16,6 +18,21 @@ public import VerifiedGarbage.Proof.Framework.Registration namespace VG.Artifacts.MlDsaArith.X86_64 +/-- Notes on `vg_mldsa_dot_inverse{c}`. -/ +def dotNotes : List String := + ["The function computes on four coefficients at a time in SSE2 registers: it sums the products in \ + 64-bit lanes, reduces each sum once, then runs `vg_mldsa_inv_ntt`'s code, with a table of the 256 \ + zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's \ + mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning."] + +/-- Notes on `vg_mldsa_dot_inverse{c}_avx2`. -/ +def dotNotesAvx2 : List String := + ["The function computes on eight coefficients at a time in AVX2 registers: it sums the products in \ + 64-bit lanes, reduces each sum once to the sum times `2⁻³²`, then runs \ + `vg_mldsa_montgomery_inv_ntt_avx2`'s code, which cancels the factor, with a table of the 256 zetas \ + that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's \ + mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning."] + def artifacts : List Artifact := [ Spec.MlDsa.nttApi.artifact X86_64.target { notes := ["The function computes on four coefficients at a time in SSE2 registers, with a table of \ @@ -101,6 +118,36 @@ def artifacts : List Artifact := [ notes := ["The function computes on eight coefficients at a time in AVX2 registers."] code := Impl.MlDsa.X86_64.Arith.subAvx2 verified := Proof.MlDsa.X86_64.Arith.subY_verified + features := ["avx", "avx2"] }, + (Spec.MlDsa.dotInverseApi 4).artifact X86_64.target { + notes := dotNotes + code := Impl.MlDsa.X86_64.Arith.dotInvSse2 4 + verified := Proof.MlDsa.X86_64.Arith.DotS.verified4 }, + (Spec.MlDsa.dotInverseApi 5).artifact X86_64.target { + notes := dotNotes + code := Impl.MlDsa.X86_64.Arith.dotInvSse2 5 + verified := Proof.MlDsa.X86_64.Arith.DotS.verified5 }, + (Spec.MlDsa.dotInverseApi 7).artifact X86_64.target { + notes := dotNotes + code := Impl.MlDsa.X86_64.Arith.dotInvSse2 7 + verified := Proof.MlDsa.X86_64.Arith.DotS.verified7 }, + (Spec.MlDsa.dotInverseApi 4).artifact X86_64.target { + suffix := "_avx2" + notes := dotNotesAvx2 + code := Impl.MlDsa.X86_64.Arith.dotInvAvx2 4 + verified := Proof.MlDsa.X86_64.Arith.YDot.verified4 + features := ["avx", "avx2"] }, + (Spec.MlDsa.dotInverseApi 5).artifact X86_64.target { + suffix := "_avx2" + notes := dotNotesAvx2 + code := Impl.MlDsa.X86_64.Arith.dotInvAvx2 5 + verified := Proof.MlDsa.X86_64.Arith.YDot.verified5 + features := ["avx", "avx2"] }, + (Spec.MlDsa.dotInverseApi 7).artifact X86_64.target { + suffix := "_avx2" + notes := dotNotesAvx2 + code := Impl.MlDsa.X86_64.Arith.dotInvAvx2 7 + verified := Proof.MlDsa.X86_64.Arith.YDot.verified7 features := ["avx", "avx2"] }] end VG.Artifacts.MlDsaArith.X86_64 diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean index 7364e83562..f68386ba40 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -206,6 +206,6 @@ def subAvx2 : Prog isa := def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, Round.highBitsAvx2, Round.lowBitsAvx2, Round.normLtAvx2, Round.makeHintAvx2, Round.useHintAvx2, Sample.Rej5.rejNTT4Avx2, - Sample.Mask4.expandMask4Avx2, "_avx2", false⟩ + Sample.Mask4.expandMask4Avx2, "_avx2", false, .block [], .block [], .block []⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean index 7140ad2029..4f45cfcbca 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -2,6 +2,7 @@ module public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Dot public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.AddSub public import VerifiedGarbage.Impl.MlDsa.X86_64.Round.Round public import VerifiedGarbage.Impl.MlDsa.X86_64.Sample.RejNtt4 @@ -47,18 +48,27 @@ structure Backend where sfx : String /-- Whether products and the inverse transform use Montgomery scaling. -/ montgomery : Bool := false + /-- `vg_mldsa_dot_inverse4`, `5` and `7`: the inverse NTT of a row's sum of products. -/ + dotInv4 : Prog isa + dotInv5 : Prog isa + dotInv7 : Prog isa /-- The SSE2 code. -/ def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, Round.highBits, Round.lowBits, Round.normLt, Round.makeHint, Round.useHint, Sample.Rej4.rejNTT4, - Sample.Mask4.expandMask4, "", false⟩ + Sample.Mask4.expandMask4, "", false, dotInvSse2 4, dotInvSse2 5, + dotInvSse2 7⟩ + +/-- `vg_mldsa_dot_inverse{c}`, for `c` of 4, 5 and 7. -/ +abbrev Backend.dotInv (B : Backend) (c : Nat) : Prog isa := + if c = 4 then B.dotInv4 else if c = 5 then B.dotInv5 else B.dotInv7 /-- Every function empty, which the proofs that the functions calling a backend never write `rsp` (and load MXCSR only to restore it) evaluate in its place. -/ def Backend.empty : Backend := ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], - .block [], .block [], .block [], .block [], "", false⟩ + .block [], .block [], .block [], .block [], "", false, .block [], .block [], .block []⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Dot.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Dot.lean new file mode 100644 index 0000000000..ef297a5e4e --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Dot.lean @@ -0,0 +1,63 @@ +module + +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Mul +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt + +/-! +# ML-DSA on x86-64: a row's sum of products, four coefficients at a time + +`vg_mldsa_dot_inverse{c}` sums the products of the coefficients of `c` pairs +of polynomials `a[j]`, `b[j]` (each below `3 q`) in quadwords: `pmuludq` +multiplies the even doublewords of its operands, and those shifted down by +32 bits are the odd ones, so `xmm5` accumulates the products of the even +coefficients and `xmm6` those of the odd ones (`dotMac`). A sum of at most +seven products is below `63 q² < 2³² q`, so one Montgomery reduction of each +quadword (`vredc`), merged as in `vmont`, leaves the sum times `2⁻³²` below +`2 q`, which `vcsub` reduces (`dotRed`). + +The SSE2 code (`dotInvSse2`), whose inverse NTT is not scaled, multiplies +each sum by `2⁶⁴ mod q` (in `xmm11`, `mulPro`) with `vmont` (`dotPlain`), which +cancels the `2⁻³²`. +-/ + +@[expose] public section + +namespace VG.Impl.MlDsa.X86_64.Arith + +open VG.X86_64 +open VG.Impl.MlKem.X86_64 (xb xmov withMxcsr rcxLoop) + +/-- The sums, zero. -/ +def dotZero : List Instr := [xb .pxor .xmm5 .xmm5, xb .pxor .xmm6 .xmm6] + +/-- The products of the doublewords of `xmm0` and `xmm1` added to the +quadwords of `xmm5` (even doublewords) and `xmm6` (odd ones). -/ +def dotMac : List Instr := + [xmov .xmm2 .xmm0, .xop (.shift .psrlq .xmm2 32), xmov .xmm3 .xmm1, .xop (.shift .psrlq .xmm3 32), + xb .pmuludq .xmm2 .xmm3, xb .pmuludq .xmm0 .xmm1, xb .paddq .xmm5 .xmm0, xb .paddq .xmm6 .xmm2] + +/-- The sums, reduced: `(P + m q) / 2³²` of each quadword, merged into the +doublewords of `xmm5`, then reduced to `[0, q)`. -/ +def dotRed : List Instr := + vredc .xmm5 .xmm2 ++ [.xop (.shift .psrlq .xmm5 32)] ++ vredc .xmm6 .xmm2 ++ [xb .por .xmm5 .xmm6] ++ + vcsub .xmm5 .xmm2 + +/-- The sum times `2⁻³²`, times `2⁶⁴ mod q` (in `xmm11`) times `2⁻³²`: the sum +modulo `q`. -/ +def dotPlain : List Instr := vmont .xmm5 .xmm11 .xmm11 .xmm2 .xmm4 ++ vcsub .xmm5 .xmm2 + +/-- The coefficients of `a[j]` and `b[j]`. -/ +def dotLoadsS (j : Nat) : List Instr := [.movdquLoad .xmm0 (at_ .r10 (1024 * j)), .movdquLoad .xmm1 (at_ .rdx (1024 * j))] + +/-- Four coefficients of the sum of the `c` products. -/ +def dotBodyS (c : Nat) : List Instr := + dotZero ++ (List.range c).flatMap (fun j => dotLoadsS j ++ dotMac) ++ dotRed ++ dotPlain ++ + [.movdquStore (at_ .r8 0) .xmm5, .alu .add .r8 (.imm 16), .alu .add .r10 (.imm 16), .alu .add .rdx (.imm 16)] + +/-- `vg_mldsa_dot_inverse{c}`: the sum of the `c` products, then `NTT⁻¹`, in +one MXCSR region. -/ +def dotInvSse2 (c : Nat) : Prog isa := + .seq (.block [.mov .r10 (.reg .rsi), .mov .rsi (.reg .rcx)]) <| withMxcsr .rsi 768 <| + .seq (.block (mulPro ++ [.mov .r8 (.reg .rdi)])) (.seq (rcxLoop 64 (dotBodyS c)) nttInvBody) + +end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Montgomery.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Montgomery.lean index c685bb933f..d2f896fa0a 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Montgomery.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Montgomery.lean @@ -1,6 +1,7 @@ module public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Lazy +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Dot /-! # Montgomery-scaled AVX2 products for ML-DSA -/ @@ -65,12 +66,32 @@ def ylast : Prog isa := /-- `NTT⁻¹` with Montgomery scaling: the eight layers with `vibflyLazy`, whose coefficients stay below `256 q < 2^32`, the last two of them with the reducing multiplication by `R/256`, which accepts any doubleword (`ylast`). -/ -def montNttInvAvx2 : Prog isa := VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 <| +def montInvBody : Prog isa := .seq (.block ypro) (.seq (.block (yconst .xmm11 1072693376)) (.seq (ylay1 vibflyLazy 248 yzeta8R (-32)) (.seq (ylay2 vibflyLazy 124 0x5F 0x0A (-16)) (.seq (ylay4 vibflyLazy 62 0x55 0x00 (-8)) (.seq (ylay vibflyLazy 8 31 (-4)) (.seq (ylay vibflyLazy 16 15 (-4)) (.seq (ylay vibflyLazy 32 7 (-4)) (.seq ylast (.block yepi))))))))) +def montNttInvAvx2 : Prog isa := VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 montInvBody + +/-! ## A row's sum of products, then `NTT⁻¹` -/ + +/-- The coefficients of `a[j]` and `b[j]`. -/ +def dotLoads (j : Nat) : List Instr := + [.vmovdquLoad .l256 .xmm0 (at_ .r10 (1024 * j)), .vmovdquLoad .l256 .xmm1 (at_ .rdx (1024 * j))] + +/-- Eight coefficients of the sum of the `c` products, scaled by `2⁻³²`. -/ +def dotBody (c : Nat) : List Instr := + toY dotZero ++ (List.range c).flatMap (fun j => dotLoads j ++ toY dotMac) ++ + toY dotRed ++ [.vmovdquStore .l256 (at_ .r8 0) .xmm5, .alu .add .r8 (.imm 32), .alu .add .r10 (.imm 32), + .alu .add .rdx (.imm 32)] + +/-- `vg_mldsa_dot_inverse{c}_avx2`: the sum of the `c` products scaled by +`2⁻³²`, then `NTT⁻¹` with Montgomery scaling, in one MXCSR region. -/ +def dotInvAvx2 (c : Nat) : Prog isa := + .seq (.block [.mov .r10 (.reg .rsi), .mov .rsi (.reg .rcx)]) <| VG.Impl.MlKem.X86_64.withMxcsr .rsi 768 <| + .seq (.block (yconsts ++ [.mov .r8 (.reg .rdi)])) (.seq (rcxLoop 32 (dotBody c)) montInvBody) + /-- AVX2 arithmetic with lazy forward NTT, scaled products and a compensating inverse NTT. -/ def Backend.avx2Combined : Backend := { Backend.avx2 with @@ -78,6 +99,9 @@ def Backend.avx2Combined : Backend := invNtt := montNttInvAvx2 mul := montMulAvx2 mulAdd := montMulAddAvx2 - montgomery := true } + montgomery := true + dotInv4 := dotInvAvx2 4 + dotInv5 := dotInvAvx2 5 + dotInv7 := dotInvAvx2 7 } end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean index 219c8fb3ee..df4e20f2d2 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Ntt.lean @@ -99,10 +99,13 @@ def ntt : Prog isa := withMxcsr .rsi 768 <| (.seq (vlay vbfly 16 8 4) (.seq (vlay vbfly 8 16 4) (.seq (vlay vbfly 4 32 4) (.seq (vlay2 vbfly 64 0x50 8) (vlay1 vbfly 128 0xE4 16)))))))) -def nttInv : Prog isa := withMxcsr .rsi 768 <| +/-- `NTT⁻¹` but for `withMxcsr`. -/ +def nttInvBody : Prog isa := .seq (.block vpro) (.seq (vlay1 vibfly 252 0x1B (-16)) (.seq (vlay2 vibfly 126 0x05 (-8)) (.seq (vlay vibfly 4 63 (-4)) (.seq (vlay vibfly 8 31 (-4)) (.seq (vlay vibfly 16 15 (-4)) (.seq (vlay vibfly 32 7 (-4)) (.seq (vlay vibfly 64 3 (-4)) (.seq (vlay vibfly 128 1 (-4)) vscale)))))))) +def nttInv : Prog isa := withMxcsr .rsi 768 nttInvBody + end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean index 2270aa5334..2ab821b7be 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean @@ -34,7 +34,6 @@ structure Prims where ntt : Prog isa invNtt : Prog isa mul : Prog isa - mulAdd : Prog isa add : Prog isa sub : Prog isa rejNTT : Prog isa @@ -55,6 +54,10 @@ structure Prims where /-- What the names of the polynomial arithmetic's functions end with (`Arith.Backend`). -/ sfx : String := "" montgomery : Bool := false + /-- `vg_mldsa_dot_inverse4`, `5` and `7` -/ + dotInv4 : Prog isa := .block [] + dotInv5 : Prog isa := .block [] + dotInv7 : Prog isa := .block [] /-! ## The layout of the working space (in bytes) @@ -184,9 +187,19 @@ def nttAt (f : Ptr) : Prog isa := callP ("vg_mldsa_ntt" ++ P.sfx) P.ntt [.ptr f, def invNttAt (f : Ptr) : Prog isa := callP ((if P.montgomery then "vg_mldsa_montgomery_inv_ntt" else "vg_mldsa_inv_ntt") ++ P.sfx) P.invNtt [.ptr f, .ptr (sc oPS)] +/-- `vg_mldsa_dot_inverse{c}`, for `c` of 4, 5 and 7. -/ +abbrev Prims.dotInv (P : Prims) (c : Nat) : Prog isa := if c = 4 then P.dotInv4 else if c = 5 then P.dotInv5 else P.dotInv7 + +/-- The name of `vg_mldsa_dot_inverse{c}`, written out (the kernel evaluates +code, and `toString` on `Nat` is well-founded recursion). -/ +abbrev dotInvName (c : Nat) : String := + if c = 4 then "vg_mldsa_dot_inverse4" else if c = 5 then "vg_mldsa_dot_inverse5" else "vg_mldsa_dot_inverse7" + +def dotInvAt (count : Nat) (h f g : Ptr) : Prog isa := + callP (dotInvName count ++ P.sfx) (P.dotInv count) [.ptr h, .ptr f, .ptr g, .ptr (sc oPS)] + def mulAt (h f g : Ptr) : Prog isa := callP ((if P.montgomery then "vg_mldsa_montgomery_multiply_ntt" else "vg_mldsa_multiply_ntt") ++ P.sfx) P.mul [.ptr h, .ptr f, .ptr g] -def mulAddAt (h f g : Ptr) : Prog isa := callP ((if P.montgomery then "vg_mldsa_montgomery_multiply_add_ntt" else "vg_mldsa_multiply_add_ntt") ++ P.sfx) P.mulAdd [.ptr h, .ptr f, .ptr g] def addAt (f g : Ptr) : Prog isa := callP ("vg_mldsa_add" ++ P.sfx) P.add [.ptr f, .ptr g] diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean index 8346deed78..38cc3b787d 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Sign.lean @@ -177,9 +177,7 @@ def mask4 (g : Nat) : Prog isa := .seq (seqR (cpM4 g) 0 4) (.seq (mask4At P p.γ₁ (yP p (4 * g)) (r4P p)) (seqR (yhR P p) (4 * g) 4)) /-- `w[i] = NTT⁻¹(∑_j Â[i, j] ŷ[j])`. -/ -def rowW (i : Nat) : Prog isa := - .seq (mulAt P (wP p i) (aP p i 0) (yhP p 0)) - (.seq (seqR (fun j => mulAddAt P (wP p i) (aP p i j) (yhP p j)) 1 (p.ℓ - 1)) (invNttAt P (wP p i))) +def rowW (i : Nat) : Prog isa := dotInvAt P p.ℓ (wP p i) (aP p i 0) (yhP p 0) /-- `w1Encode(HighBits(w[i]))` to `W1 + i · w1Len`. -/ def w1R (i : Nat) : Prog isa := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/Arith/DotSum.lean b/lean/VerifiedGarbage/Proof/MlDsa/Arith/DotSum.lean new file mode 100644 index 0000000000..4020b2e272 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/Arith/DotSum.lean @@ -0,0 +1,54 @@ +module + +public import VerifiedGarbage.Spec.MlDsa.FusedInverse +import VerifiedGarbage.Proof.MlDsa.Arith.Zq +public import VerifiedGarbage.Proof.Framework.PowLit +public import VerifiedGarbage.Spec.MlDsa + +/-! # ML-DSA: a coefficient of a row's sum of products + +Coefficient `i` of `dotNTT f g c` is the sum `dsum` of the products of the +natural numbers that the coefficients `i` of `f j` and `g j` reduce from. -/ + +@[expose] public section + +namespace VG.Proof.MlDsa.Arith +open VG.Spec.MlDsa + +/-- `∑_{j < c} A j · B j`. -/ +def dsum (A B : Nat → Nat) : Nat → Nat + | 0 => 0 + | c + 1 => dsum A B c + A c * B c + +theorem dsum_le {A B : Nat → Nat} {b : Nat} : ∀ {c}, (∀ j < c, A j * B j ≤ b) → dsum A B c ≤ c * b + | 0, _ => by simp [dsum] + | c + 1, h => by + rw [Nat.succ_mul]; exact Nat.add_le_add (dsum_le fun j hj => h j (by omega)) (h c (by omega)) + +theorem dotNTT_succ (f g : Nat → Poly) (c : Nat) : + dotNTT f g (c + 1) = add (dotNTT f g c) (multiplyNTT (f c) (g c)) := by + simp only [dotNTT, List.range_succ, List.map_append, List.map_cons, List.map_nil, List.foldl_append, + List.foldl_cons, List.foldl_nil] + +theorem dotNTT_get {f g : Nat → Poly} {A B : Nat → Nat} {i : Nat} (hi : i < n) : + ∀ {c : Nat}, (∀ j < c, (f j)[i]! = Fin.ofNat q (A j)) → (∀ j < c, (g j)[i]! = Fin.ofNat q (B j)) → + (dotNTT f g c)[i]! = Fin.ofNat q (dsum A B c) + | 0, _, _ => by + show zero[i]! = _ + rw [getElem!_eq zero hi] + simp only [zero, Vector.getElem_replicate] + rfl + | c + 1, hf, hg => by + rw [dotNTT_succ, add_get _ _ hi, mul_get _ _ hi, dotNTT_get hi (fun j hj => hf j (by omega)) + (fun j hj => hg j (by omega)), hf c (by omega), hg c (by omega)] + apply Fin.ext + simp only [dsum, Fin.val_add, Fin.val_mul, Fin.val_ofNat, Nat.add_mod, Nat.mul_mod, Nat.mod_mod] + +theorem dotNTT_congr {f g f' g' : Nat → Poly} {c : Nat} (hf : ∀ j < c, f j = f' j) (hg : ∀ j < c, g j = g' j) : + dotNTT f g c = dotNTT f' g' c := by + unfold dotNTT + congr 1 + exact List.map_congr_left fun j hj => by + rw [hf j (List.mem_range.mp hj), hg j (List.mem_range.mp hj)] + +end VG.Proof.MlDsa.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Avx2Lit.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Avx2Lit.lean index a7496ccf7c..426987fa57 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Avx2Lit.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Avx2Lit.lean @@ -31,6 +31,7 @@ namespace VG open Impl.MlDsa.X86_64 materialize_code Arith.lazyNtt +materialize_code Arith.montInvBody materialize_code Arith.montNttInvAvx2 materialize_code Arith.montMulAvx2 materialize_code Arith.montMulAddAvx2 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean index 637fcc0723..ccf228ed62 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -9,6 +9,7 @@ public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Dot public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.Bits public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.MakeHint public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.NormLt @@ -81,6 +82,7 @@ structure BackendOk (B : Backend) : Prop where useHint : FnOk (fun S => Spec.MlDsa.useHintContract X86_64.abi S) B.useHint rej4 : Rej4Ok B.rej4 expandMask4 : Mask4Ok B.expandMask4 + dotInv : ∀ c, c = 4 ∨ c = 5 ∨ c = 7 → FnOk (fun S => Spec.MlDsa.dotInverseContract c X86_64.abi S) (B.dotInv c) /-- An implementation of the polynomial arithmetic on x86-64. -/ structure ArithImpl where @@ -124,7 +126,15 @@ def ArithImpl.sse2 : ArithImpl where rej4 := ⟨Rej4.rejNTT4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, by decide +kernel, Code.all_of_allInstrs (by decide +kernel), fun _ _ _ => Rej4.rejNTT4_ret⟩ expandMask4 := ⟨Mask4.expandMask4_verified, Proof.MlKem.X86_64.nosp_of (by decide +kernel), by decide +kernel, - by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ } + by decide +kernel, Code.all_of_allInstrs (by decide +kernel)⟩ + dotInv := fun c hc => by + rcases hc with rfl | rfl | rfl + · exact FnOk.of Arith.DotS.verified4 (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + · exact FnOk.of Arith.DotS.verified5 (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + · exact FnOk.of Arith.DotS.verified7 (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) } features := [] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean index ef37aa0369..10659fef66 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -7,6 +7,7 @@ public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.LazyNtt public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMontgomery public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMontgomeryInv +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YDot public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YBits public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YHint public import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YNorm @@ -55,7 +56,12 @@ def ArithImpl.avx2 : ArithImpl where rej4 := ⟨Rej4.Segment.rejNTT4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by lit_decide), by lit_decide, by lit_decide, Code.all_of_allInstrs (by lit_decide), fun _ _ _ => Rej4.Segment.rejNTT4Avx2_ret⟩ expandMask4 := ⟨Mask4.expandMask4Avx2_verified, Proof.MlKem.X86_64.nosp_of (by lit_decide), - by lit_decide, by lit_decide, Code.all_of_allInstrs (by lit_decide)⟩ } + by lit_decide, by lit_decide, Code.all_of_allInstrs (by lit_decide)⟩ + dotInv := fun c hc => by + rcases hc with rfl | rfl | rfl + · exact FnOk.of Arith.YDot.verified4 (by lit_decide) (by lit_decide) (by lit_decide) (by lit_decide) + · exact FnOk.of Arith.YDot.verified5 (by lit_decide) (by lit_decide) (by lit_decide) (by lit_decide) + · exact FnOk.of Arith.YDot.verified7 (by lit_decide) (by lit_decide) (by lit_decide) (by lit_decide) } features := ["avx", "avx2"] end VG.Proof.MlDsa.X86_64 diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Dot.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Dot.lean new file mode 100644 index 0000000000..41fbaaa3ea --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Dot.lean @@ -0,0 +1,372 @@ +module + +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.DotCore +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul +public import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr +import VerifiedGarbage.Proof.Framework.X86_64.Avx +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VMem +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +import VerifiedGarbage.Proof.MlKem.X86_64.WritesOnly +import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr +import VerifiedGarbage.Proof.MlKem.X86_64.VLay +import VerifiedGarbage.Proof.Framework.Offset +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.DotCore +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.NttInv +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul +meta import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr +meta import VerifiedGarbage.Proof.Framework.X86_64.Avx +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VMem +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +meta import VerifiedGarbage.Proof.MlKem.X86_64.WritesOnly +meta import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr +meta import VerifiedGarbage.Proof.MlKem.X86_64.VLay +meta import VerifiedGarbage.Proof.Framework.Offset +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.MontgomeryCore +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.MontgomeryCore +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Ntt +public import VerifiedGarbage.Proof.Framework.X86_64.LaneSse +public import VerifiedGarbage.Proof.MlDsa.Arith.Mem +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLanes + +/-! +# ML-DSA on x86-64: `vg_mldsa_dot_inverse{c}` + +As `vg_mldsa_dot_inverse{c}_avx2` (`YDot.lean`) on four coefficients at a +time in SSE2 registers: each of the 64 iterations sums the `c` products in +quadwords (`macs_ok`) and reduces them (`dot_lane`) to the sum times `2⁻³²`, +which `dotPlain` multiplies by `2³²` (`plain_dw`), storing the sums modulo `q` +(`step`); then the inverse NTT (`invBody_ok`). +-/ + +@[expose] public section + +namespace VG.Proof.MlDsa.X86_64.Arith.DotS + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlDsa.X86_64.Arith +open VG.Proof.MlKem.X86_64 (writesIn writesOnly_of Keep XOnly WP.keep writesOnly gprPreserved_of ifp ifn GOnly + wp_rcxLoop add_ofNat_zero withMxcsr_ok mxR xmm_setXmm) +open VG.Impl.MlKem.X86_64 (xb xmov rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs montgomeryRInv) + +/-- A 128-bit load. -/ +theorem xld_ok {s : State} {p : Reg} {off : Nat} {d : XReg} + (h : InRegions (s.rd ++ s.wr) (s.gpr p + BitVec.ofNat 64 off) 16) : + WP isa (.block [.movdquLoad d (at_ p off)]) s fun s' => + s'.xmm d = s.mem.readW (s.gpr p + BitVec.ofNat 64 off) 128 ∧ XOnly [d] s s' := by + vrund [h] + refine ⟨⟨rfl, rfl, rfl, rfl, rfl⟩, fun r hr => ?_⟩ + rw [xmm_setXmm, ifn (by simpa using hr)] + +/-- The sums of the first `n` products of the four coefficients `Av j t`, +`Bv j t` (`t < 4`) in the quadwords of `xmm5` (even `t`) and `xmm6` (odd). -/ +def AccS (s : State) (Av Bv : Nat → Nat → Nat) (n : Nat) : Prop := + ∀ k < 2, + (qword (s.xmm .xmm5) k).toNat = dsum (fun j => Av j (2 * k)) (fun j => Bv j (2 * k)) n ∧ + (qword (s.xmm .xmm6) k).toNat = dsum (fun j => Av j (2 * k + 1)) (fun j => Bv j (2 * k + 1)) n + +/-- The first `n` products. -/ +theorem macs_ok {c : Nat} (hc : c ≤ 7) {Av Bv : Nat → Nat → Nat} (hA : ∀ j < c, ∀ t < 4, Av j t < 3 * q) + (hB : ∀ j < c, ∀ t < 4, Bv j t < 3 * q) {s : State} + (hr : ∀ j < c, InRegions (s.rd ++ s.wr) (s.gpr .r10 + BitVec.ofNat 64 (1024 * j)) 16) + (hr' : ∀ j < c, InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 (1024 * j)) 16) + (ha : ∀ j < c, ∀ e < 4, + (dword (s.mem.readW (s.gpr .r10 + BitVec.ofNat 64 (1024 * j)) 128) e).toNat = Av j e) + (hb : ∀ j < c, ∀ e < 4, + (dword (s.mem.readW (s.gpr .rdx + BitVec.ofNat 64 (1024 * j)) 128) e).toNat = Bv j e) + (h0 : AccS s Av Bv 0) : + ∀ n ≤ c, WP isa (.block ((List.range n).flatMap fun j => dotLoadsS j ++ dotMac)) s fun s' => + AccS s' Av Bv n ∧ XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm5, .xmm6] s s' + | 0, _ => by + simp only [List.range_zero, List.flatMap_nil] + vrund + exact ⟨h0, XOnly.refl _ s⟩ + | n + 1, hn => by + rw [List.range_succ, List.flatMap_append, List.flatMap_cons, List.flatMap_nil, List.append_nil, + WP.block_append_iff] + refine WP.mono (macs_ok hc hA hB hr hr' ha hb h0 n (by omega)) fun s1 ⟨A1, o1⟩ => ?_ + rw [dotLoadsS, show ∀ (x y : Instr) (l : List Instr), [x, y] ++ l = [x] ++ ([y] ++ l) from fun _ _ _ => rfl, + WP.block_append_iff] + refine WP.mono (xld_ok (by rw [o1.rd, o1.wr, o1.gpr]; exact hr n (by omega))) fun s2 ⟨L2, o2⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (xld_ok (by rw [o2.rd, o2.wr, o2.gpr, o1.rd, o1.wr, o1.gpr]; exact hr' n (by omega))) + fun s3 ⟨L3, o3⟩ => ?_ + refine WP.mono (dotMac_ok s3) fun s4 ⟨M5, M6, o4⟩ => ⟨fun k hk => ?_, (((o1.trans o2).trans o3).trans o4).mono (by simp)⟩ + have x0 : s3.xmm .xmm0 = s.mem.readW (s.gpr .r10 + BitVec.ofNat 64 (1024 * n)) 128 := by + rw [o3.xmm _ (by decide), L2, o1.mem, o1.gpr] + have x1 : s3.xmm .xmm1 = s.mem.readW (s.gpr .rdx + BitVec.ofNat 64 (1024 * n)) 128 := by + rw [L3, o2.mem, o2.gpr, o1.mem, o1.gpr] + have x5 : s3.xmm .xmm5 = s1.xmm .xmm5 := by rw [o3.xmm _ (by decide), o2.xmm _ (by decide)] + have x6 : s3.xmm .xmm6 = s1.xmm .xmm6 := by rw [o3.xmm _ (by decide), o2.xmm _ (by decide)] + have a0 := ha n (by omega) (2 * k) (by omega) + have a1 := ha n (by omega) (2 * k + 1) (by omega) + have b0 := hb n (by omega) (2 * k) (by omega) + have b1 := hb n (by omega) (2 * k + 1) (by omega) + rw [← x0] at a0 a1 + rw [← x1] at b0 b1 + refine ⟨?_, ?_⟩ + · rw [M5, qword_macE hk (by rw [x5, (A1 k hk).1, a0, b0]; exact prod_bound hc hA hB (by omega) (by omega)), x5, + (A1 k hk).1, a0, b0] + rfl + · rw [M6, qword_macO hk (by rw [x6, (A1 k hk).2, a1, b1]; exact prod_bound hc hA hB (by omega) (by omega)), x6, + (A1 k hk).2, a1, b1] + rfl + +theorem dotPlain_ok {s : State} (hc : VConsts s) : + WP isa (.block dotPlain) s fun s' => s'.xmm .xmm5 = csubV (montV (s.xmm .xmm5) (s.xmm .xmm11) (s.xmm .xmm11)) ∧ + XOnly [.xmm2, .xmm4, .xmm5] s s' := by + simp only [dotPlain, vmont, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv] + exact ⟨rfl, by xonly⟩ + +/-- The sum times `2⁻³²`, times `2⁶⁴` times `2⁻³²`: the sum modulo `q`. -/ +theorem plain_dw {x : BitVec 128} {P : Nat} {e : Nat} (he : e < 4) + (hx : (dword x e).toNat = (Fin.ofNat q P * montgomeryRInv).val) : + (dword (csubV (montV x r2V r2V)) e).toNat = (Fin.ofNat q P : Zq).val := by + have hzo : ∀ j < 2, dword r2V (2 * j) = dword r2V (2 * j + 1) := fun j hj => by + rw [dword_r2V (by omega), dword_r2V (by omega)] + have hb : ∀ i < 4, (dword x i).toNat * (dword r2V i).toNat < q * 2 ^ 32 := fun i hi => by + rw [dword_r2V hi] + exact Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le (dword x i).isLt (Nat.le_refl _) (by decide)) (by decide) + have hm := dword_montV hzo hb he + rw [dword_csubV _ he, csubL_toNat (by rw [hm]; exact mont_lt (hb e he)), hm, condSub_mont (hb e he), mont_inverse, + dword_r2V he, hx] + have h3 : ∀ (z : Zq) (a b : Nat), a < q → b < q → (z.val * a * b) % q = (z * Fin.ofNat q a * Fin.ofNat q b).val := + fun z a b ha hb => by + rw [Fin.val_mul, Fin.val_mul, Fin.val_ofNat, Fin.val_ofNat, Nat.mod_eq_of_lt ha, Nat.mod_eq_of_lt hb, + Nat.mod_mul_mod] + rw [show (2365951#32).toNat = 2365951 from rfl, h3 _ _ _ (by decide) (by decide), Fin.mul_assoc, Fin.mul_assoc, + show montgomeryRInv * (Fin.ofNat q 2365951 * Fin.ofNat q 8265825) = 1 by decide, Fin.mul_one] + +/-- Coefficient `k` of the sum of the products. -/ +def dotP (m : VG.Mem) (a b : Addr) (c k : Nat) : Zq := + Fin.ofNat q (dsum (fun j => famV m a j k) (fun j => famV m b j k) c) + +/-- After `i` iterations of the loop, from `s₀`. -/ +structure DI (o a b : Addr) (c : Nat) (s₀ : State) (i : Nat) (s : State) : Prop where + r8 : s.gpr .r8 = coeffAddr o (4 * i) + r10 : s.gpr .r10 = coeffAddr a (4 * i) + rdx : s.gpr .rdx = coeffAddr b (4 * i) + keep : Keep [.r8, .r10, .rdx, .rcx, .rax] s₀ s + vc : VConsts s + x11 : s.xmm .xmm11 = r2V + frame : Frame [pR o] s₀.mem s.mem + done : ∀ k < 4 * i, (coeffAt s.mem o k).toNat = (dotP s₀.mem a b c k).val + +section +variable {o a b : Addr} {c : Nat} {s₀ : State} (hc7 : c ≤ 7) (hwo : pR o ∈ s₀.wr) + (hra : ∀ j < c, ∀ k, k + 4 ≤ 256 → InRegions (s₀.rd ++ s₀.wr) (coeffAddr (a + BitVec.ofNat 64 (1024 * j)) k) 16) + (hrb : ∀ j < c, ∀ k, k + 4 ≤ 256 → InRegions (s₀.rd ++ s₀.wr) (coeffAddr (b + BitVec.ofNat 64 (1024 * j)) k) 16) + (hda : ∀ j < c, (pR (a + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) + (hdb : ∀ j < c, (pR (b + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) + (hA : ∀ j < c, ∀ k < 256, famV s₀.mem a j k < 3 * q) (hB : ∀ j < c, ∀ k < 256, famV s₀.mem b j k < 3 * q) +include hc7 hwo hra hrb hda hdb hA hB + +theorem step {i : Nat} (hi : i < 64) {s : State} (hI : DI o a b c s₀ i s) : + WP isa (.block (dotBodyS c ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + DI o a b c s₀ (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 4 * i + 4 ≤ 256 := by omega + have hv : ∀ (p : Addr), (∀ j < c, (pR (p + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) → ∀ j < c, + ∀ e < 4, (dword (s.mem.readW (coeffAddr p (4 * i) + BitVec.ofNat 64 (1024 * j)) 128) e).toNat = + famV s₀.mem p j (4 * i + e) := fun p hd j hj e he => by + rw [addr_fam, dword_readW _ _ he, coeffAddr_add, ← coeffAt_eq, famV, + coeffAt_frame hI.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd j hj) (by rw [n_eq]; omega)] + simp only [dotBodyS, List.append_assoc] + rw [WP.block_append_iff] + refine WP.mono (dotZero_ok s) fun s1 ⟨z5, z6, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (macs_ok hc7 (Av := fun j t => famV s₀.mem a j (4 * i + t)) (Bv := fun j t => famV s₀.mem b j (4 * i + t)) + (fun j hj t ht => hA j hj _ (by omega)) (fun j hj t ht => hB j hj _ (by omega)) + (fun j hj => by + rw [o1.rd, o1.wr, o1.gpr, hI.r10, hI.keep.2.1, hI.keep.2.2, addr_fam]; exact hra j hj _ j0) + (fun j hj => by + rw [o1.rd, o1.wr, o1.gpr, hI.rdx, hI.keep.2.1, hI.keep.2.2, addr_fam]; exact hrb j hj _ j0) + (fun j hj e he => by rw [o1.mem, o1.gpr, hI.r10]; exact hv a hda j hj e he) + (fun j hj e he => by rw [o1.mem, o1.gpr, hI.rdx]; exact hv b hdb j hj e he) + (fun k hk => by rw [z5, z6]; exact ⟨by simp [qword, dsum], by simp [qword, dsum]⟩) c (Nat.le_refl _)) + fun s2 ⟨A2, o2⟩ => ?_ + have o12 := o1.trans o2 + rw [WP.block_append_iff] + refine WP.mono (dotRed_ok (xonly_vconsts o12 hI.vc (by decide) (by decide))) fun s3 ⟨R3, o3⟩ => ?_ + have o13 := o12.trans o3 + rw [WP.block_append_iff] + refine WP.mono (dotPlain_ok (xonly_vconsts o13 hI.vc (by decide) (by decide))) fun s4 ⟨P4, o4⟩ => ?_ + have o14 := o13.trans o4 + have x11 : s3.xmm .xmm11 = r2V := by rw [o13.xmm _ (by decide), hI.x11] + have hres : ∀ e < 4, (dword (s4.xmm .xmm5) e).toNat = (dotP s₀.mem a b c (4 * i + e)).val := by + intro e he + rw [P4, x11] + refine plain_dw he ?_ + rw [R3, dot_lane (A := fun j t => famV s₀.mem a j (4 * i + t)) (B := fun j t => famV s₀.mem b j (4 * i + t)) hc7 + (fun j hj t ht => hA j hj _ (by omega)) (fun j hj t ht => hB j hj _ (by omega)) + (fun k hk => (A2 k hk).1) (fun k hk => (A2 k hk).2) he] + have g4 : s4.gpr .r8 = coeffAddr o (4 * i) := by rw [o14.gpr, hI.r8] + have w0 : InRegions s4.wr (coeffAddr o (4 * i)) 16 := by rw [o14.wr, hI.keep.2.2]; exact f_in hwo j0 + vrund [g4, w0] + rw [o14.mem] + have p16 : ∀ p : Addr, coeffAddr p (4 * i) + 16 = coeffAddr p (4 * (i + 1)) := fun p => by + rw [show (16 : BitVec 64) = BitVec.ofNat 64 (4 * 4) from rfl, coeffAddr_add, show 4 * i + 4 = 4 * (i + 1) by omega] + refine ⟨⟨?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, ?_, fun k hk => ?_⟩, by rw [o14.gpr], by rw [o14.gpr]⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + exact p16 o + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [o14.gpr, hI.r10]; exact p16 a + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [o14.gpr, hI.rdx]; exact p16 b + · have hr0 := hr + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2.1, hr.2.2.1, hr.2.2.2.1, ite_false] + rw [o14.gpr, hI.keep.1 r hr0] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags]; rw [o14.rd, hI.keep.2.1] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags]; rw [o14.wr, hI.keep.2.2] + · exact ⟨by simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; rw [o14.xmm _ (by decide)]; exact hI.vc.q, + by simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; rw [o14.xmm _ (by decide)]; exact hI.vc.qinv⟩ + · simp only [RegUpd.xmm_setReg, RegUpd.xmm_setFlags]; rw [o14.xmm _ (by decide), hI.x11] + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags] + exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains o j0) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags] + rw [coeffAt_write128 _ _ j0 _ (by omega)] + split + · rw [hres _ (by omega), show 4 * i + (k - 4 * i) = k by omega] + · exact hI.done k (by omega) + + +/-- The loop, from the constants and the pointers, in `s₀`. -/ +theorem pass_ok (h8 : s₀.gpr .r8 = o) (h10 : s₀.gpr .r10 = a) (hdx : s₀.gpr .rdx = b) (hc : VConsts s₀) + (h11 : s₀.xmm .xmm11 = r2V) : + WP isa (rcxLoop 64 (dotBodyS c)) s₀ fun s' => PolyIs s'.mem o (Vector.ofFn fun k => dotP s₀.mem a b c k.val) ∧ + Keep [.r8, .r10, .rdx, .rcx, .rax] s₀ s' ∧ Frame [pR o] s₀.mem s'.mem := by + refine WP.mono (wp_rcxLoop (N := 64) (by decide) (by decide) (DI o a b c s₀) + (fun u og _ => ⟨by rw [og.keep.gpr (by decide), h8, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [og.keep.gpr (by decide), h10, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [og.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + og.keep.mono (by simp), ⟨by rw [og.xmm]; exact hc.q, by rw [og.xmm]; exact hc.qinv⟩, + by rw [og.xmm]; exact h11, by rw [og.mem]; exact Frame.refl _ _, fun k hk => absurd hk (by omega)⟩) + (fun i hi u hI => step hc7 hwo hra hrb hda hdb hA hB hi hI)) fun u hI => ⟨⟨fun k hk => ?_, ?_⟩, hI.keep, hI.frame⟩ + · rw [n_eq] at hk + rw [hI.done k (by omega)]; exact (dotP s₀.mem a b c k).isLt + · apply Vector.ext; intro k hk + rw [polyAt, Vector.getElem_ofFn, Vector.getElem_ofFn] + apply Fin.ext + rw [Fin.val_ofNat, hI.done k (by rw [n_eq] at hk; omega), Nat.mod_eq_of_lt (dotP s₀.mem a b c k).isLt] + +end + +/-! ## The function -/ + +/-- The result: the sums. -/ +theorem dotP_eq {m m' : VG.Mem} {a b : Addr} {c : Nat} + (ha : ∀ j < c, ∀ k < 256, coeffAt m' (a + BitVec.ofNat 64 (1024 * j)) k = coeffAt m (a + BitVec.ofNat 64 (1024 * j)) k) + (hb : ∀ j < c, ∀ k < 256, coeffAt m' (b + BitVec.ofNat 64 (1024 * j)) k = coeffAt m (b + BitVec.ofNat 64 (1024 * j)) k) : + (Vector.ofFn fun k => dotP m' a b c k.val) = Spec.MlDsa.dotNTT + (fun j => polyAt m (a + BitVec.ofNat 64 (1024 * j))) (fun j => polyAt m (b + BitVec.ofNat 64 (1024 * j))) c := by + apply Vector.ext; intro k hk + rw [Vector.getElem_ofFn, ← getElem!_pos (Spec.MlDsa.dotNTT _ _ c) k hk, + dotNTT_get (A := fun j => famV m' a j k) (B := fun j => famV m' b j k) hk + (fun j hj => by rw [polyAt_get _ _ hk, famV, ha j hj k hk]) (fun j hj => by rw [polyAt_get _ _ hk, famV, hb j hj k hk])] + rfl + +theorem correct {c : Nat} (hc7 : c ≤ 7) + (hk : Code.allInstrs (writesIn [.rax, .rcx, .rdx, .r8, .r9, .r10, .r11, .rsi]) (dotInvSse2 c) = true) + (hk' : Code.allInstrs (writesIn [.rax, .rcx, .rdx, .r8, .r9, .r10]) + (.seq (.block (mulPro ++ ([.mov .r8 (.reg .rdi)] : List Instr))) (.seq (rcxLoop 64 (dotBodyS c)) nttInvBody)) = true) + (hctl : ctlOk (dotInvSse2 c) = true) (s : State) (hs : (dotK c).pre s) : + ∃ t s', Exec isa (dotInvSse2 c) s t s' ∧ abiPreserved s s' ∧ (dotK c).post s s' := by + obtain ⟨hrd, hwr, d1, d2, d3, d4, d5, r1, r2, pa, pb⟩ := hs + have hwo : pR (s.gpr .rdi) ∈ s.wr := by rw [hwr]; simp + have hws : pR (s.gpr .rcx) ∈ s.wr := by rw [hwr]; simp + have hW : WP isa (dotInvSse2 c) s fun s' => PolyIs s'.mem (s.gpr .rdi) (Spec.MlDsa.nttInv (Spec.MlDsa.dotNTT + (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem s'.mem := by + refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .r10 = s.gpr .rsi ∧ w.gpr .rsi = s.gpr .rcx ∧ + GOnly [.r10, .rsi] s w) + (by vrund; gonlyd) fun w ⟨h10, hsi, og⟩ => ?_) + refine WP.mono (withMxcsr_ok (r := .rsi) (Q := fun u => PolyIs u.mem (s.gpr .rdi) (Spec.MlDsa.nttInv + (Spec.MlDsa.dotNTT (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem u.mem) ⟨by decide, by decide⟩ [.rax, .rcx, .rdx, .r8, .r9, .r10] + ⟨by decide, by decide⟩ hsi (by rw [og.keep.2.2]; exact hws) (writesOnly_of hk') (fun s1 k1 f1 => ?_)) + fun s' ⟨s2, ⟨hP, hf⟩, hf', _⟩ => ⟨?_, ?_⟩ + rotate_left + · exact polyIs_frame hf' (fun r hr => by + rw [List.mem_singleton.mp hr]; exact d3.sub_right (mx_sub' _)) hP + · exact hf.trans (frame_fs hf' fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _)) + have hdi1 : s1.gpr .rdi = s.gpr .rdi := by rw [k1.gpr (by decide), og.keep.gpr (by decide)] + have hsi1 : s1.gpr .rsi = s.gpr .rcx := by rw [k1.gpr (by decide), hsi] + have h101 : s1.gpr .r10 = s.gpr .rsi := by rw [k1.gpr (by decide), h10] + have hdx1 : s1.gpr .rdx = s.gpr .rdx := by rw [k1.gpr (by decide), og.keep.gpr (by decide)] + have wr1 : s1.wr = s.wr := by rw [k1.2.2, og.keep.2.2] + have rd1 : s1.rd = s.rd := by rw [k1.2.1, og.keep.2.1] + have hm : ∀ p : Addr, (pR (s.gpr .rcx)).Disjoint (famR p c) → ∀ j < c, ∀ k < 256, + coeffAt s1.mem (p + BitVec.ofNat 64 (1024 * j)) k = coeffAt s.mem (p + BitVec.ofNat 64 (1024 * j)) k := + fun p hd j hj k hk => by + rw [coeffAt_frame f1 (fun r hr => by + rw [List.mem_singleton.mp hr]; exact (hd.sub_right (fam_sub p hj)).symm.sub_right (mx_sub' _)) + (by rw [n_eq]; exact hk), og.mem] + refine WP.seq ?_ + rw [WP.block_append_iff] + refine WP.mono (mulPro_ok s1) fun s2 ⟨c2, x2, _, k2, m2⟩ => ?_ + have hdi2 : s2.gpr .rdi = s.gpr .rdi := by rw [k2.gpr (by decide), hdi1] + refine WP.mono (Q := fun (s3 : State) => s3.gpr .r8 = s.gpr .rdi ∧ GOnly [.r8] s2 s3) + (by vrund [hdi2]; gonlyd) fun s3 ⟨h8, og3⟩ => ?_ + have c3 : VConsts s3 := ⟨by rw [og3.xmm]; exact c2.q, by rw [og3.xmm]; exact c2.qinv⟩ + have x3 : s3.xmm .xmm11 = r2V := by rw [og3.xmm]; exact x2 + have k13 : Keep [.rax, .r8] s1 s3 := (k2.trans og3.keep).mono (by simp) + have m3 : s3.mem = s1.mem := by rw [og3.mem, m2] + have hr : ∀ p : Addr, famR p c ∈ s.rd → ∀ j < c, ∀ k, k + 4 ≤ 256 → + InRegions (s3.rd ++ s3.wr) (coeffAddr (p + BitVec.ofNat 64 (1024 * j)) k) 16 := fun p hp j hj k hk => + ⟨famR p c, by rw [k13.2.1, rd1]; exact List.mem_append_left _ hp, by + rw [coeffAddr, VG.Offset.add_add]; exact VG.Offset.contains_base _ (by omega) (by omega)⟩ + refine WP.seq (WP.mono (pass_ok (s₀ := s3) hc7 (by rw [k13.2.2, wr1]; exact hwo) + (hr _ (by rw [hrd]; simp)) (hr _ (by rw [hrd]; simp)) + (fun j hj => (d1.sub_right (fam_sub _ hj)).symm) (fun j hj => (d2.sub_right (fam_sub _ hj)).symm) + (fun j hj k hk => by rw [famV, m3, hm _ d4 j hj k hk]; exact pa j hj k (by rw [n_eq]; exact hk)) + (fun j hj k hk => by rw [famV, m3, hm _ d5 j hj k hk]; exact pb j hj k (by rw [n_eq]; exact hk)) + h8 (by rw [k13.gpr (by decide), h101]) (by rw [k13.gpr (by decide), hdx1]) c3 x3) fun u ⟨hP, ku, fu⟩ => ?_) + refine WP.mono (invBody_ok (by rw [ku.gpr (by decide), k13.gpr (by decide), hdi1]) + (by rw [ku.gpr (by decide), k13.gpr (by decide), hsi1]) hP + (by rw [ku.2.2, k13.2.2, wr1]; exact hwo) (by rw [ku.2.2, k13.2.2, wr1]; exact hws) d3) fun v ⟨hv, fv⟩ => ⟨?_, ?_⟩ + · rw [dotP_eq (m := s.mem) (fun j hj k hk => by rw [m3, hm _ d4 j hj k hk]) + (fun j hj k hk => by rw [m3, hm _ d5 j hj k hk])] at hv + exact hv + · have f1' : Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem s1.mem := by + rw [← og.mem]; exact frame_fs f1 fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _) + have fu' : Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s1.mem u.mem := by + rw [← m3]; exact frame_fs fu fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inl fun _ h => h + exact (f1'.trans fu').trans fv + obtain ⟨tr, s', he, ⟨hP, hf⟩, hk2⟩ := WP.keep [.rax, .rcx, .rdx, .r8, .r9, .r10, .r11, .rsi] hW (writesOnly_of hk) + exact ⟨tr, s', he, abiPreserved_of_ctl hctl he (gprPreserved_of hk2 (by decide) hf (by simpa using ⟨r1, r2⟩)), hP⟩ + +theorem ct4 : ConstantTime isa (dotK 4).pre (dotK 4).pub (dotInvSse2 4) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 4) (by taint_decide) + +theorem verified4 : Verified X86_64.target (dotInvSse2 4) (Spec.MlDsa.dotInverseContract 4 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct4 + (by dot_implies dotSat 4) + +theorem ct5 : ConstantTime isa (dotK 5).pre (dotK 5).pub (dotInvSse2 5) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 5) (by taint_decide) + +theorem verified5 : Verified X86_64.target (dotInvSse2 5) (Spec.MlDsa.dotInverseContract 5 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct5 + (by dot_implies dotSat 5) + +theorem ct7 : ConstantTime isa (dotK 7).pre (dotK 7).pub (dotInvSse2 7) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 7) (by taint_decide) + +theorem verified7 : Verified X86_64.target (dotInvSse2 7) (Spec.MlDsa.dotInverseContract 7 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct7 + (by dot_implies dotSat 7) + +end VG.Proof.MlDsa.X86_64.Arith.DotS diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/DotCore.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/DotCore.lean new file mode 100644 index 0000000000..825e765f27 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/DotCore.lean @@ -0,0 +1,211 @@ +module + +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.MontgomeryCore +public import VerifiedGarbage.Proof.MlDsa.Arith.DotSum +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Dot +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic +public import VerifiedGarbage.Proof.Framework.TacticTemplate +public meta import VerifiedGarbage.Proof.Framework.TacticTemplate +public import VerifiedGarbage.Spec.MlDsa.FusedInverse +public import VerifiedGarbage.Proof.MlDsa.Arith.Mem +import VerifiedGarbage.Proof.Framework.Offset +import VerifiedGarbage.Proof.Framework.X86_64.Exec +import VerifiedGarbage.Proof.Framework.X86_64.Avx +import VerifiedGarbage.Proof.MlKem.X86_64.Wp +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Backend +public import VerifiedGarbage.Proof.Framework.X86_64.LaneSse +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLanes +public import VerifiedGarbage.Spec.MlDsa.Montgomery +public meta import Lean.Elab.Tactic.Basic + +/-! +# ML-DSA on x86-64: a row's sum of products, in the doublewords of an SSE register + +`dotMac` adds the products of the even doublewords of `xmm0` and `xmm1` to +the quadwords of `xmm5` (`macE`) and those of the odd ones to `xmm6` +(`macO`); `dotRed` reduces each quadword (`redV`, `dword_redV`) and merges +them. From quadword sums `dsum` of at most seven products of coefficients +below `3 q`, each doubleword of the result is the sum times `2⁻³²`, reduced +(`dot_lane`). +-/ + +@[expose] public section + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlKem.X86_64 (XOnly) +open VG.Impl.MlKem.X86_64 (xb xmov) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt PolyIs montgomeryRInv) + +/-- The quadwords of `x` plus `m · q` (`vredc`). -/ +def vredcV (x : BitVec 128) : BitVec 128 := + XBinOp.eval .paddq x (XBinOp.eval .pmuludq (XBinOp.eval .pmuludq x qinvV) qV) + +/-- `dotRed`'s merge of the reductions of `e` (even) and `o` (odd). -/ +def redV (e o : BitVec 128) : BitVec 128 := XBinOp.eval .por (XShiftOp.eval .psrlq (vredcV e) 32) (vredcV o) + +/-- The products of the even doublewords of `a` and `b` added to `acc`. -/ +def macE (acc a b : BitVec 128) : BitVec 128 := XBinOp.eval .paddq acc (XBinOp.eval .pmuludq a b) + +/-- The products of the odd doublewords of `a` and `b` added to `acc`. -/ +def macO (acc a b : BitVec 128) : BitVec 128 := + XBinOp.eval .paddq acc (XBinOp.eval .pmuludq (XShiftOp.eval .psrlq a 32) (XShiftOp.eval .psrlq b 32)) + +theorem dotZero_ok (s : State) : + WP isa (.block dotZero) s fun s' => s'.xmm .xmm5 = 0 ∧ s'.xmm .xmm6 = 0 ∧ XOnly [.xmm5, .xmm6] s s' := by + simp only [dotZero, xb] + vrun + exact ⟨by simp [XBinOp.eval], by simp [XBinOp.eval], by xonly⟩ + +theorem dotMac_ok (s : State) : + WP isa (.block dotMac) s fun s' => s'.xmm .xmm5 = macE (s.xmm .xmm5) (s.xmm .xmm0) (s.xmm .xmm1) ∧ + s'.xmm .xmm6 = macO (s.xmm .xmm6) (s.xmm .xmm0) (s.xmm .xmm1) ∧ + XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm5, .xmm6] s s' := by + simp only [dotMac, xmov, xb] + vrun [eval_movdqa] + exact ⟨rfl, rfl, by xonly⟩ + +theorem dotRed_ok {s : State} (hc : VConsts s) : + WP isa (.block dotRed) s fun s' => s'.xmm .xmm5 = csubV (redV (s.xmm .xmm5) (s.xmm .xmm6)) ∧ + XOnly [.xmm2, .xmm5, .xmm6] s s' := by + simp only [dotRed, vredc, vcsub, vcadd, xmov, xb, List.cons_append, List.nil_append] + vrun [eval_movdqa] + rw [hc.q, hc.qinv] + exact ⟨rfl, by xonly⟩ + +theorem qword_macE {acc a b : BitVec 128} {k : Nat} (hk : k < 2) + (h : (qword acc k).toNat + (dword a (2 * k)).toNat * (dword b (2 * k)).toNat < 2 ^ 64) : + (qword (macE acc a b) k).toNat = (qword acc k).toNat + (dword a (2 * k)).toNat * (dword b (2 * k)).toNat := by + rw [macE, qword_paddq _ _ hk, BitVec.toNat_add, toNat_qword_pmuludq _ _ hk, Nat.mod_eq_of_lt h] + +theorem dword_psrlq32 (a : BitVec 128) {k : Nat} (hk : k < 2) : + dword (XShiftOp.eval .psrlq a 32) (2 * k) = dword a (2 * k + 1) := by + apply BitVec.eq_of_toNat_eq + rw [dword_lo, dword_hi, qword_psrlq32 _ hk, lo_shr32] + +theorem qword_macO {acc a b : BitVec 128} {k : Nat} (hk : k < 2) + (h : (qword acc k).toNat + (dword a (2 * k + 1)).toNat * (dword b (2 * k + 1)).toNat < 2 ^ 64) : + (qword (macO acc a b) k).toNat = + (qword acc k).toNat + (dword a (2 * k + 1)).toNat * (dword b (2 * k + 1)).toNat := by + rw [macO, qword_paddq _ _ hk, BitVec.toNat_add, toNat_qword_pmuludq _ _ hk, dword_psrlq32 _ hk, + dword_psrlq32 _ hk, Nat.mod_eq_of_lt h] + +theorem dword_redV {e o : BitVec 128} (he : ∀ k < 2, (qword e k).toNat < q * 2 ^ 32) + (ho : ∀ k < 2, (qword o k).toNat < q * 2 ^ 32) {k : Nat} (hk : k < 2) : + (dword (redV e o) (2 * k)).toNat = mont (qword e k).toNat ∧ + (dword (redV e o) (2 * k + 1)).toNat = mont (qword o k).toNat := by + have hqi : (lo32 (qword qinvV k)).toNat = montQInv := lo32_qword_qinvV hk + have hqq : (lo32 (qword qV k)).toNat = q := lo32_qword_qV hk + have ve : qword (vredcV e) k = redc (qword e k) (qword qinvV k) (qword qV k) := qword_vredc _ hk + have vo : qword (vredcV o) k = redc (qword o k) (qword qinvV k) (qword qV k) := qword_vredc _ hk + constructor + · rw [redV, dword_por, dword_lo, dword_lo, qword_psrlq32 _ hk, ve, vo, redc_lo hqi hqq (ho k hk), + or_zero_toNat, lo_shr32, redc_hi hqi hqq (he k hk)] + · rw [redV, dword_por, dword_hi, dword_hi, qword_psrlq32 _ hk, hi_shr32, zero_or_toNat, vo, + redc_hi hqi hqq (ho k hk)] + +/-- A sum of at most seven products of coefficients below `3 q`, reduced: +each doubleword of `dotRed`'s result from the sums in the quadwords. -/ +theorem dot_lane {e o : BitVec 128} {A B : Nat → Nat → Nat} {c : Nat} (hc : c ≤ 7) + (hA : ∀ j < c, ∀ i < 4, A j i < 3 * q) (hB : ∀ j < c, ∀ i < 4, B j i < 3 * q) + (he : ∀ k < 2, (qword e k).toNat = dsum (fun j => A j (2 * k)) (fun j => B j (2 * k)) c) + (ho : ∀ k < 2, (qword o k).toNat = dsum (fun j => A j (2 * k + 1)) (fun j => B j (2 * k + 1)) c) + {i : Nat} (hi : i < 4) : + (dword (csubV (redV e o)) i).toNat = (Fin.ofNat q (dsum (fun j => A j i) (fun j => B j i) c) * montgomeryRInv).val := by + have hb : ∀ i < 4, dsum (fun j => A j i) (fun j => B j i) c < q * 2 ^ 32 := fun i hi => by + have h1 := dsum_le (A := fun j => A j i) (B := fun j => B j i) (b := 3 * q * (3 * q)) (c := c) + (fun j hj => Nat.mul_le_mul (Nat.le_of_lt (hA j hj i hi)) (Nat.le_of_lt (hB j hj i hi))) + exact Nat.lt_of_le_of_lt (Nat.le_trans h1 (Nat.mul_le_mul_right _ hc)) (by decide) + have he' : ∀ k < 2, (qword e k).toNat < q * 2 ^ 32 := fun k hk => by rw [he k hk]; exact hb _ (by omega) + have ho' : ∀ k < 2, (qword o k).toNat < q * 2 ^ 32 := fun k hk => by rw [ho k hk]; exact hb _ (by omega) + have fin : ∀ x : Nat, condSub (mont x) = mont x % q → mont x % q = x * 8265825 % q → + condSub (mont x) = (Fin.ofNat q x * montgomeryRInv).val := fun x h1 h2 => by + rw [h1, h2, Fin.val_mul, Fin.val_ofNat, Nat.mul_mod]; rfl + obtain ⟨k, hk, rfl | rfl⟩ : ∃ k < 2, i = 2 * k ∨ i = 2 * k + 1 := ⟨i / 2, by omega, by omega⟩ + · rw [dword_csubV _ hi, csubL_toNat (by rw [(dword_redV he' ho' hk).1]; exact mont_lt (he' k hk)), + (dword_redV he' ho' hk).1, he k hk] + exact fin _ (condSub_mont (hb _ hi)) (mont_inverse _) + · rw [dword_csubV _ hi, csubL_toNat (by rw [(dword_redV he' ho' hk).2]; exact mont_lt (ho' k hk)), + (dword_redV he' ho' hk).2, ho k hk] + exact fin _ (condSub_mont (hb _ hi)) (mont_inverse _) + +/-! ## The function's contract, for every implementation -/ + +/-- Coefficient `k` of polynomial `j` of the family at `p`. -/ +def famV (m : VG.Mem) (p : Addr) (j k : Nat) : Nat := (coeffAt m (p + BitVec.ofNat 64 (1024 * j)) k).toNat + +theorem prod_bound {c T : Nat} (hc : c ≤ 7) {Av Bv : Nat → Nat → Nat} (hA : ∀ j < c, ∀ t < T, Av j t < 3 * q) + (hB : ∀ j < c, ∀ t < T, Bv j t < 3 * q) {n : Nat} (hn : n < c) {t : Nat} (ht : t < T) : + dsum (fun j => Av j t) (fun j => Bv j t) n + Av n t * Bv n t < 2 ^ 64 := by + have h := dsum_le (A := fun j => Av j t) (B := fun j => Bv j t) (b := 3 * q * (3 * q)) (c := n + 1) + (fun j hj => Nat.mul_le_mul (Nat.le_of_lt (hA j (by omega) t ht)) (Nat.le_of_lt (hB j (by omega) t ht))) + rw [dsum] at h + exact Nat.lt_of_le_of_lt (Nat.le_trans h (Nat.mul_le_mul_right _ (show n + 1 ≤ 8 by omega))) (by decide) + +theorem addr_fam (p : Addr) (k j : Nat) : + coeffAddr p k + BitVec.ofNat 64 (1024 * j) = coeffAddr (p + BitVec.ofNat 64 (1024 * j)) k := by + simp only [coeffAddr]; rw [VG.Offset.add_add, VG.Offset.add_add, Nat.add_comm] + +/-- A family of `c` polynomials. -/ +abbrev famR (p : Addr) (c : Nat) : Region := ⟨p, 1024 * c⟩ + +/-- `vg_mldsa_dot_inverse{c}(out = rdi, a = rsi, b = rdx, scratch = rcx)`. -/ +def dotK (c : Nat) : Contract isa where + pre s := + s.rd = [famR (s.gpr .rsi) c, famR (s.gpr .rdx) c] ∧ s.wr = [pR (s.gpr .rdi), pR (s.gpr .rcx)] ∧ + (pR (s.gpr .rdi)).Disjoint (famR (s.gpr .rsi) c) ∧ (pR (s.gpr .rdi)).Disjoint (famR (s.gpr .rdx) c) ∧ + (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rcx)) ∧ (pR (s.gpr .rcx)).Disjoint (famR (s.gpr .rsi) c) ∧ + (pR (s.gpr .rcx)).Disjoint (famR (s.gpr .rdx) c) ∧ (retR s).Disjoint (pR (s.gpr .rdi)) ∧ + (retR s).Disjoint (pR (s.gpr .rcx)) ∧ + (∀ j < c, Spec.MlDsa.PositiveReduced s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) ∧ + (∀ j < c, Spec.MlDsa.PositiveReduced s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) + post s s' := PolyIs s'.mem (s.gpr .rdi) (Spec.MlDsa.nttInv (Spec.MlDsa.dotNTT + (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) + pub s₁ s₂ := s₁.gpr .rdi = s₂.gpr .rdi ∧ s₁.gpr .rsi = s₂.gpr .rsi ∧ s₁.gpr .rdx = s₂.gpr .rdx ∧ + s₁.gpr .rcx = s₂.gpr .rcx ∧ s₁.gpr .rsp = s₂.gpr .rsp + +/-- A family's polynomial `j`, within it. -/ +theorem fam_sub (p : Addr) {c j : Nat} (hj : j < c) : Region.Sub (pR (p + BitVec.ofNat 64 (1024 * j))) (famR p c) := + VG.Offset.sub_base p (by rw [show 1024 * c = 1024 * (c - 1) + 1024 by omega]; omega) + +/-- The pointers and `rsp` are public. -/ +def dotτ : X86_64.Taint.T := X86_64.Taint.ofRegs [.rdi, .rsi, .rdx, .rcx, .rsp] + +theorem dot_agree (c : Nat) (s₁ s₂ : State) (_ : (dotK c).pre s₁) (_ : (dotK c).pre s₂) + (hp : (dotK c).pub s₁ s₂) : X86_64.Taint.Agree dotτ s₁ s₂ := + X86_64.Taint.agree_ofRegs fun r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl | rfl + exacts [hp.1, hp.2.1, hp.2.2.1, hp.2.2.2.1, hp.2.2.2.2] + +/-- A state satisfying the precondition. -/ +def dotSat (c : Nat) : State where + gpr r := match r with + | .rdi => 0x1000 | .rsi => 0x2000 | .rdx => 0x4000 | .rcx => 0x6000 | .rsp => 0x8000 | _ => 0 + cf := none + zf := none + sf := none + of := none + mem _ := 0 + rd := [⟨0x2000, 1024 * c⟩, ⟨0x4000, 1024 * c⟩] + wr := [⟨0x1000, 1024⟩, ⟨0x6000, 1024⟩] + +/-- The implication from `dotInverseContract c`, for a literal `c`. -/ +tactic_template "dot_implies " w:term : tactic => `(tactic| exact + { pre := by sig_implies_pre [Spec.MlDsa.dotInverseContract, Spec.MlDsa.dotInverseSig, dotK, X86_64.abi, X86_64.argRegs] + post := by sig_implies_post [Spec.MlDsa.dotInverseContract, Spec.MlDsa.dotInverseSig, dotK, X86_64.abi, X86_64.argRegs] + pub := by sig_implies_pub [Spec.MlDsa.dotInverseContract, Spec.MlDsa.dotInverseSig, dotK, X86_64.abi, X86_64.argRegs] + sat := by + refine ⟨$w, ?_⟩ + sig_pre [Spec.MlDsa.dotInverseContract, Spec.MlDsa.dotInverseSig, dotK, X86_64.abi, X86_64.argRegs] + and_intros + all_goals first + | rfl + | decide + | exact Region.disjoint_of_sep (by decide) + | (intro _ _ i hi; exact Nat.lt_trans (reduced_zero _ i hi) (by decide)) }) + +end VG.Proof.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean index 7089027633..86d209ae7a 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/NttInv.lean @@ -194,23 +194,48 @@ theorem vscale_ok {fP sP : Addr} (_hd : (pR sP).Disjoint (pR fP)) {G : Poly} (s rw [n_eq] at hk rw [hS' k hk, ifp (by omega), map_mul_get _ _ (by rw [n_eq]; exact hk)] +/-- Everything but `withMxcsr`, on any reduced `F` at `fP`, with the scratch at `sP`. -/ +theorem invBody_ok {fP sP : Addr} {F : Poly} {s : State} (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hF : PolyIs s.mem fP F) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) (hd : (pR fP).Disjoint (pR sP)) : + WP isa nttInvBody s fun s' => PolyIs s'.mem fP (nttInv F) ∧ Frame [pR fP, pR sP] s.mem s'.mem := by + have hd' : (pR sP).Disjoint (pR fP) := hd.symm + refine WP.seq (WP.mono (vpro_ok hsi hw) fun s2 ⟨hT, hc, hf2, k2, _⟩ => ?_) + have hF2 : PolyIs s2.mem fP F := polyIs_frame hf2 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) hF + have hdi : s2.gpr .rdi = fP := by rw [k2.gpr (by decide), hdi] + have hsi : s2.gpr .rsi = sP := by rw [k2.gpr (by decide), hsi] + have hwf : pR fP ∈ s2.wr := by rw [k2.2.2]; exact hwf + have hw : pR sP ∈ s2.wr := by rw [k2.2.2]; exact hw + refine WP.mono (Q := LI fP sP s2 ((nttInvLens.foldl nttInvLayer F).map (· * 8347681))) ?_ + fun s3 hI => ⟨by rw [nttInv_eq_layers]; exact hI.P, ?_⟩ + rotate_left + · refine (frame_fs hf2 ?_).trans (frame_fs hI.frame ?_) <;> + intro r hr <;> simp only [List.mem_singleton] at hr <;> subst hr + exacts [.inr fun _ h => h, .inl fun _ h => h] + simp only [nttInvLens, List.foldl_cons, List.foldl_nil] + have hI : LI fP sP s2 F s2 := ⟨hF2, hT, hc, Keep.refl _ _, Frame.refl _ _⟩ + refine LI.seq hdi hsi hwf hw hd' (invLay1_ok hd') ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay2_ok hd') ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 4 63 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 8 31 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 16 15 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 32 7 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 64 3 (by decide) (by decide)) ?_ hI + refine fun _ hI => LI.seq hdi hsi hwf hw hd' (invLay_ok hd' 128 1 (by decide) (by decide)) ?_ hI + exact fun _ hI => LI.last hdi hsi hwf hw hd' (vscale_ok hd') hI + theorem nttInv_correct (s : State) (hs : (inPlaceK nttInv).pre s) : ∃ t s', Exec isa Impl.MlDsa.X86_64.Arith.nttInv s t s' ∧ abiPreserved s s' ∧ (inPlaceK nttInv).post s s' := by - have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm - refine mx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => - WP.mono (nttBody_ok hs k1 f1 - (G := (nttInvLens.foldl nttInvLayer (polyAt s.mem (s.gpr .rdi))).map (· * 8347681)) - fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [nttInv_eq_layers]; exact hP, hf⟩ - simp only [nttInvLens, List.foldl_cons, List.foldl_nil] - refine LI.seq hdi hsi hwf hw hd (invLay1_ok hd) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay2_ok hd) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 4 63 (by decide) (by decide)) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 8 31 (by decide) (by decide)) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 16 15 (by decide) (by decide)) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 32 7 (by decide) (by decide)) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 64 3 (by decide) (by decide)) ?_ hI - refine fun _ hI => LI.seq hdi hsi hwf hw hd (invLay_ok hd 128 1 (by decide) (by decide)) ?_ hI - exact fun _ hI => LI.last hdi hsi hwf hw hd (vscale_ok hd) hI + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hwf : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + refine mx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => ?_ + have hF1 : PolyIs s1.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame f1 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) + ⟨hs.2.2.2.2.2, rfl⟩ + refine WP.mono (invBody_ok (k1.gpr (by decide)) (k1.gpr (by decide)) hF1 (by rw [k1.2.2]; exact hwf) + (by rw [k1.2.2]; exact hw) hd) fun s' ⟨hP, hf⟩ => ⟨hP, ?_⟩ + exact (frame_fs f1 fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _)).trans hf theorem nttInv_ct : ConstantTime isa (inPlaceK nttInv).pre (inPlaceK nttInv).pub Impl.MlDsa.X86_64.Arith.nttInv := VG.Taint.constantTime (A := taint) (X86_64.Taint.ofRegs [.rdi, .rsi, .rsp]) inPlace_agree (by taint_decide) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YDot.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YDot.lean new file mode 100644 index 0000000000..3198114b38 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YDot.lean @@ -0,0 +1,369 @@ +module + +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.DotCore +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMontgomeryInv +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase +public import VerifiedGarbage.Proof.MlDsa.Arith.Representation +public import VerifiedGarbage.Proof.Framework.TacticTemplate +public meta import VerifiedGarbage.Proof.Framework.TacticTemplate +public import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr +import VerifiedGarbage.Proof.Framework.X86_64.Avx +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay +import VerifiedGarbage.Proof.MlKem.X86_64.WritesOnly +import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr +import VerifiedGarbage.Proof.Framework.Offset +import VerifiedGarbage.Proof.MlKem.X86_64.YNttLay +import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.DotCore +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMontgomeryInv +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YBase +meta import VerifiedGarbage.Proof.MlDsa.Arith.Representation +meta import VerifiedGarbage.Proof.Framework.X86_64.Mxcsr +meta import VerifiedGarbage.Proof.Framework.X86_64.Avx +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.VLay +meta import VerifiedGarbage.Proof.MlKem.X86_64.WritesOnly +meta import VerifiedGarbage.Proof.MlKem.X86_64.VMxcsr +meta import VerifiedGarbage.Proof.Framework.Offset +meta import VerifiedGarbage.Proof.MlKem.X86_64.YNttLay +meta import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Ntt +public import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Montgomery +public import VerifiedGarbage.Proof.MlDsa.Arith.Mem +public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Basic +public import VerifiedGarbage.Proof.MlKem.X86_64.YLanes +public import VerifiedGarbage.Spec.MlDsa.Montgomery + +/-! +# ML-DSA on x86-64: `vg_mldsa_dot_inverse{c}_avx2` + +Each of the 32 iterations of the loop sums, in each lane, the `c` products +of eight coefficients of `a[j]` and `b[j]` in quadwords (`dotMac`, `macs_ok`) +and reduces them (`dotRed`, `dot_lane`), storing the sums times `2⁻³²` +(`step`); then the inverse NTT with Montgomery scaling (`body_ok`) cancels +the factor (`Representation.inverse_encode`). +-/ + +@[expose] public section + +namespace VG.Proof.MlDsa.X86_64.Arith.YDot + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Arith +open VG.Proof.MlDsa.Arith +open VG.Proof.MlDsa.X86_64.Arith +open VG.Proof.MlKem.X86_64 (writesIn writesOnly_of Keep XOnly YOnly ylanes yld_ok yconst_ok WP.keep writesOnly + gprPreserved_of ifp ifn GOnly wp_rcxLoopY add_ofNat_zero lane_setReg lane_setFlags sx32 State.setMem_ymm + withMxcsr_ok mxR) +open VG.Impl.MlKem.X86_64 (xb xmov toY yconst rcxLoop) +open VG.Spec.MlDsa (q n Poly Zq coeffAt polyAt Reduced PolyIs montgomeryRInv) + +theorem lane_dotZero : laneSseBlock (toY dotZero) = some dotZero := by decide +kernel +theorem lane_dotMac : laneSseBlock (toY dotMac) = some dotMac := by decide +kernel +theorem lane_dotRed : laneSseBlock (toY dotRed) = some dotRed := by decide +kernel + +/-- The sums of the first `n` products of the eight coefficients `Av j t`, +`Bv j t` (`t < 8`) in the quadwords of the lanes of `ymm5` (even `t`) and +`ymm6` (odd). -/ +def AccOk (s : State) (Av Bv : Nat → Nat → Nat) (n : Nat) : Prop := + ∀ l < 2, ∀ k < 2, + (qword (s.lane .xmm5 l) k).toNat = dsum (fun j => Av j (4 * l + 2 * k)) (fun j => Bv j (4 * l + 2 * k)) n ∧ + (qword (s.lane .xmm6 l) k).toNat = dsum (fun j => Av j (4 * l + 2 * k + 1)) (fun j => Bv j (4 * l + 2 * k + 1)) n + +/-- The first `n` products. -/ +theorem macs_ok {c : Nat} (hc : c ≤ 7) {Av Bv : Nat → Nat → Nat} (hA : ∀ j < c, ∀ t < 8, Av j t < 3 * q) + (hB : ∀ j < c, ∀ t < 8, Bv j t < 3 * q) {s : State} + (hr : ∀ j < c, InRegions (s.rd ++ s.wr) (s.gpr .r10 + BitVec.ofNat 64 (1024 * j)) 32) + (hr' : ∀ j < c, InRegions (s.rd ++ s.wr) (s.gpr .rdx + BitVec.ofNat 64 (1024 * j)) 32) + (ha : ∀ j < c, ∀ l < 2, ∀ e < 4, (dword (s.mem.readW (s.gpr .r10 + BitVec.ofNat 64 (1024 * j) + + BitVec.ofNat 64 (16 * l)) 128) e).toNat = Av j (4 * l + e)) + (hb : ∀ j < c, ∀ l < 2, ∀ e < 4, (dword (s.mem.readW (s.gpr .rdx + BitVec.ofNat 64 (1024 * j) + + BitVec.ofNat 64 (16 * l)) 128) e).toNat = Bv j (4 * l + e)) + (h0 : AccOk s Av Bv 0) : + ∀ n ≤ c, WP isa (.block ((List.range n).flatMap fun j => dotLoads j ++ toY dotMac)) s fun s' => + AccOk s' Av Bv n ∧ YOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm5, .xmm6] s s' + | 0, _ => by + simp only [List.range_zero, List.flatMap_nil] + vrund + exact ⟨h0, YOnly.refl _ s⟩ + | n + 1, hn => by + rw [List.range_succ, List.flatMap_append, List.flatMap_cons, List.flatMap_nil, List.append_nil, + WP.block_append_iff] + refine WP.mono (macs_ok hc hA hB hr hr' ha hb h0 n (by omega)) fun s1 ⟨A1, o1⟩ => ?_ + rw [dotLoads, show ∀ (x y : Instr) (l : List Instr), [x, y] ++ l = [x] ++ ([y] ++ l) from fun _ _ _ => rfl, + WP.block_append_iff] + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr]; exact hr n (by omega))) fun s2 ⟨L2, o2⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yld_ok (by rw [o2.rd, o2.wr, o2.gpr, o1.rd, o1.wr, o1.gpr]; exact hr' n (by omega))) + fun s3 ⟨L3, o3⟩ => ?_ + refine WP.mono (ylanes lane_dotMac (P := fun l t => + t.xmm .xmm5 = macE ((s3.proj l).xmm .xmm5) ((s3.proj l).xmm .xmm0) ((s3.proj l).xmm .xmm1) ∧ + t.xmm .xmm6 = macO ((s3.proj l).xmm .xmm6) ((s3.proj l).xmm .xmm0) ((s3.proj l).xmm .xmm1)) + fun l _ => WP.mono (dotMac_ok _) fun t ⟨h5, h6, ht⟩ => ⟨⟨h5, h6⟩, ht⟩) fun s4 ⟨M4, o4⟩ => ⟨?_, ?_⟩ + · intro l hl k hk + have x0 : (s3.proj l).xmm .xmm0 = s.mem.readW (s.gpr .r10 + BitVec.ofNat 64 (1024 * n) + + BitVec.ofNat 64 (16 * l)) 128 := by + rw [State.proj_xmm, o3.lane _ (by decide) l hl, L2 l hl, o1.mem, o1.gpr] + have x1 : (s3.proj l).xmm .xmm1 = s.mem.readW (s.gpr .rdx + BitVec.ofNat 64 (1024 * n) + + BitVec.ofNat 64 (16 * l)) 128 := by + rw [State.proj_xmm, L3 l hl, o2.mem, o2.gpr, o1.mem, o1.gpr] + have x5 : (s3.proj l).xmm .xmm5 = s1.lane .xmm5 l := by + rw [State.proj_xmm, o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl] + have x6 : (s3.proj l).xmm .xmm6 = s1.lane .xmm6 l := by + rw [State.proj_xmm, o3.lane _ (by decide) l hl, o2.lane _ (by decide) l hl] + have e5 : s4.lane .xmm5 l = _ := (M4 l hl).1 + have e6 : s4.lane .xmm6 l = _ := (M4 l hl).2 + have a0 := ha n (by omega) l hl (2 * k) (by omega) + have a1 := ha n (by omega) l hl (2 * k + 1) (by omega) + have b0 := hb n (by omega) l hl (2 * k) (by omega) + have b1 := hb n (by omega) l hl (2 * k + 1) (by omega) + rw [← x0] at a0 a1 + rw [← x1] at b0 b1 + rw [show 4 * l + (2 * k + 1) = 4 * l + 2 * k + 1 by omega] at a1 b1 + refine ⟨?_, ?_⟩ + · rw [e5, qword_macE hk (by rw [x5, (A1 l hl k hk).1, a0, b0]; exact prod_bound hc hA hB (by omega) (by omega)), x5, + (A1 l hl k hk).1, a0, b0] + rfl + · rw [e6, qword_macO hk (by rw [x6, (A1 l hl k hk).2, a1, b1]; exact prod_bound hc hA hB (by omega) (by omega)), x6, + (A1 l hl k hk).2, a1, b1] + rfl + · exact (((o1.trans o2).trans o3).trans o4).mono (by simp) + +/-- Coefficient `k` of the result: the sum of the products, times `2⁻³²`. -/ +def dotOut (m : VG.Mem) (a b : Addr) (c k : Nat) : Zq := + Fin.ofNat q (dsum (fun j => famV m a j k) (fun j => famV m b j k) c) * montgomeryRInv + +/-- After `i` iterations of the loop, from `s₀`. -/ +structure DI (o a b : Addr) (c : Nat) (s₀ : State) (i : Nat) (s : State) : Prop where + r8 : s.gpr .r8 = coeffAddr o (8 * i) + r10 : s.gpr .r10 = coeffAddr a (8 * i) + rdx : s.gpr .rdx = coeffAddr b (8 * i) + keep : Keep [.r8, .r10, .rdx, .rcx, .rax] s₀ s + yc : YConsts s + frame : Frame [pR o] s₀.mem s.mem + done : ∀ k < 8 * i, (coeffAt s.mem o k).toNat = (dotOut s₀.mem a b c k).val + +section +variable {o a b : Addr} {c : Nat} {s₀ : State} (hc7 : c ≤ 7) (hwo : pR o ∈ s₀.wr) + (hra : ∀ j < c, ∀ k, k + 8 ≤ 256 → InRegions (s₀.rd ++ s₀.wr) (coeffAddr (a + BitVec.ofNat 64 (1024 * j)) k) 32) + (hrb : ∀ j < c, ∀ k, k + 8 ≤ 256 → InRegions (s₀.rd ++ s₀.wr) (coeffAddr (b + BitVec.ofNat 64 (1024 * j)) k) 32) + (hda : ∀ j < c, (pR (a + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) + (hdb : ∀ j < c, (pR (b + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) + (hA : ∀ j < c, ∀ k < 256, famV s₀.mem a j k < 3 * q) (hB : ∀ j < c, ∀ k < 256, famV s₀.mem b j k < 3 * q) +include hc7 hwo hra hrb hda hdb hA hB + +theorem step {i : Nat} (hi : i < 32) {s : State} (hI : DI o a b c s₀ i s) : + WP isa (.block (dotBody c ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + DI o a b c s₀ (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 8 * i + 8 ≤ 256 := by omega + -- the values of the coefficients loaded + have hv : ∀ (p : Addr), (∀ j < c, (pR (p + BitVec.ofNat 64 (1024 * j))).Disjoint (pR o)) → ∀ j < c, ∀ l < 2, + ∀ e < 4, (dword (s.mem.readW (coeffAddr p (8 * i) + BitVec.ofNat 64 (1024 * j) + BitVec.ofNat 64 (16 * l)) 128) + e).toNat = famV s₀.mem p j (8 * i + (4 * l + e)) := fun p hd j hj l hl e he => by + rw [addr_fam, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, famV, ← Nat.add_assoc, + coeffAt_frame hI.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd j hj) (by rw [n_eq]; omega)] + simp only [dotBody, List.append_assoc] + rw [WP.block_append_iff] + refine WP.mono (ylanes lane_dotZero (P := fun l t => t.xmm .xmm5 = 0 ∧ t.xmm .xmm6 = 0) + fun l _ => WP.mono (dotZero_ok _) fun t ⟨h5, h6, ht⟩ => ⟨⟨h5, h6⟩, ht⟩) fun s1 ⟨Z1, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (macs_ok hc7 (Av := fun j t => famV s₀.mem a j (8 * i + t)) (Bv := fun j t => famV s₀.mem b j (8 * i + t)) + (fun j hj t ht => hA j hj _ (by omega)) (fun j hj t ht => hB j hj _ (by omega)) + (fun j hj => by + rw [o1.rd, o1.wr, o1.gpr, hI.r10, hI.keep.2.1, hI.keep.2.2, addr_fam]; exact hra j hj _ j0) + (fun j hj => by + rw [o1.rd, o1.wr, o1.gpr, hI.rdx, hI.keep.2.1, hI.keep.2.2, addr_fam]; exact hrb j hj _ j0) + (fun j hj l hl e he => by rw [o1.mem, o1.gpr, hI.r10]; exact hv a hda j hj l hl e he) + (fun j hj l hl e he => by rw [o1.mem, o1.gpr, hI.rdx]; exact hv b hdb j hj l hl e he) + (fun l hl k hk => by + have e5 : s1.lane .xmm5 l = _ := (Z1 l hl).1 + have e6 : s1.lane .xmm6 l = _ := (Z1 l hl).2 + rw [e5, e6]; exact ⟨by simp [qword, dsum], by simp [qword, dsum]⟩) c (Nat.le_refl _)) fun s2 ⟨A2, o2⟩ => ?_ + have o12 := o1.trans o2 + rw [WP.block_append_iff] + refine WP.mono (ylanes lane_dotRed (P := fun l t => t.xmm .xmm5 = csubV (redV ((s2.proj l).xmm .xmm5) ((s2.proj l).xmm .xmm6))) + fun l hl => WP.mono (dotRed_ok (yonly_yconsts o12 hI.yc (by decide) (by decide) l hl)) fun t ⟨h5, ht⟩ => ⟨h5, ht⟩) + fun s3 ⟨R3, o3⟩ => ?_ + have o13 := o12.trans o3 + -- the eight results + have hres : ∀ l < 2, ∀ e < 4, (dword (s3.lane .xmm5 l) e).toNat = (dotOut s₀.mem a b c (8 * i + 4 * l + e)).val := by + intro l hl e he + have e3 : s3.lane .xmm5 l = _ := R3 l hl + rw [e3, dot_lane (e := (s2.proj l).xmm .xmm5) (o := (s2.proj l).xmm .xmm6) + (A := fun j t => famV s₀.mem a j (8 * i + (4 * l + t))) + (B := fun j t => famV s₀.mem b j (8 * i + (4 * l + t))) hc7 + (fun j hj t ht => hA j hj _ (by omega)) (fun j hj t ht => hB j hj _ (by omega)) + (fun k hk => by rw [State.proj_xmm]; exact (A2 l hl k hk).1) + (fun k hk => by rw [State.proj_xmm, (A2 l hl k hk).2]; simp only [Nat.add_assoc]) he] + simp only [dotOut, Nat.add_assoc] + have g3 : s3.gpr .r8 = coeffAddr o (8 * i) := by rw [o13.gpr, hI.r8] + have w0 : InRegions s3.wr (s3.gpr .r8) 32 := by + rw [o13.wr, g3, hI.keep.2.2]; exact f_in32 hwo j0 + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, w0, sx32] + rw [g3, o13.mem] + have p8 : coeffAddr o (8 * i) + 32 = coeffAddr o (8 * (i + 1)) := by + rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, show 8 * i + 8 = 8 * (i + 1) by omega] + have pa : ∀ p : Addr, coeffAddr p (8 * i) + 32 = coeffAddr p (8 * (i + 1)) := fun p => by + rw [show (32 : BitVec 64) = BitVec.ofNat 64 (4 * 8) from rfl, coeffAddr_add, show 8 * i + 8 = 8 * (i + 1) by omega] + refine ⟨⟨?_, ?_, ?_, ⟨fun r hr => ?_, ?_, ?_⟩, ?_, ?_, fun k hk => ?_⟩, by rw [o13.gpr], by rw [o13.gpr]⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + exact p8 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o13.gpr, hI.r10]; exact pa a + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq, State.setMem_gpr] + rw [o13.gpr, hI.rdx]; exact pa b + · have hr0 := hr + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, hr.1, hr.2.1, hr.2.2.1, hr.2.2.2.1, ite_false, + State.setMem_gpr] + rw [o13.gpr, hI.keep.1 r hr0] + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags, State.setMem_rd]; rw [o13.rd, hI.keep.2.1] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags, State.setMem_wr]; rw [o13.wr, hI.keep.2.2] + · exact ylanes_gpr (s := s3) (fun r l => by simp only [lane_setReg, lane_setFlags, State.setMem_lane]) o13 hI.yc + (by decide) (by decide) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags, State.setMem_mem] + rw [coeffAt_write256 _ _ j0 _ (by omega)] + split + · rw [State.ymm, extract_ymm _ _ (by omega)] + split + · have := hres 0 (by decide) (k - 8 * i) (by omega) + rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this; exact this + · have := hres 1 (by decide) (k - 8 * i - 4) (by omega) + rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this; exact this + · exact hI.done k (by omega) + +/-- The loop, from the constants and the pointers, in `s₀`. -/ +theorem pass_ok (h8 : s₀.gpr .r8 = o) (h10 : s₀.gpr .r10 = a) (hdx : s₀.gpr .rdx = b) (hc : YConsts s₀) : + WP isa (rcxLoop 32 (dotBody c)) s₀ fun s' => PolyIs s'.mem o (Vector.ofFn fun k => dotOut s₀.mem a b c k.val) ∧ + Keep [.r8, .r10, .rdx, .rcx, .rax] s₀ s' ∧ Frame [pR o] s₀.mem s'.mem := by + refine WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide) (DI o a b c s₀) + (fun u og hy _ => ⟨by rw [og.keep.gpr (by decide), h8, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [og.keep.gpr (by decide), h10, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + by rw [og.keep.gpr (by decide), hdx, Nat.mul_zero, coeffAddr, Nat.mul_zero, add_ofNat_zero], + og.keep.mono (by simp), ylanes_gpr (s := s₀) (og.lane hy) (YOnly.refl [] s₀) hc (by decide) (by decide), + by rw [og.mem]; exact Frame.refl _ _, fun k hk => absurd hk (by omega)⟩) + (fun i hi u hI => step hc7 hwo hra hrb hda hdb hA hB hi hI)) fun u hI => ⟨⟨fun k hk => ?_, ?_⟩, hI.keep, hI.frame⟩ + · rw [n_eq] at hk + rw [hI.done k (by omega)]; exact (dotOut s₀.mem a b c k).isLt + · apply Vector.ext; intro k hk + rw [polyAt, Vector.getElem_ofFn, Vector.getElem_ofFn] + apply Fin.ext + rw [Fin.val_ofNat, hI.done k (by rw [n_eq] at hk; omega), Nat.mod_eq_of_lt (dotOut s₀.mem a b c k).isLt] + +end + +/-! ## The function -/ + +/-- The result: the sums times `2⁻³²`, which the inverse transform's factor cancels. -/ +theorem dotOut_eq {m m' : VG.Mem} {a b : Addr} {c : Nat} + (ha : ∀ j < c, ∀ k < 256, coeffAt m' (a + BitVec.ofNat 64 (1024 * j)) k = coeffAt m (a + BitVec.ofNat 64 (1024 * j)) k) + (hb : ∀ j < c, ∀ k < 256, coeffAt m' (b + BitVec.ofNat 64 (1024 * j)) k = coeffAt m (b + BitVec.ofNat 64 (1024 * j)) k) : + (Vector.ofFn fun k => dotOut m' a b c k.val) = Representation.encode true (Spec.MlDsa.dotNTT + (fun j => polyAt m (a + BitVec.ofNat 64 (1024 * j))) (fun j => polyAt m (b + BitVec.ofNat 64 (1024 * j))) c) := by + apply Vector.ext; intro k hk + rw [Vector.getElem_ofFn, ← getElem!_pos (Representation.encode true _) k hk] + change _ = (Montgomery.scale montgomeryRInv _)[k]! + rw [Montgomery.scale_get, dotNTT_get (A := fun j => famV m' a j k) (B := fun j => famV m' b j k) hk + (fun j hj => by rw [polyAt_get _ _ hk, famV, ha j hj k hk]) (fun j hj => by rw [polyAt_get _ _ hk, famV, hb j hj k hk])] + rfl + +theorem correct {c : Nat} (hc7 : c ≤ 7) + (hk : Code.allInstrs (writesIn [.rax, .rcx, .rdx, .r8, .r9, .r10, .r11, .rsi]) (dotInvAvx2 c) = true) + (hk' : Code.allInstrs (writesIn [.rax, .rcx, .rdx, .r8, .r9, .r10]) + (.seq (.block (yconsts ++ ([.mov .r8 (.reg .rdi)] : List Instr))) (.seq (rcxLoop 32 (dotBody c)) montInvBody)) = true) + (hctl : ctlOk (dotInvAvx2 c) = true) (s : State) (hs : (dotK c).pre s) : + ∃ t s', Exec isa (dotInvAvx2 c) s t s' ∧ abiPreserved s s' ∧ (dotK c).post s s' := by + obtain ⟨hrd, hwr, d1, d2, d3, d4, d5, r1, r2, pa, pb⟩ := hs + have hwo : pR (s.gpr .rdi) ∈ s.wr := by rw [hwr]; simp + have hws : pR (s.gpr .rcx) ∈ s.wr := by rw [hwr]; simp + have hW : WP isa (dotInvAvx2 c) s fun s' => PolyIs s'.mem (s.gpr .rdi) (Spec.MlDsa.nttInv (Spec.MlDsa.dotNTT + (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem s'.mem := by + refine WP.seq (WP.mono (Q := fun (w : State) => w.gpr .r10 = s.gpr .rsi ∧ w.gpr .rsi = s.gpr .rcx ∧ + GOnly [.r10, .rsi] s w ∧ w.ymmHi = s.ymmHi) + (by vrund; exact ⟨by gonlyd, rfl⟩) fun w ⟨h10, hsi, og, hy⟩ => ?_) + refine WP.mono (withMxcsr_ok (r := .rsi) (Q := fun u => PolyIs u.mem (s.gpr .rdi) (Spec.MlDsa.nttInv + (Spec.MlDsa.dotNTT (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) ∧ + Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem u.mem) ⟨by decide, by decide⟩ [.rax, .rcx, .rdx, .r8, .r9, .r10] + ⟨by decide, by decide⟩ hsi (by rw [og.keep.2.2]; exact hws) (writesOnly_of hk') (fun s1 k1 f1 => ?_)) + fun s' ⟨s2, ⟨hP, hf⟩, hf', _⟩ => ⟨?_, ?_⟩ + rotate_left + · exact polyIs_frame hf' (fun r hr => by + rw [List.mem_singleton.mp hr]; exact d3.sub_right (mx_sub' _)) hP + · exact hf.trans (frame_fs hf' fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _)) + have hdi1 : s1.gpr .rdi = s.gpr .rdi := by rw [k1.gpr (by decide), og.keep.gpr (by decide)] + have hsi1 : s1.gpr .rsi = s.gpr .rcx := by rw [k1.gpr (by decide), hsi] + have h101 : s1.gpr .r10 = s.gpr .rsi := by rw [k1.gpr (by decide), h10] + have hdx1 : s1.gpr .rdx = s.gpr .rdx := by rw [k1.gpr (by decide), og.keep.gpr (by decide)] + have wr1 : s1.wr = s.wr := by rw [k1.2.2, og.keep.2.2] + have rd1 : s1.rd = s.rd := by rw [k1.2.1, og.keep.2.1] + have hm : ∀ p : Addr, (pR (s.gpr .rcx)).Disjoint (famR p c) → ∀ j < c, ∀ k < 256, + coeffAt s1.mem (p + BitVec.ofNat 64 (1024 * j)) k = coeffAt s.mem (p + BitVec.ofNat 64 (1024 * j)) k := + fun p hd j hj k hk => by + rw [coeffAt_frame f1 (fun r hr => by + rw [List.mem_singleton.mp hr]; exact (hd.sub_right (fam_sub p hj)).symm.sub_right (mx_sub' _)) + (by rw [n_eq]; exact hk), og.mem] + refine WP.seq ?_ + rw [WP.block_append_iff] + refine WP.mono (yconsts_ok s1) fun s2 ⟨c2, k2, m2, _, _⟩ => ?_ + have hdi2 : s2.gpr .rdi = s.gpr .rdi := by rw [k2.gpr (by decide), hdi1] + refine WP.mono (Q := fun (s3 : State) => s3.gpr .r8 = s.gpr .rdi ∧ GOnly [.r8] s2 s3 ∧ s3.ymmHi = s2.ymmHi) + (by vrund [hdi2]; exact ⟨by gonlyd, rfl⟩) fun s3 ⟨h8, og3, hy3⟩ => ?_ + have c3 : YConsts s3 := ylanes_gpr (s := s2) (GOnly.lane og3 hy3) (YOnly.refl [] s2) c2 (by decide) (by decide) + have k13 : Keep [.rax, .r8] s1 s3 := (k2.trans og3.keep).mono (by simp) + have m3 : s3.mem = s1.mem := by rw [og3.mem, m2] + have hr : ∀ p : Addr, famR p c ∈ s.rd → ∀ j < c, ∀ k, k + 8 ≤ 256 → + InRegions (s3.rd ++ s3.wr) (coeffAddr (p + BitVec.ofNat 64 (1024 * j)) k) 32 := fun p hp j hj k hk => + ⟨famR p c, by rw [k13.2.1, rd1]; exact List.mem_append_left _ hp, by + rw [coeffAddr, VG.Offset.add_add]; exact VG.Offset.contains_base _ (by omega) (by omega)⟩ + refine WP.seq (WP.mono (pass_ok (s₀ := s3) hc7 (by rw [k13.2.2, wr1]; exact hwo) + (hr _ (by rw [hrd]; simp)) (hr _ (by rw [hrd]; simp)) + (fun j hj => (d1.sub_right (fam_sub _ hj)).symm) (fun j hj => (d2.sub_right (fam_sub _ hj)).symm) + (fun j hj k hk => by rw [famV, m3, hm _ d4 j hj k hk]; exact pa j hj k (by rw [n_eq]; exact hk)) + (fun j hj k hk => by rw [famV, m3, hm _ d5 j hj k hk]; exact pb j hj k (by rw [n_eq]; exact hk)) + h8 (by rw [k13.gpr (by decide), h101]) (by rw [k13.gpr (by decide), hdx1]) c3) fun u ⟨hP, ku, fu⟩ => ?_) + refine WP.mono (MontgomeryInv.body_ok (by rw [ku.gpr (by decide), k13.gpr (by decide), hdi1]) + (by rw [ku.gpr (by decide), k13.gpr (by decide), hsi1]) hP + (by rw [ku.2.2, k13.2.2, wr1]; exact hwo) (by rw [ku.2.2, k13.2.2, wr1]; exact hws) d3) fun v ⟨hv, fv⟩ => ⟨?_, ?_⟩ + · have e : Spec.MlDsa.montgomeryNttInv (Representation.encode true (Spec.MlDsa.dotNTT + (fun j => polyAt s.mem (s.gpr .rsi + BitVec.ofNat 64 (1024 * j))) + (fun j => polyAt s.mem (s.gpr .rdx + BitVec.ofNat 64 (1024 * j))) c)) = _ := + Representation.inverse_encode true _ + rw [dotOut_eq (m := s.mem) (fun j hj k hk => by rw [m3, hm _ d4 j hj k hk]) + (fun j hj k hk => by rw [m3, hm _ d5 j hj k hk]), e] at hv + exact hv + · have f1' : Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s.mem s1.mem := by + rw [← og.mem]; exact frame_fs f1 fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _) + have fu' : Frame [pR (s.gpr .rdi), pR (s.gpr .rcx)] s1.mem u.mem := by + rw [← m3]; exact frame_fs fu fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inl fun _ h => h + exact (f1'.trans fu').trans fv + obtain ⟨tr, s', he, ⟨hP, hf⟩, hk2⟩ := WP.keep [.rax, .rcx, .rdx, .r8, .r9, .r10, .r11, .rsi] hW (writesOnly_of hk) + exact ⟨tr, s', he, abiPreserved_of_ctl hctl he (gprPreserved_of hk2 (by decide) hf (by simpa using ⟨r1, r2⟩)), hP⟩ + +theorem ct4 : ConstantTime isa (dotK 4).pre (dotK 4).pub (dotInvAvx2 4) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 4) (by taint_decide) + +theorem ct5 : ConstantTime isa (dotK 5).pre (dotK 5).pub (dotInvAvx2 5) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 5) (by taint_decide) + +theorem ct7 : ConstantTime isa (dotK 7).pre (dotK 7).pub (dotInvAvx2 7) := + VG.Taint.constantTime (A := taint) dotτ (dot_agree 7) (by taint_decide) + +theorem verified4 : Verified X86_64.target (dotInvAvx2 4) (Spec.MlDsa.dotInverseContract 4 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct4 + (by dot_implies dotSat 4) + +theorem verified5 : Verified X86_64.target (dotInvAvx2 5) (Spec.MlDsa.dotInverseContract 5 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct5 + (by dot_implies dotSat 5) + +theorem verified7 : Verified X86_64.target (dotInvAvx2 7) (Spec.MlDsa.dotInverseContract 7 X86_64.abi) := + Verified.of_correct (correct (by decide) (by decide +kernel) (by decide +kernel) (by decide +kernel)) ct7 + (by dot_implies dotSat 7) + +end VG.Proof.MlDsa.X86_64.Arith.YDot diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMontgomeryInv.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMontgomeryInv.lean index 2986d62891..ffc36384c1 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMontgomeryInv.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/YMontgomeryInv.lean @@ -653,33 +653,54 @@ theorem map_residue {b : Nat} {F : VG.Proof.MlDsa.Arith.Lazy.Poly} {g : Poly} (h apply Vector.ext; intro i hi rw [Vector.getElem_map, Vector.getElem_map, ← getElem!_pos F i hi, ← getElem!_pos g i hi, (h i hi).2] -theorem correct (s : State) (hs : (inPlaceK Spec.MlDsa.montgomeryNttInv).pre s) : - ∃ t s', Exec isa montNttInvAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK Spec.MlDsa.montgomeryNttInv).post s s' := by - have hd : (pR (s.gpr .rsi)).Disjoint (pR (s.gpr .rdi)) := hs.2.2.1.symm - refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => - WP.mono (ynttBody_ok hs k1 f1 - (G := (nttInvLens.foldl nttInvLayer (polyAt s.mem (s.gpr .rdi))).map (· * 16382)) - fun s2 hI hdi hsi hwf hw => ?_) fun s' ⟨hP, hf⟩ => ⟨by rw [inv_eq]; exact hP, hf⟩ +/-- Everything but `withMxcsr`, on any reduced `F` at `fP`, with the scratch at `sP`. -/ +theorem body_ok {fP sP : Addr} {F : Poly} {s : State} (hdi : s.gpr .rdi = fP) (hsi : s.gpr .rsi = sP) + (hF : PolyIs s.mem fP F) (hwf : pR fP ∈ s.wr) (hw : pR sP ∈ s.wr) (hd : (pR fP).Disjoint (pR sP)) : + WP isa montInvBody s fun s' => PolyIs s'.mem fP (Spec.MlDsa.montgomeryNttInv F) ∧ + Frame [pR fP, pR sP] s.mem s'.mem := by + have hd' : (pR sP).Disjoint (pR fP) := hd.symm + refine WP.seq (WP.mono (ypro_ok hsi hw) fun s2 ⟨hT, hc0, hf2, k2, _⟩ => ?_) + have hF2 : PolyIs s2.mem fP F := + polyIs_frame hf2 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd) hF + have hdi : s2.gpr .rdi = fP := by rw [k2.gpr (by decide), hdi] + have hsi : s2.gpr .rsi = sP := by rw [k2.gpr (by decide), hsi] + have hwf : pR fP ∈ s2.wr := by rw [k2.2.2]; exact hwf + have hw : pR sP ∈ s2.wr := by rw [k2.2.2]; exact hw refine WP.seq (WP.mono (yconst_ok .xmm11 1072693376 s2) fun w ⟨lc, kc, mc, _, oc⟩ => ?_) have hc : Lazy.YConstsW Lazy.bInvC w := fun l hl => ⟨⟨by - rw [State.proj_xmm, oc _ (by decide) l hl]; exact (hI.c l hl).q, - by rw [State.proj_xmm, oc _ (by decide) l hl]; exact (hI.c l hl).qinv⟩, + rw [State.proj_xmm, oc _ (by decide) l hl]; exact (hc0 l hl).q, + by rw [State.proj_xmm, oc _ (by decide) l hl]; exact (hc0 l hl).qinv⟩, by rw [State.proj_xmm, lc l hl]; rfl⟩ - have hi : Lazy.LII (s.gpr .rdi) (s.gpr .rsi) s2 (Lazy.lift (polyAt s.mem (s.gpr .rdi))) w := - ⟨by rw [mc]; exact Lazy.lift_mem hI.P, by rw [mc]; exact hI.T, hc, - kc.mono (by simp), by rw [mc]; exact Frame.refl _ _⟩ - refine Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay1_ok hd) ?_ hi - refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay2_ok hd) ?_ hI - refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay4_ok hd) ?_ hI - refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay_ok hd 8 31 (by decide) (by decide)) ?_ hI - refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay_ok hd 16 15 (by decide) (by decide)) ?_ hI - refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd (Lazy.yinvLay_ok hd 32 7 (by decide) (by decide)) ?_ hI + have hi : Lazy.LII fP sP s2 (Lazy.lift F) w := + ⟨by rw [mc]; exact Lazy.lift_mem hF2, by rw [mc]; exact hT, hc, kc.mono (by simp), by rw [mc]; exact Frame.refl _ _⟩ + refine Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay1_ok hd') ?_ hi + refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay2_ok hd') ?_ hI + refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay4_ok hd') ?_ hI + refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay_ok hd' 8 31 (by decide) (by decide)) ?_ hI + refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay_ok hd' 16 15 (by decide) (by decide)) ?_ hI + refine fun _ hI => Lazy.LII.seq hdi hsi hwf hw hd' (Lazy.yinvLay_ok hd' 32 7 (by decide) (by decide)) ?_ hI intro u hu - refine WP.seq (WP.mono (ylast_ok (fun k hk => (Lazy.transformInv6_rel (polyAt s.mem (s.gpr .rdi)) k hk).1) u + refine WP.seq (WP.mono (ylast_ok (fun k hk => (Lazy.transformInv6_rel F k hk).1) u hu.c (by rw [hu.keep.gpr (by decide), hdi]) hu.P (by rw [hu.keep.2.2]; exact hwf)) fun v ⟨hv, _, hb⟩ => ?_) refine WP.mono (Q := fun (u' : State) => u'.mem = v.mem) (by simp only [yepi]; vrund; rfl) fun u' hm => ?_ - rw [hm, ← map_residue (Lazy.transformInv_rel (polyAt s.mem (s.gpr .rdi)))] - exact ⟨hv, hu.frame.trans hb⟩ + rw [hm, inv_eq, ← map_residue (Lazy.transformInv_rel F)] + refine ⟨hv, (frame_fs hf2 ?_).trans (frame_fs (hu.frame.trans hb) ?_)⟩ <;> + intro r hr <;> simp only [List.mem_singleton] at hr <;> subst hr + exacts [.inr fun _ h => h, .inl fun _ h => h] + +theorem correct (s : State) (hs : (inPlaceK Spec.MlDsa.montgomeryNttInv).pre s) : + ∃ t s', Exec isa montNttInvAvx2 s t s' ∧ abiPreserved s s' ∧ (inPlaceK Spec.MlDsa.montgomeryNttInv).post s s' := by + have hw : pR (s.gpr .rsi) ∈ s.wr := by rw [hs.2.1]; simp + have hwf : pR (s.gpr .rdi) ∈ s.wr := by rw [hs.2.1]; simp + have hd : (pR (s.gpr .rdi)).Disjoint (pR (s.gpr .rsi)) := hs.2.2.1 + refine ymx_correct s hs (by decide +kernel) (by decide +kernel) (by decide +kernel) fun s1 k1 f1 => ?_ + have hF1 : PolyIs s1.mem (s.gpr .rdi) (polyAt s.mem (s.gpr .rdi)) := + polyIs_frame f1 (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.sub_right (mx_sub' _)) + ⟨hs.2.2.2.2.2, rfl⟩ + refine WP.mono (body_ok (k1.gpr (by decide)) (k1.gpr (by decide)) hF1 (by rw [k1.2.2]; exact hwf) + (by rw [k1.2.2]; exact hw) hd) fun s' ⟨hP, hf⟩ => ⟨hP, ?_⟩ + exact (frame_fs f1 fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact .inr (mx_sub' _)).trans hf theorem ct : ConstantTime isa (inPlaceK Spec.MlDsa.montgomeryNttInv).pre (inPlaceK Spec.MlDsa.montgomeryNttInv).pub montNttInvAvx2 := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean index 83d0221f1c..ffa45d8a09 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/KeyGen/Inst.lean @@ -11,6 +11,7 @@ public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend public import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Same public import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Base public import VerifiedGarbage.Proof.MlDsa.X86_64.KeyGen.Call +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Montgomery /-! # ML-DSA key generation on x86-64, with this library's primitives diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean index 3e2e005cfb..acb89df856 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Message/SignFn.lean @@ -73,13 +73,16 @@ theorem signFn (v : ArithImpl) {p : Params} (hp : p ∈ params) : have dep2 : ∀ {c : Prog isa}, c.depth ≤ 2 → decide (c.depth ≤ 3) = true := fun h => decide_eq_true (by omega) refine ⟨sign_correct (prims_okWith v) h3 (Proof.MlDsa.X86_64.Sign.sign_ctl v h3), sign_ct (prims_okWith v) h3, noSp_of (Same.ok (sign_same (Comp.all _) (noSpB_of v.ok.ntt.nosp) (noSpB_of v.ok.invNtt.nosp) - (noSpB_of v.ok.mul.nosp) (noSpB_of v.ok.mulAdd.nosp) (noSpB_of v.ok.add.nosp) (noSpB_of v.ok.sub.nosp) + (noSpB_of v.ok.mul.nosp) (noSpB_of v.ok.add.nosp) (noSpB_of v.ok.sub.nosp) (noSpB_of v.ok.rej4.nosp) (noSpB_of v.ok.expandMask4.nosp) (noSpB_of v.ok.highBits.nosp) - (noSpB_of v.ok.lowBits.nosp) (noSpB_of v.ok.normLt.nosp) (noSpB_of v.ok.makeHint.nosp) p) (sign0_noSp h3)), + (noSpB_of v.ok.lowBits.nosp) (noSpB_of v.ok.normLt.nosp) (noSpB_of v.ok.makeHint.nosp) + (noSpB_of (v.ok.dotInv 4 (.inl rfl)).nosp) (noSpB_of (v.ok.dotInv 5 (.inr (.inl rfl))).nosp) + (noSpB_of (v.ok.dotInv 7 (.inr (.inr rfl))).nosp) p) (sign0_noSp h3)), of_decide_eq_true (Same.ok (sign_same (depthCompN 3) (dep2 v.ok.ntt.depth) (dep2 v.ok.invNtt.depth) - (dep2 v.ok.mul.depth) (dep2 v.ok.mulAdd.depth) (dep2 v.ok.add.depth) (dep2 v.ok.sub.depth) + (dep2 v.ok.mul.depth) (dep2 v.ok.add.depth) (dep2 v.ok.sub.depth) (dep v.ok.rej4.depth) (dep v.ok.expandMask4.depth) (dep2 v.ok.highBits.depth) (dep2 v.ok.lowBits.depth) - (dep2 v.ok.normLt.depth) (dep2 v.ok.makeHint.depth) p) (sign0_depth h3))⟩ + (dep2 v.ok.normLt.depth) (dep2 v.ok.makeHint.depth) (dep2 (v.ok.dotInv 4 (.inl rfl)).depth) + (dep2 (v.ok.dotInv 5 (.inr (.inl rfl))).depth) (dep2 (v.ok.dotInv 7 (.inr (.inr rfl))).depth) p) (sign0_depth h3))⟩ theorem kabs_spSafe : Impl.Sha3.X86_64.Stream.absorb.all (fun i => !isa.writesSp i) = true := by lit_decide theorem kpad_spSafe : Impl.Sha3.X86_64.Stream.pad.all (fun i => !isa.writesSp i) = true := by lit_decide diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean index 5fca772771..6035f8ffef 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean @@ -49,7 +49,6 @@ def prims : Prims where ntt := Impl.MlDsa.X86_64.Arith.ntt invNtt := Impl.MlDsa.X86_64.Arith.nttInv mul := Impl.MlDsa.X86_64.Arith.mul - mulAdd := Impl.MlDsa.X86_64.Arith.mulAdd add := Impl.MlDsa.X86_64.Arith.add sub := Impl.MlDsa.X86_64.Arith.sub rejNTT := Impl.MlDsa.X86_64.Sample.rejNTT @@ -72,7 +71,6 @@ def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := ntt := B.ntt invNtt := B.invNtt mul := B.mul - mulAdd := B.mulAdd add := B.add sub := B.sub rej4 := B.rej4 @@ -82,7 +80,10 @@ def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := normLt := B.normLt makeHint := B.makeHint sfx := B.sfx - montgomery := B.montgomery } + montgomery := B.montgomery + dotInv4 := B.dotInv4 + dotInv5 := B.dotInv5 + dotInv7 := B.dotInv7 } theorem nosp_of {c : Prog isa} (h : c.allInstrs (fun i => !Taint.clobbers i .rsp) = true) : NoSp c := by rw [Code.allInstrs_eq] at h @@ -148,7 +149,6 @@ def prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) signStack where ntt := calleeOf v.ok.ntt invNtt := calleeOf v.ok.invNtt mul := calleeOf v.ok.mul - mulAdd := calleeOf v.ok.mulAdd add := calleeOf v.ok.add sub := calleeOf v.ok.sub rejNTT := (⟨16, by decide, Proof.MlDsa.X86_64.Sample.rejNTT_verified, nosp_of (by decide +kernel), @@ -191,6 +191,7 @@ def prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) signStack where have := v.ok.expandMask4.depth show 8 * (v.code.expandMask4.depth + 1) ≤ signStack unfold signStack; omega⟩ + dotInv := fun c hc => calleeOf (v.ok.dotInv c hc) rej4Ret := by exact (fun s₁ s₂ t₁ t₂ s₁' s₂' ⟨h₁, h₂, hp⟩ e₁ e₂ => ⟨v.ok.rej4.ver.2.1 s₁ s₂ t₁ t₂ s₁' s₂' h₁ h₂ hp e₁ e₂, show _ = _ by diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean index a5561534f5..156b35c3a7 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseC.lean @@ -2,6 +2,7 @@ module public import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.PhaseD public import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Blocks +import VerifiedGarbage.Proof.MlDsa.Arith.DotSum import VerifiedGarbage.Proof.Framework.WriteBytes import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.PrimsD import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.PhaseA4 @@ -501,68 +502,46 @@ theorem ICw.step {p : Params} {D : Nat} {σ s s' : State} {t i : Nat} (h : ICw p have I : ICm p D σ t p.ℓ s' := (ICm.mk h.l h.y h.yh).step hP hc.1 exact ⟨I.l, I.y, I.yh, Fam.keep h.l.st.lay hP hc.2 h.w⟩ -/-- `∑_{j < m} Â[i, j] ŷ[j]`, summed from `j = 0` with `AddNTT`. -/ -abbrev wAcc (p : Params) (σ : State) (κ i m : Nat) : Poly := - ((List.range m).map fun j => multiplyNTT (Am p σ i j) (YHv p σ κ j)).foldl add zero - -theorem add_zero_left (x : Poly) : add zero x = x := by - apply Vector.ext - intro j hj - simp [add, zero] - -theorem wAcc_one (p : Params) (σ : State) (κ i : Nat) : - wAcc p σ κ i 1 = multiplyNTT (Am p σ i 0) (YHv p σ κ 0) := by - simp only [wAcc, List.range_one, List.map_cons, List.map_nil, List.foldl_cons, List.foldl_nil, add_zero_left] - -theorem wAcc_succ (p : Params) (σ : State) (κ i m : Nat) : - wAcc p σ κ i (m + 1) = add (wAcc p σ κ i m) (multiplyNTT (Am p σ i m) (YHv p σ κ m)) := by - simp only [wAcc, List.range_succ, List.map_append, List.map_cons, List.map_nil, List.foldl_append, - List.foldl_cons, List.foldl_nil] - theorem aP_ij (p : Params) (i j : Nat) : aP p i j = pS (aBase p + (p.ℓ * i + j)) := by show pS (5 + 4 * p.k + 3 * p.ℓ + p.ℓ * i + j) = _ rw [Nat.add_assoc (5 + 4 * p.k + 3 * p.ℓ)] +theorem famP_aP (p : Params) (i j : Nat) : famP (aP p i 0) j = aP p i j := by + simp only [famP, aP, pS, sc, oP, Prod.mk.injEq, true_and]; omega + +theorem famP_yhP (p : Params) (j : Nat) : famP (yhP p 0) j = yhP p j := by + simp only [famP, yhP, pS, sc, oP, Prod.mk.injEq, true_and]; omega + /-- What `w[i]` needs of the layout. -/ def wChk (p : Params) (i : Nat) : Bool := let w := pS (wBase p + i) - (List.range p.ℓ).all (fun j => mulChk (sgB p) (sgW p) w (aP p i j) (yhP p j)) && ipChk (sgB p) (sgW p) w && - icwChk p [(w, 1024)] i && icwChk p [(w, 1024), (sc oPS, 1024)] i && decide (0 < p.ℓ) + dotChk (sgB p) (sgW p) w (aP p i 0) (yhP p 0) p.ℓ && icwChk p [(w, 1024), (sc oPS, 1024)] i && + decide (p.ℓ = 4 ∨ p.ℓ = 5 ∨ p.ℓ = 7) + +/-- `Â[i, j]`, reduced, for `j < ℓ`. -/ +theorem rowA {p : Params} {D : Nat} {σ : State} {t i : Nat} (hi : i < p.k) {s : State} (I : ICw p D σ t i s) + {j : Nat} (hj : j < p.ℓ) : PolyIs s.mem (pa s (aP p i j)) (Am p σ i j) := by + have := I.l.k.d.im.A (p.ℓ * i + j) (by + have := Nat.mul_le_mul_left p.ℓ (show i + 1 ≤ p.k from hi) + rw [Nat.mul_add, Nat.mul_one, Nat.mul_comm p.ℓ p.k] at this; omega) + rw [aVal_ij hj] at this + rw [aP_ij]; exact this theorem rowW_ok {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {σ : State} {t i : Nat} (hc : wChk p i = true) (hi : i < p.k) {s : State} (h : ICw p D σ t i s) : WP isa (rowW P p i) s (ICw p D σ t (i + 1)) := by - simp only [wChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hc - obtain ⟨⟨⟨⟨cm, ci⟩, c1⟩, c2⟩, hl⟩ := hc - have hA : ∀ {s : State} (I : ICw p D σ t i s) j, j < p.ℓ → PolyIs s.mem (pa s (aP p i j)) (Am p σ i j) := - fun I j hj => by - have := I.l.k.d.im.A (p.ℓ * i + j) (by - have := Nat.mul_le_mul_left p.ℓ (show i + 1 ≤ p.k from hi) - rw [Nat.mul_add, Nat.mul_one, Nat.mul_comm p.ℓ p.k] at this; omega) - rw [aVal_ij hj] at this - rw [aP_ij]; exact this - have hY : ∀ {s : State} (I : ICw p D σ t i s) j, j < p.ℓ → PolyIs s.mem (pa s (yhP p j)) (YHv p σ (p.ℓ * t) j) := - fun I j hj => I.yh j hj + simp only [wChk, Bool.and_eq_true, decide_eq_true_eq] at hc + obtain ⟨⟨cd, c2⟩, hl⟩ := hc unfold rowW - refine WP.seq (WP.mono (mulAt_ok hP.mul h.l.st.lay (cm 0 hl) (hA h 0 hl).1 (hY h 0 hl).1) - fun s1 ⟨hP1, _, hq1⟩ => ?_) - have I1 := h.step hP1 c1 - rw [product, (hA h 0 hl).2, (hY h 0 hl).2, ← wAcc_one] at hq1 - refine WP.seq (WP.mono (seqR_ok (I := fun j s => ICw p D σ t i s ∧ Pl s (wBase p + i) (encode P.montgomery (wAcc p σ (p.ℓ * t) i j))) - (p.ℓ - 1) 1 (fun j hj1 hj s ⟨I, hw⟩ => ?_) s1 ⟨I1, by show PolyIs _ _ _; rw [hP1.pa (pS_bases _)]; exact hq1⟩) - fun s2 ⟨I2, hw2⟩ => ?_) - · refine WP.mono (mulAddAt_ok hP.mulAdd I.l.st.lay (cm j (by omega)) hw.1 (hA I j (by omega)).1 - (hY I j (by omega)).1) fun s' ⟨hP', _, hq'⟩ => ⟨I.step hP' c1, ?_⟩ - rw [product, hw.2, (hA I j (by omega)).2, (hY I j (by omega)).2, ← encode_add, ← wAcc_succ] at hq' - show PolyIs _ _ _ - rw [hP'.pa (pS_bases _)]; exact hq' - rw [show 1 + (p.ℓ - 1) = p.ℓ by omega] at hw2 - refine WP.mono (ipAt_ok (t := inverse P.montgomery) hP.invNtt I2.l.st.lay ci hw2.1) fun s3 ⟨hP3, _, hq3⟩ => ?_ - have I3 := I2.step hP3 c2 + refine WP.mono (dotAt_ok (hP.dotInv p.ℓ hl) h.l.st.lay cd + (fun j hj => by rw [famP_aP]; exact (rowA hi h hj).1) (fun j hj => by rw [famP_yhP]; exact (h.yh j hj).1)) + fun s3 ⟨hP3, _, hq3⟩ => ?_ + have I3 := h.step hP3 c2 refine ⟨I3.l, I3.y, I3.yh, Fam.snoc I3.w ?_⟩ show PolyIs _ _ _ rw [hP3.pa (pS_bases _)] - rw [hw2.2, inverse_encode] at hq3 + rw [VG.Proof.MlDsa.Arith.dotNTT_congr (fun j hj => by rw [famP_aP, (rowA hi h hj).2]) + (fun j hj => by rw [famP_yhP, (h.yh j hj).2])] at hq3 exact hq3 /-! ## `w₁` and `c̃` -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean index 092f3e3f0e..99326a2dc9 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PhaseCCT.lean @@ -215,38 +215,13 @@ theorem rowW_trL {P : Prims} {D : Nat} (hP : PrimsOk P D) {p : Params} {t i : Na (hi : i < p.k) : RelCT isa (fun x y => LRel D (sgR p) (sgW p) x y ∧ (∃ σ, ICw p D σ t i x) ∧ ∃ σ, ICw p D σ t i y) (rowW P p i) fun _ _ => True := by - simp only [wChk, Bool.and_eq_true, List.all_eq_true, List.mem_range, decide_eq_true_eq] at hc - obtain ⟨⟨⟨⟨cm, ci⟩, c1⟩, _⟩, hl⟩ := hc - have hA : ∀ {σ s : State} (I : ICw p D σ t i s) j, j < p.ℓ → Reduced s.mem (pa s (aP p i j)) := - fun I j hj => by - have := I.l.k.d.im.A (p.ℓ * i + j) (by - have := Nat.mul_le_mul_left p.ℓ (show i + 1 ≤ p.k from hi) - rw [Nat.mul_add, Nat.mul_one, Nat.mul_comm p.ℓ p.k] at this; omega) - rw [aP_ij]; exact this.1 - let J : State → Prop := fun s => (∃ σ, ICw p D σ t i s) ∧ Reduced s.mem (pa s (wP p i)) + simp only [wChk, Bool.and_eq_true, decide_eq_true_eq] at hc + obtain ⟨⟨cd, _⟩, hl⟩ := hc unfold rowW - refine seqL (J := J) ?_ ?_ (seqL (J := J) ?_ ?_ ?_) - · refine RelCT.mono (mulAt_tr hP.mul (cm 0 hl)) (fun x y ⟨R, ⟨σ₁, I₁⟩, ⟨σ₂, I₂⟩⟩ => - ⟨R, ⟨hA I₁ 0 hl, (I₁.yh 0 hl).1⟩, ⟨hA I₂ 0 hl, (I₂.yh 0 hl).1⟩⟩) fun _ _ h => h - · rintro x L ⟨σ, I⟩ - refine WP.mono (mulAt_ok hP.mul L (cm 0 hl) (hA I 0 hl) (I.yh 0 hl).1) fun x' ⟨hP', _, hq⟩ => - ⟨⟨_, hP'⟩, ⟨σ, I.step hP' c1⟩, by rw [hP'.pa (pS_bases _)]; exact hq.1⟩ - · refine RelCT.mono (seqR_tr (R := fun _ x y => LRel D (sgR p) (sgW p) x y ∧ J x ∧ J y) (p.ℓ - 1) 1 - fun j hj1 hj => stepSelf ?_ ?_) (fun _ _ h => h) fun _ _ _ => trivial - · refine RelCT.mono (mulAddAt_tr hP.mulAdd (cm j (by omega))) (fun x y ⟨R, ⟨⟨σ₁, I₁⟩, r₁⟩, ⟨⟨σ₂, I₂⟩, r₂⟩⟩ => - ⟨R, ⟨r₁, hA I₁ j (by omega), (I₁.yh j (by omega)).1⟩, ⟨r₂, hA I₂ j (by omega), (I₂.yh j (by omega)).1⟩⟩) - fun _ _ h => h - · rintro x L ⟨⟨σ, I⟩, r⟩ - refine WP.mono (mulAddAt_ok hP.mulAdd L (cm j (by omega)) r (hA I j (by omega)) (I.yh j (by omega)).1) - fun x' ⟨hP', _, hq⟩ => ⟨⟨_, hP'⟩, ⟨σ, I.step hP' c1⟩, by rw [hP'.pa (pS_bases _)]; exact hq.1⟩ - · rintro x L ⟨I, r⟩ - exact WP.mono (seqR_ok (I := fun _ s => ∃ W, PostB D x s W ∧ J s) (p.ℓ - 1) 1 - (fun j hj1 hj s ⟨W, hW, ⟨σ, I'⟩, r'⟩ => WP.mono (mulAddAt_ok hP.mulAdd I'.l.st.lay (cm j (by omega)) r' - (hA I' j (by omega)) (I'.yh j (by omega)).1) fun x' ⟨hP', _, hq⟩ => - ⟨_, hW.trans hP' (fun _ h => List.mem_append_left _ h) (fun _ h => List.mem_append_right _ h), - ⟨σ, I'.step hP' c1⟩, by rw [hP'.pa (pS_bases _)]; exact hq.1⟩) x ⟨[], PostB.refl D x [], I, r⟩) - fun x' ⟨W, hW, j⟩ => ⟨⟨W, hW⟩, j⟩ - · exact RelCT.mono (ipAt_tr (t := inverse P.montgomery) hP.invNtt ci) (fun _ _ h => ⟨h.1, h.2.1.2, h.2.2.2⟩) fun _ _ h => h + exact RelCT.mono (dotAt_tr (hP.dotInv p.ℓ hl) cd) (fun x y ⟨R, ⟨_, I₁⟩, ⟨_, I₂⟩⟩ => + ⟨R, ⟨fun j hj => by rw [famP_aP]; exact (rowA hi I₁ hj).1, fun j hj => by rw [famP_yhP]; exact (I₁.yh j hj).1⟩, + ⟨fun j hj => by rw [famP_aP]; exact (rowA hi I₂ hj).1, fun j hj => by rw [famP_yhP]; exact (I₂.yh j hj).1⟩⟩) + fun _ _ h => h /-! ## `w₁` -/ diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean index 5e4db9f9b3..467d0d77eb 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Prims.lean @@ -2,6 +2,9 @@ module public import VerifiedGarbage.Proof.MlDsa.Arith.Representation public import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Glue +public import VerifiedGarbage.Spec.MlDsa.FusedInverse +import VerifiedGarbage.Proof.MlDsa.Arith.DotSum +import VerifiedGarbage.Proof.Framework.AddrArith /-! # ML-DSA signing on x86-64: the primitives it calls @@ -36,7 +39,6 @@ structure PrimsOk (P : Prims) (D : Nat) where ntt : Callee (fun S => nttContract X86_64.abi S) D P.ntt invNtt : Callee (fun S => inverseContract P.montgomery X86_64.abi S) D P.invNtt mul : Callee (fun S => productContract P.montgomery X86_64.abi S) D P.mul - mulAdd : Callee (fun S => accumulateContract P.montgomery X86_64.abi S) D P.mulAdd add : Callee (fun S => addContract X86_64.abi S) D P.add sub : Callee (fun S => subContract X86_64.abi S) D P.sub rejNTT : Callee (fun S => rejNTTContract X86_64.abi S) D P.rejNTT @@ -52,6 +54,7 @@ structure PrimsOk (P : Prims) (D : Nat) where hintBitPack : Callee (fun S => hintBitPackContract X86_64.abi S) D P.hintBitPack rej4 : Callee (fun S => rejNTT4Contract X86_64.abi S) D P.rej4 expandMask4 : Callee (fun S => expandMask4Contract X86_64.abi S) D P.expandMask4 + dotInv : ∀ c, c = 4 ∨ c = 5 ∨ c = 7 → Callee (fun S => dotInverseContract c X86_64.abi S) D (P.dotInv c) /-- `vg_mldsa_rej_ntt_poly`'s result depends only on its public data (its seed). -/ rejRet : RetPub (rejNTTContract X86_64.abi rejNTT.S) P.rejNTT /-- `vg_mldsa_rej_ntt_poly` succeeds only if `RejNTTPoly` finishes within `maxBounds`. -/ @@ -271,22 +274,6 @@ theorem mulPre {S : Nat} (hS : S + 8 ≤ D) {s s1 : State} (A : At D rbs wbs s s simp only [List.mem_cons, List.mem_nil_iff, or_false, forall_eq_or_imp, forall_eq] exact ⟨A.stk hS i1, A.stk hS i2, A.stk hS i3⟩ -theorem mulAddPre {S : Nat} (hS : S + 8 ≤ D) {s s1 : State} (A : At D rbs wbs s s1) {h f g : Ptr} - (hc : mulChk (rbs ++ wbs) wbs h f g = true) (hA : ArgsIn [.ptr h, .ptr f, .ptr g] s s1) - (rh : Reduced s.mem (pa s h)) (rf : Reduced s.mem (pa s f)) (rg : Reduced s.mem (pa s g)) : - (accumulateContract P.montgomery X86_64.abi S).pre (s1.callEntry.withRegions [pR (pa s f), pR (pa s g)] [pR (pa s h)]) := by - obtain ⟨_, i1, i2, i3, d12, d13, _⟩ := mulChk_spec hc - obtain ⟨e1, e2, e3⟩ := argsIn3 hA - have hD : 8 ≤ D := by omega - have hwf := ce_wfS (ws := [64, 64, 64]) (by decide) hS (A.rsp ▸ A.L.sp) [pR (pa s f), pR (pa s g)] [pR (pa s h)] - sig_pre [accumulateContract, mulSig, X86_64.abi, VG.X86_64.argRegs] - simp only [e1, e2, e3, Arg.val] - refine ⟨hwf, trivial, trivial, A.L.disj d12, A.L.disj d13, A.ret i1 hD, A.ret i2 hD, ?_, A.L.nwp i1, A.L.nwp i2, - A.L.nwp i3, A.red' i1 hD rh, A.red' i2 hD rf, A.red' i3 hD rg⟩ - refine Sig.conj_cons.mpr ⟨A.ret i3 hD, conj_stk [pR (pa s h), pR (pa s f), pR (pa s g)] ?_⟩ - simp only [List.mem_cons, List.mem_nil_iff, or_false, forall_eq_or_imp, forall_eq] - exact ⟨A.stk hS i1, A.stk hS i2, A.stk hS i3⟩ - theorem mulAt_ok {P : Prims} (C : Callee (fun S => productContract P.montgomery X86_64.abi S) D P.mul) {s : State} (L : Lay D rbs wbs s) {h f g : Ptr} (hc : mulChk (rbs ++ wbs) wbs h f g = true) (rf : Reduced s.mem (pa s f)) (rg : Reduced s.mem (pa s g)) : @@ -305,25 +292,6 @@ theorem mulAt_ok {P : Prims} (C : Callee (fun S => productContract P.montgomery simp only [e1, e2, e3, Arg.val, hm₂, A.poly' i2 hD, A.poly' i3 hD] at hq exact hq -theorem mulAddAt_ok {P : Prims} (C : Callee (fun S => accumulateContract P.montgomery X86_64.abi S) D P.mulAdd) - {s : State} (L : Lay D rbs wbs s) {h f g : Ptr} (hc : mulChk (rbs ++ wbs) wbs h f g = true) - (rh : Reduced s.mem (pa s h)) (rf : Reduced s.mem (pa s f)) (rg : Reduced s.mem (pa s g)) : - WP isa (mulAddAt P h f g) s fun s' => PPostB D s s' [(h, 1024)] ∧ - (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ - PolyIs s'.mem (pa s h) - (add (polyAt s.mem (pa s h)) (product P.montgomery (polyAt s.mem (pa s f)) (polyAt s.mem (pa s g)))) := by - obtain ⟨w1, i1, i2, i3, _, _, ok⟩ := mulChk_spec hc - have hD : 8 ≤ D := by have := C.hS; omega - refine WP.mono (callP_ok C.ver.1 C.nosp C.depth L.dsm ok - (fun s1 hA hm k => mulAddPre C.hS (At.of L hm k) hc hA rh rf rg) - (Covers.append_left (Covers.cons (L.cR i2) (L.cR i3)) (Covers.right (L.cW w1))) (L.cW w1)) - fun s' ⟨hpost, hcs, s1, hA, hm, k, s₂, hm₂, _, hq⟩ => ⟨hpost, hcs, ?_⟩ - have A := At.of L hm k - obtain ⟨e1, e2, e3⟩ := argsIn3 hA - sig_post [accumulateContract, accumulate, mulSig, X86_64.abi, VG.X86_64.argRegs] at hq - simp only [e1, e2, e3, Arg.val, hm₂, A.poly' i1 hD, A.poly' i2 hD, A.poly' i3 hD] at hq - exact hq - /-- Two runs in the same layout, with the arguments of a call in their registers. -/ theorem argsRel {D : Nat} {rbs wbs : List (Reg × Nat)} {x y x1 y1 : State} (R : LRel D rbs wbs x y) {as : List Arg} (hx : ArgsIn as x x1) (hy : ArgsIn as y y1) @@ -355,26 +323,106 @@ theorem mulAt_tr {P : Prims} (C : Callee (fun S => productContract P.montgomery simp only [hx1, hx2, hx3, hy1, hy2, hy3, Arg.val, R.pa i1, R.pa i2, R.pa i3, (At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp, and_self] -theorem mulAddAt_tr {P : Prims} (C : Callee (fun S => accumulateContract P.montgomery X86_64.abi S) D P.mulAdd) - {h f g : Ptr} (hc : mulChk (rbs ++ wbs) wbs h f g = true) : +/-! ## A row's sum of products and its inverse transform -/ + +/-- Polynomial `j` of the family at `p`. -/ +abbrev famP (p : Ptr) (j : Nat) : Ptr := (p.1, p.2 + 1024 * j) + +theorem pa_famP (s : State) (p : Ptr) (j : Nat) : pa s (famP p j) = pa s p + BitVec.ofNat 64 (1024 * j) := by + simp only [pa]; rw [VG.Offset.add_add] + +/-- What a call of `vg_mldsa_dot_inverse{c}` on `h`, the families `f` and `g` +and `scratch` needs of the layout. -/ +def dotChk (bs wbs : List (Reg × Nat)) (h f g : Ptr) (c : Nat) : Bool := + inB wbs h 1024 && inB wbs (sc oPS) 1024 && inB bs h 1024 && inB bs (sc oPS) 1024 && + inB bs f (c * 256 * 4) && inB bs g (c * 256 * 4) && sepB bs h 1024 f (c * 256 * 4) && + sepB bs h 1024 g (c * 256 * 4) && sepB bs h 1024 (sc oPS) 1024 && sepB bs (sc oPS) 1024 f (c * 256 * 4) && + sepB bs (sc oPS) 1024 g (c * 256 * 4) && + (List.range c).all (fun j => inB bs (famP f j) 1024 && inB bs (famP g j) 1024) && + [Arg.ptr h, .ptr f, .ptr g, .ptr (sc oPS)].all Arg.ok + +theorem dotChk_spec {bs wbs : List (Reg × Nat)} {h f g : Ptr} {c : Nat} (hc : dotChk bs wbs h f g c = true) : + inB wbs h 1024 = true ∧ inB wbs (sc oPS) 1024 = true ∧ inB bs h 1024 = true ∧ inB bs (sc oPS) 1024 = true ∧ + inB bs f (c * 256 * 4) = true ∧ inB bs g (c * 256 * 4) = true ∧ sepB bs h 1024 f (c * 256 * 4) = true ∧ + sepB bs h 1024 g (c * 256 * 4) = true ∧ sepB bs h 1024 (sc oPS) 1024 = true ∧ + sepB bs (sc oPS) 1024 f (c * 256 * 4) = true ∧ sepB bs (sc oPS) 1024 g (c * 256 * 4) = true ∧ + (∀ j < c, inB bs (famP f j) 1024 = true ∧ inB bs (famP g j) 1024 = true) ∧ + [Arg.ptr h, .ptr f, .ptr g, .ptr (sc oPS)].all Arg.ok = true := by + simp only [dotChk, Bool.and_eq_true] at hc + obtain ⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨⟨h1, h2⟩, h3⟩, h4⟩, h5⟩, h6⟩, h7⟩, h8⟩, h9⟩, h10⟩, h11⟩, h12⟩, h13⟩ := hc + refine ⟨h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, fun j hj => ?_, h13⟩ + have := List.all_eq_true.mp h12 j (List.mem_range.mpr hj) + simpa only [Bool.and_eq_true] using this + +theorem dotPre {c S : Nat} (hS : S + 8 ≤ D) {s s1 : State} (A : At D rbs wbs s s1) {h f g : Ptr} + (hc : dotChk (rbs ++ wbs) wbs h f g c = true) (hA : ArgsIn [.ptr h, .ptr f, .ptr g, .ptr (sc oPS)] s s1) + (rf : ∀ j < c, Reduced s.mem (pa s (famP f j))) (rg : ∀ j < c, Reduced s.mem (pa s (famP g j))) : + (dotInverseContract c X86_64.abi S).pre (s1.callEntry.withRegions [⟨pa s f, c * 256 * 4⟩, ⟨pa s g, c * 256 * 4⟩] + [pR (pa s h), pR (pa s (sc oPS))]) := by + obtain ⟨w1, w2, i1, i2, i3, i4, d1, d2, d3, d4, d5, hfam, _⟩ := dotChk_spec hc + obtain ⟨e1, e2, e3, e4⟩ := argsIn4 hA + have hD : 8 ≤ D := by omega + have hwf := ce_wfS (ws := [64, 64, 64, 64]) (by decide) hS (A.rsp ▸ A.L.sp) + [⟨pa s f, c * 256 * 4⟩, ⟨pa s g, c * 256 * 4⟩] [pR (pa s h), pR (pa s (sc oPS))] + sig_pre [dotInverseContract, dotInverseSig, X86_64.abi, VG.X86_64.argRegs] + simp only [e1, e2, e3, e4, Arg.val] + have pos : ∀ {p : Ptr}, (∀ j < c, inB (rbs ++ wbs) (famP p j) 1024 = true) → (∀ j < c, Reduced s.mem (pa s (famP p j))) → + ∀ j < c, PositiveReduced (s1.mem.writeW (s1.gpr .rsp - 8) (s1.unknowns 0)) (pa s p + BitVec.ofNat 64 (1024 * j)) := + fun hi hr j hj k hk => by + rw [← pa_famP, A.coeff' (hi j hj) hD (by exact hk)] + exact Nat.lt_trans (hr j hj k hk) (by decide) + refine ⟨hwf, trivial, trivial, A.L.disj d1, A.L.disj d2, A.L.disj d3, (A.L.disj d4).symm, (A.L.disj d5).symm, + A.ret i1 hD, A.ret i3 hD, A.ret i4 hD, ?_, A.L.nwp i1, A.L.nwp i3, A.L.nwp i4, A.L.nwp i2, + pos (fun j hj => (hfam j hj).1) rf, pos (fun j hj => (hfam j hj).2) rg⟩ + refine Sig.conj_cons.mpr ⟨A.ret i2 hD, conj_stk [pR (pa s h), ⟨pa s f, c * 256 * 4⟩, ⟨pa s g, c * 256 * 4⟩, + pR (pa s (sc oPS))] ?_⟩ + simp only [List.mem_cons, List.mem_nil_iff, or_false, forall_eq_or_imp, forall_eq] + exact ⟨A.stk hS i1, A.stk hS i3, A.stk hS i4, A.stk hS i2⟩ + + +theorem dotAt_ok {P : Prims} {c : Nat} (C : Callee (fun S => dotInverseContract c X86_64.abi S) D (P.dotInv c)) + {s : State} (L : Lay D rbs wbs s) {h f g : Ptr} (hc : dotChk (rbs ++ wbs) wbs h f g c = true) + (rf : ∀ j < c, Reduced s.mem (pa s (famP f j))) (rg : ∀ j < c, Reduced s.mem (pa s (famP g j))) : + WP isa (dotInvAt P c h f g) s fun s' => PPostB D s s' [(h, 1024), (sc oPS, 1024)] ∧ + (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ + PolyIs s'.mem (pa s h) (nttInv (dotNTT (fun j => polyAt s.mem (pa s (famP f j))) + (fun j => polyAt s.mem (pa s (famP g j))) c)) := by + obtain ⟨w1, w2, i1, i2, i3, i4, _, _, _, _, _, hfam, ok⟩ := dotChk_spec hc + have hD : 8 ≤ D := by have := C.hS; omega + refine WP.mono (callP_ok C.ver.1 C.nosp C.depth L.dsm ok + (fun s1 hA hm k => dotPre C.hS (At.of L hm k) hc hA rf rg) + (Covers.append_left (Covers.cons (L.cR i3) (L.cR i4)) (Covers.right (Covers.cons (L.cW w1) (L.cW w2)))) + (Covers.cons (L.cW w1) (L.cW w2))) + fun s' ⟨hpost, hcs, s1, hA, hm, k, s₂, hm₂, _, hq⟩ => ⟨hpost, hcs, ?_⟩ + have A := At.of L hm k + obtain ⟨e1, e2, e3, _⟩ := argsIn4 hA + sig_post [dotInverseContract, dotInverseSig, X86_64.abi, VG.X86_64.argRegs] at hq + simp only [e1, e2, e3, Arg.val, hm₂] at hq + rw [VG.Proof.MlDsa.Arith.dotNTT_congr (fun j hj => by rw [← pa_famP, A.poly' (hfam j hj).1 hD]) + (fun j hj => by rw [← pa_famP, A.poly' (hfam j hj).2 hD])] at hq + exact hq + +theorem dotAt_tr {P : Prims} {c : Nat} (C : Callee (fun S => dotInverseContract c X86_64.abi S) D (P.dotInv c)) + {h f g : Ptr} (hc : dotChk (rbs ++ wbs) wbs h f g c = true) : RelCT isa (fun x y => LRel D rbs wbs x y ∧ - (Reduced x.mem (pa x h) ∧ Reduced x.mem (pa x f) ∧ Reduced x.mem (pa x g)) ∧ - (Reduced y.mem (pa y h) ∧ Reduced y.mem (pa y f) ∧ Reduced y.mem (pa y g))) (mulAddAt P h f g) - fun _ _ => True := by - obtain ⟨w1, i1, i2, i3, _, _, ok⟩ := mulChk_spec hc + ((∀ j < c, Reduced x.mem (pa x (famP f j))) ∧ ∀ j < c, Reduced x.mem (pa x (famP g j))) ∧ + ((∀ j < c, Reduced y.mem (pa y (famP f j))) ∧ ∀ j < c, Reduced y.mem (pa y (famP g j)))) + (dotInvAt P c h f g) fun _ _ => True := by + obtain ⟨w1, w2, i1, i2, i3, i4, _, _, _, _, _, _, ok⟩ := dotChk_spec hc refine callP_tr C.ver.1 C.ver.2.1 ok - fun x y x1 y1 ⟨R, ⟨rhx, rfx, rgx⟩, ⟨rhy, rfy, rgy⟩⟩ ⟨⟨hAx, hmx⟩, kx⟩ ⟨⟨hAy, hmy⟩, ky⟩ => - ⟨_, _, _, _, mulAddPre C.hS (At.of R.lx hmx kx) hc hAx rhx rfx rgx, - mulAddPre C.hS (At.of R.ly hmy ky) hc hAy rhy rfy rgy, ?_, - by rw [kx.2.1, kx.2.2]; exact Covers.append_left (Covers.cons (R.lx.cR i2) (R.lx.cR i3)) (Covers.right (R.lx.cW w1)), - by rw [kx.2.2]; exact R.lx.cW w1, - by rw [ky.2.1, ky.2.2]; exact Covers.append_left (Covers.cons (R.ly.cR i2) (R.ly.cR i3)) (Covers.right (R.ly.cW w1)), - by rw [ky.2.2]; exact R.ly.cW w1, + fun x y x1 y1 ⟨R, ⟨rfx, rgx⟩, ⟨rfy, rgy⟩⟩ ⟨⟨hAx, hmx⟩, kx⟩ ⟨⟨hAy, hmy⟩, ky⟩ => + ⟨_, _, _, _, dotPre C.hS (At.of R.lx hmx kx) hc hAx rfx rgx, dotPre C.hS (At.of R.ly hmy ky) hc hAy rfy rgy, ?_, + by rw [kx.2.1, kx.2.2]; exact (Covers.append_left (Covers.cons (R.lx.cR i3) (R.lx.cR i4)) + (Covers.right (Covers.cons (R.lx.cW w1) (R.lx.cW w2)))), + by rw [kx.2.2]; exact Covers.cons (R.lx.cW w1) (R.lx.cW w2), + by rw [ky.2.1, ky.2.2]; exact (Covers.append_left (Covers.cons (R.ly.cR i3) (R.ly.cR i4)) + (Covers.right (Covers.cons (R.ly.cW w1) (R.ly.cW w2)))), + by rw [ky.2.2]; exact Covers.cons (R.ly.cW w1) (R.ly.cW w2), by rw [(At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp]⟩ - obtain ⟨hx1, hx2, hx3⟩ := argsIn3 hAx - obtain ⟨hy1, hy2, hy3⟩ := argsIn3 hAy - sig_pub [accumulateContract, mulSig, X86_64.abi, VG.X86_64.argRegs] - simp only [hx1, hx2, hx3, hy1, hy2, hy3, Arg.val, R.pa i1, R.pa i2, R.pa i3, + obtain ⟨hx1, hx2, hx3, hx4⟩ := argsIn4 hAx + obtain ⟨hy1, hy2, hy3, hy4⟩ := argsIn4 hAy + sig_pub [dotInverseContract, dotInverseSig, X86_64.abi, VG.X86_64.argRegs] + simp only [hx1, hx2, hx3, hx4, hy1, hy2, hy3, hy4, Arg.val, R.pa i1, R.pa i2, R.pa i3, R.pa i4, (At.of R.lx hmx kx).rsp, (At.of R.ly hmy ky).rsp, R.rsp, and_self] end diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean index 95bc801209..2bf515d701 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean @@ -10,6 +10,7 @@ public import VerifiedGarbage.Proof.MlDsa.Sign.Leak public import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Inv public import VerifiedGarbage.Proof.MlDsa.X86_64.Sign.Top public import VerifiedGarbage.Spec.MlDsa.Contract +import VerifiedGarbage.Impl.MlDsa.X86_64.Arith.Montgomery /-! # ML-DSA signing on x86-64: verified @@ -86,12 +87,24 @@ and the stack pointer (`sign_spSafe`) are checked by evaluating signing with no implementation of it. -/ theorem sign_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) - (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) + (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h5 : mc B.add = true) (h6 : mc B.sub = true) (h7 : mc B.rej4 = true) (h8 : mc B.expandMask4 = true) (h9 : mc B.highBits = true) - (h10 : mc B.lowBits = true) (h11 : mc B.normLt = true) (h12 : mc B.makeHint = true) (p : Params) : + (h10 : mc B.lowBits = true) (h11 : mc B.normLt = true) (h12 : mc B.makeHint = true) + (h13 : mc B.dotInv4 = true) (h14 : mc B.dotInv5 = true) (h15 : mc B.dotInv7 = true) (p : Params) : Same m (Impl.MlDsa.X86_64.Sign.sign (primsWith B) p) (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) := by + have hd : mc ((primsWith B).dotInv p.ℓ) = true := by + unfold Prims.dotInv; split + · exact h13 + · split + · exact h14 + · exact h15 + have he : (primsWith Backend.empty).dotInv p.ℓ = .block [] := by + unfold Prims.dotInv; split + · rfl + · split <;> rfl unfold Impl.MlDsa.X86_64.Sign.sign same_tac hm + all_goals rw [he]; exact Same.call hm hd _ _ theorem sign0_ctlC {p : Params} (h3 : Ok3 p) : ctlC (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) = true := by rcases h3 with rfl | rfl | rfl @@ -110,18 +123,21 @@ variable (v : ArithImpl) {p : Params} (h3 : Ok3 p) include h3 theorem sign_ctl : ctlOk (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) = true := - ctlOk_of_ctlC (Same.ok (sign_same Comp.ctlC v.ok.ntt.ctl v.ok.invNtt.ctl v.ok.mul.ctl v.ok.mulAdd.ctl + ctlOk_of_ctlC (Same.ok (sign_same Comp.ctlC v.ok.ntt.ctl v.ok.invNtt.ctl v.ok.mul.ctl v.ok.add.ctl v.ok.sub.ctl v.ok.rej4.ctl v.ok.expandMask4.ctl v.ok.highBits.ctl v.ok.lowBits.ctl - v.ok.normLt.ctl v.ok.makeHint.ctl p) (sign0_ctlC h3)) + v.ok.normLt.ctl v.ok.makeHint.ctl (v.ok.dotInv 4 (.inl rfl)).ctl (v.ok.dotInv 5 (.inr (.inl rfl))).ctl + (v.ok.dotInv 7 (.inr (.inr rfl))).ctl p) (sign0_ctlC h3)) theorem sign_spSafe : (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs (Same.ok (sign_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) - (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) + (Code.allInstrs_of_all v.ok.add.sp) (Code.allInstrs_of_all v.ok.sub.sp) (Code.allInstrs_of_all v.ok.rej4.sp) (Code.allInstrs_of_all v.ok.expandMask4.sp) (Code.allInstrs_of_all v.ok.highBits.sp) (Code.allInstrs_of_all v.ok.lowBits.sp) - (Code.allInstrs_of_all v.ok.normLt.sp) (Code.allInstrs_of_all v.ok.makeHint.sp) p) (sign0_sp h3)) + (Code.allInstrs_of_all v.ok.normLt.sp) (Code.allInstrs_of_all v.ok.makeHint.sp) + (Code.allInstrs_of_all (v.ok.dotInv 4 (.inl rfl)).sp) (Code.allInstrs_of_all (v.ok.dotInv 5 (.inr (.inl rfl))).sp) + (Code.allInstrs_of_all (v.ok.dotInv 7 (.inr (.inr rfl))).sp) p) (sign0_sp h3)) theorem sign_verified : Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) (signContractT p X86_64.abi signStack) := diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index dfef7731ef..69893e55dd 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -3307,6 +3307,4608 @@ ret ) } +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 4 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on four coefficients at a time in SSE2 registers: it sums the products in 64-bit lanes, reduces each sum once, then runs `vg_mldsa_inv_ntt`'s code, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 4096 bytes. +/// * `b` must be valid for reads of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse4(out: *mut [u32; 256], a: *const [u32; 1024], b: *const [u32; 1024], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov eax, 2365951 +movq xmm11, rax +pshufd xmm11, xmm11, 0 +mov r8, rdi +mov ecx, 64 +20: +pxor xmm5, xmm5 +pxor xmm6, xmm6 +movdqu xmm0, XMMWORD PTR [r10] +movdqu xmm1, XMMWORD PTR [rdx] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+1024] +movdqu xmm1, XMMWORD PTR [rdx+1024] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+2048] +movdqu xmm1, XMMWORD PTR [rdx+2048] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+3072] +movdqu xmm1, XMMWORD PTR [rdx+3072] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm6 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm6, xmm2 +por xmm5, xmm6 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +pshufd xmm4, xmm5, 245 +pmuludq xmm5, xmm11 +pmuludq xmm4, xmm11 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm5, xmm4 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +movdqu XMMWORD PTR [r8], xmm5 +add r8, 16 +add r10, 16 +add rdx, 16 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov rdx, rdi +mov r8, rsi +add r8, 1008 +mov ecx, 32 +21: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm2, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 27 +pshufd xmm12, xmm13, 245 +add r8, -16 +pshufd xmm0, xmm0, 216 +pshufd xmm2, xmm2, 216 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm2 +punpckhqdq xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpckldq xmm0, xmm3 +punpckhdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 504 +mov ecx, 32 +22: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 5 +pshufd xmm12, xmm13, 245 +add r8, -8 +movdqa xmm2, xmm0 +punpcklqdq xmm0, xmm1 +punpckhqdq xmm2, xmm1 +movdqa xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm3 +punpckhqdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 252 +mov eax, 32 +23: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 1 +24: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm3 +add rdx, 16 +sub rcx, 1 +jne 24b +add rdx, 16 +sub rax, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +25: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 2 +26: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+32] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+32], xmm3 +add rdx, 16 +sub rcx, 1 +jne 26b +add rdx, 32 +sub rax, 1 +jne 25b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +27: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 4 +28: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+64] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+64], xmm3 +add rdx, 16 +sub rcx, 1 +jne 28b +add rdx, 64 +sub rax, 1 +jne 27b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +29: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 8 +210: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+128] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+128], xmm3 +add rdx, 16 +sub rcx, 1 +jne 210b +add rdx, 128 +sub rax, 1 +jne 29b +mov rdx, rdi +mov r8, rsi +add r8, 12 +mov eax, 2 +211: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 16 +212: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+256] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+256], xmm3 +add rdx, 16 +sub rcx, 1 +jne 212b +add rdx, 256 +sub rax, 1 +jne 211b +mov rdx, rdi +mov r8, rsi +add r8, 4 +mov eax, 1 +213: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 32 +214: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+512] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+512], xmm3 +add rdx, 16 +sub rcx, 1 +jne 214b +add rdx, 512 +sub rax, 1 +jne 213b +mov rdx, rdi +mov eax, 16382 +movq xmm13, rax +pshufd xmm13, xmm13, 0 +movdqa xmm12, xmm13 +mov ecx, 64 +215: +movdqu xmm3, XMMWORD PTR [rdx] +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm3 +add rdx, 16 +sub rcx, 1 +jne 215b +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 5 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on four coefficients at a time in SSE2 registers: it sums the products in 64-bit lanes, reduces each sum once, then runs `vg_mldsa_inv_ntt`'s code, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 5120 bytes. +/// * `b` must be valid for reads of 5120 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse5(out: *mut [u32; 256], a: *const [u32; 1280], b: *const [u32; 1280], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov eax, 2365951 +movq xmm11, rax +pshufd xmm11, xmm11, 0 +mov r8, rdi +mov ecx, 64 +20: +pxor xmm5, xmm5 +pxor xmm6, xmm6 +movdqu xmm0, XMMWORD PTR [r10] +movdqu xmm1, XMMWORD PTR [rdx] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+1024] +movdqu xmm1, XMMWORD PTR [rdx+1024] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+2048] +movdqu xmm1, XMMWORD PTR [rdx+2048] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+3072] +movdqu xmm1, XMMWORD PTR [rdx+3072] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+4096] +movdqu xmm1, XMMWORD PTR [rdx+4096] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm6 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm6, xmm2 +por xmm5, xmm6 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +pshufd xmm4, xmm5, 245 +pmuludq xmm5, xmm11 +pmuludq xmm4, xmm11 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm5, xmm4 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +movdqu XMMWORD PTR [r8], xmm5 +add r8, 16 +add r10, 16 +add rdx, 16 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov rdx, rdi +mov r8, rsi +add r8, 1008 +mov ecx, 32 +21: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm2, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 27 +pshufd xmm12, xmm13, 245 +add r8, -16 +pshufd xmm0, xmm0, 216 +pshufd xmm2, xmm2, 216 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm2 +punpckhqdq xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpckldq xmm0, xmm3 +punpckhdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 504 +mov ecx, 32 +22: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 5 +pshufd xmm12, xmm13, 245 +add r8, -8 +movdqa xmm2, xmm0 +punpcklqdq xmm0, xmm1 +punpckhqdq xmm2, xmm1 +movdqa xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm3 +punpckhqdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 252 +mov eax, 32 +23: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 1 +24: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm3 +add rdx, 16 +sub rcx, 1 +jne 24b +add rdx, 16 +sub rax, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +25: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 2 +26: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+32] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+32], xmm3 +add rdx, 16 +sub rcx, 1 +jne 26b +add rdx, 32 +sub rax, 1 +jne 25b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +27: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 4 +28: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+64] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+64], xmm3 +add rdx, 16 +sub rcx, 1 +jne 28b +add rdx, 64 +sub rax, 1 +jne 27b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +29: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 8 +210: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+128] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+128], xmm3 +add rdx, 16 +sub rcx, 1 +jne 210b +add rdx, 128 +sub rax, 1 +jne 29b +mov rdx, rdi +mov r8, rsi +add r8, 12 +mov eax, 2 +211: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 16 +212: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+256] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+256], xmm3 +add rdx, 16 +sub rcx, 1 +jne 212b +add rdx, 256 +sub rax, 1 +jne 211b +mov rdx, rdi +mov r8, rsi +add r8, 4 +mov eax, 1 +213: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 32 +214: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+512] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+512], xmm3 +add rdx, 16 +sub rcx, 1 +jne 214b +add rdx, 512 +sub rax, 1 +jne 213b +mov rdx, rdi +mov eax, 16382 +movq xmm13, rax +pshufd xmm13, xmm13, 0 +movdqa xmm12, xmm13 +mov ecx, 64 +215: +movdqu xmm3, XMMWORD PTR [rdx] +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm3 +add rdx, 16 +sub rcx, 1 +jne 215b +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 7 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on four coefficients at a time in SSE2 registers: it sums the products in 64-bit lanes, reduces each sum once, then runs `vg_mldsa_inv_ntt`'s code, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 7168 bytes. +/// * `b` must be valid for reads of 7168 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse7(out: *mut [u32; 256], a: *const [u32; 1792], b: *const [u32; 1792], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov eax, 2365951 +movq xmm11, rax +pshufd xmm11, xmm11, 0 +mov r8, rdi +mov ecx, 64 +20: +pxor xmm5, xmm5 +pxor xmm6, xmm6 +movdqu xmm0, XMMWORD PTR [r10] +movdqu xmm1, XMMWORD PTR [rdx] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+1024] +movdqu xmm1, XMMWORD PTR [rdx+1024] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+2048] +movdqu xmm1, XMMWORD PTR [rdx+2048] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+3072] +movdqu xmm1, XMMWORD PTR [rdx+3072] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+4096] +movdqu xmm1, XMMWORD PTR [rdx+4096] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+5120] +movdqu xmm1, XMMWORD PTR [rdx+5120] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqu xmm0, XMMWORD PTR [r10+6144] +movdqu xmm1, XMMWORD PTR [rdx+6144] +movdqa xmm2, xmm0 +psrlq xmm2, 32 +movdqa xmm3, xmm1 +psrlq xmm3, 32 +pmuludq xmm2, xmm3 +pmuludq xmm0, xmm1 +paddq xmm5, xmm0 +paddq xmm6, xmm2 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm6 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm6, xmm2 +por xmm5, xmm6 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +pshufd xmm4, xmm5, 245 +pmuludq xmm5, xmm11 +pmuludq xmm4, xmm11 +movdqa xmm2, xmm5 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm5, xmm2 +psrlq xmm5, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm5, xmm4 +psubd xmm5, xmm15 +movdqa xmm2, xmm5 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm5, xmm2 +movdqu XMMWORD PTR [r8], xmm5 +add r8, 16 +add r10, 16 +add rdx, 16 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +movq xmm15, rax +pshufd xmm15, xmm15, 0 +mov eax, -58728449 +movq xmm14, rax +pshufd xmm14, xmm14, 0 +mov rdx, rdi +mov r8, rsi +add r8, 1008 +mov ecx, 32 +21: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm2, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 27 +pshufd xmm12, xmm13, 245 +add r8, -16 +pshufd xmm0, xmm0, 216 +pshufd xmm2, xmm2, 216 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm2 +punpckhqdq xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpckldq xmm0, xmm3 +punpckhdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 504 +mov ecx, 32 +22: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 5 +pshufd xmm12, xmm13, 245 +add r8, -8 +movdqa xmm2, xmm0 +punpcklqdq xmm0, xmm1 +punpckhqdq xmm2, xmm1 +movdqa xmm1, xmm2 +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqa xmm1, xmm0 +punpcklqdq xmm0, xmm3 +punpckhqdq xmm1, xmm3 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm1 +add rdx, 32 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 252 +mov eax, 32 +23: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 1 +24: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+16] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+16], xmm3 +add rdx, 16 +sub rcx, 1 +jne 24b +add rdx, 16 +sub rax, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +25: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 2 +26: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+32] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+32], xmm3 +add rdx, 16 +sub rcx, 1 +jne 26b +add rdx, 32 +sub rax, 1 +jne 25b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +27: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 4 +28: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+64] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+64], xmm3 +add rdx, 16 +sub rcx, 1 +jne 28b +add rdx, 64 +sub rax, 1 +jne 27b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +29: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 8 +210: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+128] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+128], xmm3 +add rdx, 16 +sub rcx, 1 +jne 210b +add rdx, 128 +sub rax, 1 +jne 29b +mov rdx, rdi +mov r8, rsi +add r8, 12 +mov eax, 2 +211: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 16 +212: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+256] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+256], xmm3 +add rdx, 16 +sub rcx, 1 +jne 212b +add rdx, 256 +sub rax, 1 +jne 211b +mov rdx, rdi +mov r8, rsi +add r8, 4 +mov eax, 1 +213: +movdqu xmm13, XMMWORD PTR [r8] +pshufd xmm13, xmm13, 0 +pshufd xmm12, xmm13, 245 +add r8, -4 +mov ecx, 32 +214: +movdqu xmm0, XMMWORD PTR [rdx] +movdqu xmm1, XMMWORD PTR [rdx+512] +movdqa xmm3, xmm1 +psubd xmm3, xmm0 +paddd xmm3, xmm15 +paddd xmm0, xmm1 +psubd xmm0, xmm15 +movdqa xmm2, xmm0 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm0, xmm2 +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm0 +movdqu XMMWORD PTR [rdx+512], xmm3 +add rdx, 16 +sub rcx, 1 +jne 214b +add rdx, 512 +sub rax, 1 +jne 213b +mov rdx, rdi +mov eax, 16382 +movq xmm13, rax +pshufd xmm13, xmm13, 0 +movdqa xmm12, xmm13 +mov ecx, 64 +215: +movdqu xmm3, XMMWORD PTR [rdx] +pshufd xmm4, xmm3, 245 +pmuludq xmm3, xmm13 +pmuludq xmm4, xmm12 +movdqa xmm2, xmm3 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm3, xmm2 +psrlq xmm3, 32 +movdqa xmm2, xmm4 +pmuludq xmm2, xmm14 +pmuludq xmm2, xmm15 +paddq xmm4, xmm2 +por xmm3, xmm4 +psubd xmm3, xmm15 +movdqa xmm2, xmm3 +psrad xmm2, 31 +pand xmm2, xmm15 +paddd xmm3, xmm2 +movdqu XMMWORD PTR [rdx], xmm3 +add rdx, 16 +sub rcx, 1 +jne 215b +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + +/// The CPU features `vg_mldsa_dot_inverse4_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_DOT_INVERSE4_AVX2_FEATURES: crate::cpu::Features = crate::cpu::Features::of(&["avx", "avx2"]); + +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 4 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on eight coefficients at a time in AVX2 registers: it sums the products in 64-bit lanes, reduces each sum once to the sum times `2⁻³²`, then runs `vg_mldsa_montgomery_inv_ntt_avx2`'s code, which cancels the factor, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 4096 bytes. +/// * `b` must be valid for reads of 4096 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse4_avx2(out: *mut [u32; 256], a: *const [u32; 1024], b: *const [u32; 1024], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov r8, rdi +mov ecx, 32 +20: +vpxor ymm5, ymm5, ymm5 +vpxor ymm6, ymm6, ymm6 +vmovdqu ymm0, YMMWORD PTR [r10] +vmovdqu ymm1, YMMWORD PTR [rdx] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+1024] +vmovdqu ymm1, YMMWORD PTR [rdx+1024] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+2048] +vmovdqu ymm1, YMMWORD PTR [rdx+2048] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+3072] +vmovdqu ymm1, YMMWORD PTR [rdx+3072] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpor ymm5, ymm5, ymm6 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [r8], ymm5 +add r8, 32 +add r10, 32 +add rdx, 32 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov eax, 1072693376 +vmovq xmm11, rax +vpbroadcastd ymm11, xmm11 +mov rdx, rdi +mov r8, rsi +add r8, 992 +mov ecx, 16 +21: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm2, YMMWORD PTR [rdx+32] +vmovdqu ymm13, YMMWORD PTR [r8] +vpermq ymm13, ymm13, 39 +vpshufd ymm13, ymm13, 177 +vpshufd ymm12, ymm13, 245 +add r8, -32 +vpshufd ymm0, ymm0, 216 +vpshufd ymm2, ymm2, 216 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm2 +vpunpckhqdq ymm1, ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpckldq ymm0, ymm0, ymm3 +vpunpckhdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 496 +mov ecx, 16 +22: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 10 +vpshufd ymm13, ymm13, 95 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -16 +vmovdqa ymm2, ymm0 +vpunpcklqdq ymm0, ymm0, ymm1 +vpunpckhqdq ymm2, ymm2, ymm1 +vmovdqa ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm3 +vpunpckhqdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 248 +mov ecx, 16 +23: +vmovdqu ymm4, YMMWORD PTR [rdx] +vmovdqu ymm5, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 0 +vpshufd ymm13, ymm13, 85 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -8 +vperm2i128 ymm0, ymm4, ymm5, 32 +vperm2i128 ymm1, ymm4, ymm5, 49 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vperm2i128 ymm4, ymm0, ymm3, 32 +vperm2i128 ymm5, ymm0, ymm3, 49 +vmovdqu YMMWORD PTR [rdx], ymm4 +vmovdqu YMMWORD PTR [rdx+32], ymm5 +add rdx, 64 +sub rcx, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +24: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 1 +25: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm3 +add rdx, 32 +sub rcx, 1 +jne 25b +add rdx, 32 +sub rax, 1 +jne 24b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +26: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 2 +27: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+64] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+64], ymm3 +add rdx, 32 +sub rcx, 1 +jne 27b +add rdx, 64 +sub rax, 1 +jne 26b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +28: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 4 +29: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+128] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+128], ymm3 +add rdx, 32 +sub rcx, 1 +jne 29b +add rdx, 128 +sub rax, 1 +jne 28b +mov rdx, rdi +mov eax, 7861508 +vmovq xmm7, rax +vpbroadcastd ymm7, xmm7 +mov eax, 5771523 +vmovq xmm8, rax +vpbroadcastd ymm8, xmm8 +mov eax, 4404704 +vmovq xmm9, rax +vpbroadcastd ymm9, xmm9 +mov eax, 41978 +vmovq xmm10, rax +vpbroadcastd ymm10, xmm10 +mov ecx, 8 +210: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+256] +vmovdqu ymm5, YMMWORD PTR [rdx+512] +vmovdqu ymm6, YMMWORD PTR [rdx+768] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm7 +vpmuludq ymm4, ymm4, ymm7 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm1, ymm6, ymm5 +vpaddd ymm1, ymm1, ymm11 +vpaddd ymm5, ymm5, ymm6 +vpshufd ymm4, ymm1, 245 +vpmuludq ymm1, ymm1, ymm8 +vpmuludq ymm4, ymm4, ymm8 +vpmuludq ymm2, ymm1, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm1, ymm1, ymm2 +vpsrlq ymm1, ymm1, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm1, ymm1, ymm4 +vpsubd ymm6, ymm5, ymm0 +vpaddd ymm6, ymm6, ymm11 +vpaddd ymm0, ymm0, ymm5 +vpshufd ymm4, ymm6, 245 +vpmuludq ymm6, ymm6, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpsrlq ymm6, ymm6, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm6, ymm6, ymm4 +vpsubd ymm5, ymm1, ymm3 +vpaddd ymm5, ymm5, ymm11 +vpaddd ymm3, ymm3, ymm1 +vpshufd ymm4, ymm5, 245 +vpmuludq ymm5, ymm5, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm5, ymm5, ymm4 +vpshufd ymm4, ymm0, 245 +vpmuludq ymm0, ymm0, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm0, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm0, ymm0, ymm2 +vpsrlq ymm0, ymm0, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm0, ymm0, ymm4 +vpsubd ymm0, ymm0, ymm15 +vpsrad ymm2, ymm0, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm0, ymm0, ymm2 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm3, ymm3, ymm15 +vpsrad ymm2, ymm3, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm3, ymm3, ymm2 +vpsubd ymm6, ymm6, ymm15 +vpsrad ymm2, ymm6, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm6, ymm6, ymm2 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+256], ymm3 +vmovdqu YMMWORD PTR [rdx+512], ymm6 +vmovdqu YMMWORD PTR [rdx+768], ymm5 +add rdx, 32 +sub rcx, 1 +jne 210b +vzeroupper +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + +/// The CPU features `vg_mldsa_dot_inverse5_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_DOT_INVERSE5_AVX2_FEATURES: crate::cpu::Features = crate::cpu::Features::of(&["avx", "avx2"]); + +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 5 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on eight coefficients at a time in AVX2 registers: it sums the products in 64-bit lanes, reduces each sum once to the sum times `2⁻³²`, then runs `vg_mldsa_montgomery_inv_ntt_avx2`'s code, which cancels the factor, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 5120 bytes. +/// * `b` must be valid for reads of 5120 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse5_avx2(out: *mut [u32; 256], a: *const [u32; 1280], b: *const [u32; 1280], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov r8, rdi +mov ecx, 32 +20: +vpxor ymm5, ymm5, ymm5 +vpxor ymm6, ymm6, ymm6 +vmovdqu ymm0, YMMWORD PTR [r10] +vmovdqu ymm1, YMMWORD PTR [rdx] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+1024] +vmovdqu ymm1, YMMWORD PTR [rdx+1024] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+2048] +vmovdqu ymm1, YMMWORD PTR [rdx+2048] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+3072] +vmovdqu ymm1, YMMWORD PTR [rdx+3072] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+4096] +vmovdqu ymm1, YMMWORD PTR [rdx+4096] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpor ymm5, ymm5, ymm6 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [r8], ymm5 +add r8, 32 +add r10, 32 +add rdx, 32 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov eax, 1072693376 +vmovq xmm11, rax +vpbroadcastd ymm11, xmm11 +mov rdx, rdi +mov r8, rsi +add r8, 992 +mov ecx, 16 +21: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm2, YMMWORD PTR [rdx+32] +vmovdqu ymm13, YMMWORD PTR [r8] +vpermq ymm13, ymm13, 39 +vpshufd ymm13, ymm13, 177 +vpshufd ymm12, ymm13, 245 +add r8, -32 +vpshufd ymm0, ymm0, 216 +vpshufd ymm2, ymm2, 216 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm2 +vpunpckhqdq ymm1, ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpckldq ymm0, ymm0, ymm3 +vpunpckhdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 496 +mov ecx, 16 +22: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 10 +vpshufd ymm13, ymm13, 95 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -16 +vmovdqa ymm2, ymm0 +vpunpcklqdq ymm0, ymm0, ymm1 +vpunpckhqdq ymm2, ymm2, ymm1 +vmovdqa ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm3 +vpunpckhqdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 248 +mov ecx, 16 +23: +vmovdqu ymm4, YMMWORD PTR [rdx] +vmovdqu ymm5, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 0 +vpshufd ymm13, ymm13, 85 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -8 +vperm2i128 ymm0, ymm4, ymm5, 32 +vperm2i128 ymm1, ymm4, ymm5, 49 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vperm2i128 ymm4, ymm0, ymm3, 32 +vperm2i128 ymm5, ymm0, ymm3, 49 +vmovdqu YMMWORD PTR [rdx], ymm4 +vmovdqu YMMWORD PTR [rdx+32], ymm5 +add rdx, 64 +sub rcx, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +24: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 1 +25: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm3 +add rdx, 32 +sub rcx, 1 +jne 25b +add rdx, 32 +sub rax, 1 +jne 24b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +26: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 2 +27: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+64] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+64], ymm3 +add rdx, 32 +sub rcx, 1 +jne 27b +add rdx, 64 +sub rax, 1 +jne 26b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +28: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 4 +29: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+128] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+128], ymm3 +add rdx, 32 +sub rcx, 1 +jne 29b +add rdx, 128 +sub rax, 1 +jne 28b +mov rdx, rdi +mov eax, 7861508 +vmovq xmm7, rax +vpbroadcastd ymm7, xmm7 +mov eax, 5771523 +vmovq xmm8, rax +vpbroadcastd ymm8, xmm8 +mov eax, 4404704 +vmovq xmm9, rax +vpbroadcastd ymm9, xmm9 +mov eax, 41978 +vmovq xmm10, rax +vpbroadcastd ymm10, xmm10 +mov ecx, 8 +210: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+256] +vmovdqu ymm5, YMMWORD PTR [rdx+512] +vmovdqu ymm6, YMMWORD PTR [rdx+768] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm7 +vpmuludq ymm4, ymm4, ymm7 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm1, ymm6, ymm5 +vpaddd ymm1, ymm1, ymm11 +vpaddd ymm5, ymm5, ymm6 +vpshufd ymm4, ymm1, 245 +vpmuludq ymm1, ymm1, ymm8 +vpmuludq ymm4, ymm4, ymm8 +vpmuludq ymm2, ymm1, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm1, ymm1, ymm2 +vpsrlq ymm1, ymm1, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm1, ymm1, ymm4 +vpsubd ymm6, ymm5, ymm0 +vpaddd ymm6, ymm6, ymm11 +vpaddd ymm0, ymm0, ymm5 +vpshufd ymm4, ymm6, 245 +vpmuludq ymm6, ymm6, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpsrlq ymm6, ymm6, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm6, ymm6, ymm4 +vpsubd ymm5, ymm1, ymm3 +vpaddd ymm5, ymm5, ymm11 +vpaddd ymm3, ymm3, ymm1 +vpshufd ymm4, ymm5, 245 +vpmuludq ymm5, ymm5, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm5, ymm5, ymm4 +vpshufd ymm4, ymm0, 245 +vpmuludq ymm0, ymm0, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm0, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm0, ymm0, ymm2 +vpsrlq ymm0, ymm0, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm0, ymm0, ymm4 +vpsubd ymm0, ymm0, ymm15 +vpsrad ymm2, ymm0, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm0, ymm0, ymm2 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm3, ymm3, ymm15 +vpsrad ymm2, ymm3, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm3, ymm3, ymm2 +vpsubd ymm6, ymm6, ymm15 +vpsrad ymm2, ymm6, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm6, ymm6, ymm2 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+256], ymm3 +vmovdqu YMMWORD PTR [rdx+512], ymm6 +vmovdqu YMMWORD PTR [rdx+768], ymm5 +add rdx, 32 +sub rcx, 1 +jne 210b +vzeroupper +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + +/// The CPU features `vg_mldsa_dot_inverse7_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_DOT_INVERSE7_AVX2_FEATURES: crate::cpu::Features = crate::cpu::Features::of(&["avx", "avx2"]); + +/// Writes the canonical inverse NTT of the sum of coefficientwise products of 7 pairs of polynomials. Each input family is contiguous. Timing depends only on pointers, as specified by `dotInverseContract`. +/// +/// The function computes on eight coefficients at a time in AVX2 registers: it sums the products in 64-bit lanes, reduces each sum once to the sum times `2⁻³²`, then runs `vg_mldsa_montgomery_inv_ntt_avx2`'s code, which cancels the factor, with a table of the 256 zetas that it stores in `scratch`. It sets MXCSR to `0x1FBF` around its multiplications (Intel's mitigation of MXCSR-configuration-dependent timing) and loads the caller's MXCSR back before returning. +/// +/// # Safety +/// +/// * `out` must be valid for reads and writes of 1024 bytes. +/// * `a` must be valid for reads of 7168 bytes. +/// * `b` must be valid for reads of 7168 bytes. +/// * `scratch` must be valid for reads and writes of 1024 bytes. +/// * Every coefficient of both input families must be less than 25141251 (3q). +/// * `out` and `scratch` must not overlap each other, `a` or `b` (distinct Rust objects never do). +/// * None of `out`, `a`, `b` and `scratch` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_dot_inverse7_avx2(out: *mut [u32; 256], a: *const [u32; 1792], b: *const [u32; 1792], scratch: *mut [u64; 128]) { + core::arch::naked_asm!( + r"mov r10, rsi +mov rsi, rcx +stmxcsr DWORD PTR [rsi+768] +mov r11d, DWORD PTR [rsi+768] +and r11d, 65535 +mov eax, 8127 +mov DWORD PTR [rsi+772], eax +ldmxcsr DWORD PTR [rsi+772] +lfence +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov r8, rdi +mov ecx, 32 +20: +vpxor ymm5, ymm5, ymm5 +vpxor ymm6, ymm6, ymm6 +vmovdqu ymm0, YMMWORD PTR [r10] +vmovdqu ymm1, YMMWORD PTR [rdx] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+1024] +vmovdqu ymm1, YMMWORD PTR [rdx+1024] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+2048] +vmovdqu ymm1, YMMWORD PTR [rdx+2048] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+3072] +vmovdqu ymm1, YMMWORD PTR [rdx+3072] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+4096] +vmovdqu ymm1, YMMWORD PTR [rdx+4096] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+5120] +vmovdqu ymm1, YMMWORD PTR [rdx+5120] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vmovdqu ymm0, YMMWORD PTR [r10+6144] +vmovdqu ymm1, YMMWORD PTR [rdx+6144] +vpsrlq ymm2, ymm0, 32 +vpsrlq ymm3, ymm1, 32 +vpmuludq ymm2, ymm2, ymm3 +vpmuludq ymm0, ymm0, ymm1 +vpaddq ymm5, ymm5, ymm0 +vpaddq ymm6, ymm6, ymm2 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpor ymm5, ymm5, ymm6 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [r8], ymm5 +add r8, 32 +add r10, 32 +add rdx, 32 +sub rcx, 1 +jne 20b +movabs r9, 111012023893504 +mov QWORD PTR [rsi], r9 +movabs r9, 33764919763013891 +mov QWORD PTR [rsi+8], r9 +movabs r9, 32652304184483396 +mov QWORD PTR [rsi+16], r9 +movabs r9, 2003464812134697 +mov QWORD PTR [rsi+24], r9 +movabs r9, 10107995079564843 +mov QWORD PTR [rsi+32], r9 +movabs r9, 27008953388524718 +mov QWORD PTR [rsi+40], r9 +movabs r9, 23603259066260085 +mov QWORD PTR [rsi+48], r9 +movabs r9, 11510954738022985 +mov QWORD PTR [rsi+56], r9 +movabs r9, 4398527550166616 +mov QWORD PTR [rsi+64], r9 +movabs r9, 15401443493111205 +mov QWORD PTR [rsi+72], r9 +movabs r9, 31185040284548497 +mov QWORD PTR [rsi+80], r9 +movabs r9, 26937467947448680 +mov QWORD PTR [rsi+88], r9 +movabs r9, 19416172761943511 +mov QWORD PTR [rsi+96], r9 +movabs r9, 21916122901808376 +mov QWORD PTR [rsi+104], r9 +movabs r9, 35910200088776757 +mov QWORD PTR [rsi+112], r9 +movabs r9, 1202612321726977 +mov QWORD PTR [rsi+120], r9 +movabs r9, 411354790447719 +mov QWORD PTR [rsi+128], r9 +movabs r9, 15163111458665677 +mov QWORD PTR [rsi+136], r9 +movabs r9, 20565458766169348 +mov QWORD PTR [rsi+144], r9 +movabs r9, 16816682460325845 +mov QWORD PTR [rsi+152], r9 +movabs r9, 22920956268049798 +mov QWORD PTR [rsi+160], r9 +movabs r9, 23677166863944342 +mov QWORD PTR [rsi+168], r9 +movabs r9, 34703121006855168 +mov QWORD PTR [rsi+176], r9 +movabs r9, 33677345376425628 +mov QWORD PTR [rsi+184], r9 +movabs r9, 28933472397947454 +mov QWORD PTR [rsi+192], r9 +movabs r9, 19579390106369566 +mov QWORD PTR [rsi+200], r9 +movabs r9, 26799599498134591 +mov QWORD PTR [rsi+208], r9 +movabs r9, 15889643835192413 +mov QWORD PTR [rsi+216], r9 +movabs r9, 2282148053410728 +mov QWORD PTR [rsi+224], r9 +movabs r9, 16668952760323958 +mov QWORD PTR [rsi+232], r9 +movabs r9, 25011900965946676 +mov QWORD PTR [rsi+240], r9 +movabs r9, 23977247637478740 +mov QWORD PTR [rsi+248], r9 +movabs r9, 29427887555995366 +mov QWORD PTR [rsi+256], r9 +movabs r9, 22931526182748624 +mov QWORD PTR [rsi+264], r9 +movabs r9, 14929091579459166 +mov QWORD PTR [rsi+272], r9 +movabs r9, 29185144592086471 +mov QWORD PTR [rsi+280], r9 +movabs r9, 8329290213797750 +mov QWORD PTR [rsi+288], r9 +movabs r9, 31697782066745459 +mov QWORD PTR [rsi+296], r9 +movabs r9, 22432987854353025 +mov QWORD PTR [rsi+304], r9 +movabs r9, 545125843890401 +mov QWORD PTR [rsi+312], r9 +movabs r9, 31769864501989618 +mov QWORD PTR [rsi+320], r9 +movabs r9, 11662103226140216 +mov QWORD PTR [rsi+328], r9 +movabs r9, 20130185304250276 +mov QWORD PTR [rsi+336], r9 +movabs r9, 25354781094156910 +mov QWORD PTR [rsi+344], r9 +movabs r9, 30717142252233347 +mov QWORD PTR [rsi+352], r9 +movabs r9, 30375073877558844 +mov QWORD PTR [rsi+360], r9 +movabs r9, 5794237307816926 +mov QWORD PTR [rsi+368], r9 +movabs r9, 29849966874477923 +mov QWORD PTR [rsi+376], r9 +movabs r9, 1137925820308458 +mov QWORD PTR [rsi+384], r9 +movabs r9, 13305774323778583 +mov QWORD PTR [rsi+392], r9 +movabs r9, 31268732009491712 +mov QWORD PTR [rsi+400], r9 +movabs r9, 17002134848261444 +mov QWORD PTR [rsi+408], r9 +movabs r9, 35956774717033419 +mov QWORD PTR [rsi+416], r9 +movabs r9, 22036141462600008 +mov QWORD PTR [rsi+424], r9 +movabs r9, 35087477629023596 +mov QWORD PTR [rsi+432], r9 +movabs r9, 30337763489061025 +mov QWORD PTR [rsi+440], r9 +movabs r9, 20732429908239468 +mov QWORD PTR [rsi+448], r9 +movabs r9, 28041905903253186 +mov QWORD PTR [rsi+456], r9 +movabs r9, 35231517950811284 +mov QWORD PTR [rsi+464], r9 +movabs r9, 5760968484459269 +mov QWORD PTR [rsi+472], r9 +movabs r9, 29186403016613413 +mov QWORD PTR [rsi+480], r9 +movabs r9, 29809972144732485 +mov QWORD PTR [rsi+488], r9 +movabs r9, 19324591173389987 +mov QWORD PTR [rsi+496], r9 +movabs r9, 16492506917666904 +mov QWORD PTR [rsi+504], r9 +movabs r9, 14635985826474643 +mov QWORD PTR [rsi+512], r9 +movabs r9, 16398082059229396 +mov QWORD PTR [rsi+520], r9 +movabs r9, 9638297459285875 +mov QWORD PTR [rsi+528], r9 +movabs r9, 20692959164533664 +mov QWORD PTR [rsi+536], r9 +movabs r9, 10455835889373941 +mov QWORD PTR [rsi+544], r9 +movabs r9, 15088997507073265 +mov QWORD PTR [rsi+552], r9 +movabs r9, 19847271512942753 +mov QWORD PTR [rsi+560], r9 +movabs r9, 22278162875259735 +mov QWORD PTR [rsi+568], r9 +movabs r9, 7984765110959710 +mov QWORD PTR [rsi+576], r9 +movabs r9, 3517724245221606 +mov QWORD PTR [rsi+584], r9 +movabs r9, 29065087369580419 +mov QWORD PTR [rsi+592], r9 +movabs r9, 33749496538019589 +mov QWORD PTR [rsi+600], r9 +movabs r9, 22582830675910690 +mov QWORD PTR [rsi+608], r9 +movabs r9, 13774157688799364 +mov QWORD PTR [rsi+616], r9 +movabs r9, 33738338209470846 +mov QWORD PTR [rsi+624], r9 +movabs r9, 20549610337740179 +mov QWORD PTR [rsi+632], r9 +movabs r9, 1232604074686745 +mov QWORD PTR [rsi+640], r9 +movabs r9, 17645078572608834 +mov QWORD PTR [rsi+648], r9 +movabs r9, 21638646536106727 +mov QWORD PTR [rsi+656], r9 +movabs r9, 872067341384903 +mov QWORD PTR [rsi+664], r9 +movabs r9, 11559822875647717 +mov QWORD PTR [rsi+672], r9 +movabs r9, 5433172289935931 +mov QWORD PTR [rsi+680], r9 +movabs r9, 5358487101890844 +mov QWORD PTR [rsi+688], r9 +movabs r9, 6854656137752657 +mov QWORD PTR [rsi+696], r9 +movabs r9, 5370830838457625 +mov QWORD PTR [rsi+704], r9 +movabs r9, 20753904747165841 +mov QWORD PTR [rsi+712], r9 +movabs r9, 8027804982718602 +mov QWORD PTR [rsi+720], r9 +movabs r9, 31479735164668747 +mov QWORD PTR [rsi+728], r9 +movabs r9, 5314055668205759 +mov QWORD PTR [rsi+736], r9 +movabs r9, 29850847345983039 +mov QWORD PTR [rsi+744], r9 +movabs r9, 5636951310400997 +mov QWORD PTR [rsi+752], r9 +movabs r9, 27564133741392515 +mov QWORD PTR [rsi+760], r9 +movabs r9, 8233800205883732 +mov QWORD PTR [rsi+768], r9 +movabs r9, 30088883024233849 +mov QWORD PTR [rsi+776], r9 +movabs r9, 3338009929245701 +mov QWORD PTR [rsi+784], r9 +movabs r9, 14628791756398056 +mov QWORD PTR [rsi+792], r9 +movabs r9, 23830870862829877 +mov QWORD PTR [rsi+800], r9 +movabs r9, 28061014216302585 +mov QWORD PTR [rsi+808], r9 +movabs r9, 19997999096164636 +mov QWORD PTR [rsi+816], r9 +movabs r9, 19771555530215640 +mov QWORD PTR [rsi+824], r9 +movabs r9, 10447056982320729 +mov QWORD PTR [rsi+832], r9 +movabs r9, 35286145636332471 +mov QWORD PTR [rsi+840], r9 +movabs r9, 14470225858518424 +mov QWORD PTR [rsi+848], r9 +movabs r9, 30807937853347003 +mov QWORD PTR [rsi+856], r9 +movabs r9, 699409659546815 +mov QWORD PTR [rsi+864], r9 +movabs r9, 12945035726727688 +mov QWORD PTR [rsi+872], r9 +movabs r9, 7098008983067813 +mov QWORD PTR [rsi+880], r9 +movabs r9, 28266511219523944 +mov QWORD PTR [rsi+888], r9 +movabs r9, 15135022374814013 +mov QWORD PTR [rsi+896], r9 +movabs r9, 1158610381635861 +mov QWORD PTR [rsi+904], r9 +movabs r9, 31802227079365879 +mov QWORD PTR [rsi+912], r9 +movabs r9, 2027559572878823 +mov QWORD PTR [rsi+920], r9 +movabs r9, 7402805639339334 +mov QWORD PTR [rsi+928], r9 +movabs r9, 8205002449640623 +mov QWORD PTR [rsi+936], r9 +movabs r9, 31250542829661849 +mov QWORD PTR [rsi+944], r9 +movabs r9, 19527171898598875 +mov QWORD PTR [rsi+952], r9 +movabs r9, 26390134497937253 +mov QWORD PTR [rsi+960], r9 +movabs r9, 26173917256840158 +mov QWORD PTR [rsi+968], r9 +movabs r9, 31797902045269139 +mov QWORD PTR [rsi+976], r9 +movabs r9, 20765007236602922 +mov QWORD PTR [rsi+984], r9 +movabs r9, 16834811519265361 +mov QWORD PTR [rsi+992], r9 +movabs r9, 30142973844857679 +mov QWORD PTR [rsi+1000], r9 +movabs r9, 6014775284471242 +mov QWORD PTR [rsi+1008], r9 +movabs r9, 8490214048855735 +mov QWORD PTR [rsi+1016], r9 +mov eax, 8380417 +vmovq xmm15, rax +vpbroadcastd ymm15, xmm15 +mov eax, -58728449 +vmovq xmm14, rax +vpbroadcastd ymm14, xmm14 +mov eax, 1072693376 +vmovq xmm11, rax +vpbroadcastd ymm11, xmm11 +mov rdx, rdi +mov r8, rsi +add r8, 992 +mov ecx, 16 +21: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm2, YMMWORD PTR [rdx+32] +vmovdqu ymm13, YMMWORD PTR [r8] +vpermq ymm13, ymm13, 39 +vpshufd ymm13, ymm13, 177 +vpshufd ymm12, ymm13, 245 +add r8, -32 +vpshufd ymm0, ymm0, 216 +vpshufd ymm2, ymm2, 216 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm2 +vpunpckhqdq ymm1, ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpckldq ymm0, ymm0, ymm3 +vpunpckhdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 21b +mov rdx, rdi +mov r8, rsi +add r8, 496 +mov ecx, 16 +22: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 10 +vpshufd ymm13, ymm13, 95 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -16 +vmovdqa ymm2, ymm0 +vpunpcklqdq ymm0, ymm0, ymm1 +vpunpckhqdq ymm2, ymm2, ymm1 +vmovdqa ymm1, ymm2 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqa ymm1, ymm0 +vpunpcklqdq ymm0, ymm0, ymm3 +vpunpckhqdq ymm1, ymm1, ymm3 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm1 +add rdx, 64 +sub rcx, 1 +jne 22b +mov rdx, rdi +mov r8, rsi +add r8, 248 +mov ecx, 16 +23: +vmovdqu ymm4, YMMWORD PTR [rdx] +vmovdqu ymm5, YMMWORD PTR [rdx+32] +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm2, ymm13, 0 +vpshufd ymm13, ymm13, 85 +vpblendd ymm13, ymm13, ymm2, 240 +vpshufd ymm12, ymm13, 245 +add r8, -8 +vperm2i128 ymm0, ymm4, ymm5, 32 +vperm2i128 ymm1, ymm4, ymm5, 49 +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vperm2i128 ymm4, ymm0, ymm3, 32 +vperm2i128 ymm5, ymm0, ymm3, 49 +vmovdqu YMMWORD PTR [rdx], ymm4 +vmovdqu YMMWORD PTR [rdx+32], ymm5 +add rdx, 64 +sub rcx, 1 +jne 23b +mov rdx, rdi +mov r8, rsi +add r8, 124 +mov eax, 16 +24: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 1 +25: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+32] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+32], ymm3 +add rdx, 32 +sub rcx, 1 +jne 25b +add rdx, 32 +sub rax, 1 +jne 24b +mov rdx, rdi +mov r8, rsi +add r8, 60 +mov eax, 8 +26: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 2 +27: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+64] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+64], ymm3 +add rdx, 32 +sub rcx, 1 +jne 27b +add rdx, 64 +sub rax, 1 +jne 26b +mov rdx, rdi +mov r8, rsi +add r8, 28 +mov eax, 4 +28: +vbroadcasti128 ymm13, XMMWORD PTR [r8] +vpshufd ymm13, ymm13, 0 +vpshufd ymm12, ymm13, 245 +add r8, -4 +mov ecx, 4 +29: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+128] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm13 +vpmuludq ymm4, ymm4, ymm12 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+128], ymm3 +add rdx, 32 +sub rcx, 1 +jne 29b +add rdx, 128 +sub rax, 1 +jne 28b +mov rdx, rdi +mov eax, 7861508 +vmovq xmm7, rax +vpbroadcastd ymm7, xmm7 +mov eax, 5771523 +vmovq xmm8, rax +vpbroadcastd ymm8, xmm8 +mov eax, 4404704 +vmovq xmm9, rax +vpbroadcastd ymm9, xmm9 +mov eax, 41978 +vmovq xmm10, rax +vpbroadcastd ymm10, xmm10 +mov ecx, 8 +210: +vmovdqu ymm0, YMMWORD PTR [rdx] +vmovdqu ymm1, YMMWORD PTR [rdx+256] +vmovdqu ymm5, YMMWORD PTR [rdx+512] +vmovdqu ymm6, YMMWORD PTR [rdx+768] +vpsubd ymm3, ymm1, ymm0 +vpaddd ymm3, ymm3, ymm11 +vpaddd ymm0, ymm0, ymm1 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm7 +vpmuludq ymm4, ymm4, ymm7 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm1, ymm6, ymm5 +vpaddd ymm1, ymm1, ymm11 +vpaddd ymm5, ymm5, ymm6 +vpshufd ymm4, ymm1, 245 +vpmuludq ymm1, ymm1, ymm8 +vpmuludq ymm4, ymm4, ymm8 +vpmuludq ymm2, ymm1, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm1, ymm1, ymm2 +vpsrlq ymm1, ymm1, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm1, ymm1, ymm4 +vpsubd ymm6, ymm5, ymm0 +vpaddd ymm6, ymm6, ymm11 +vpaddd ymm0, ymm0, ymm5 +vpshufd ymm4, ymm6, 245 +vpmuludq ymm6, ymm6, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm6, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm6, ymm6, ymm2 +vpsrlq ymm6, ymm6, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm6, ymm6, ymm4 +vpsubd ymm5, ymm1, ymm3 +vpaddd ymm5, ymm5, ymm11 +vpaddd ymm3, ymm3, ymm1 +vpshufd ymm4, ymm5, 245 +vpmuludq ymm5, ymm5, ymm9 +vpmuludq ymm4, ymm4, ymm9 +vpmuludq ymm2, ymm5, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm5, ymm5, ymm2 +vpsrlq ymm5, ymm5, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm5, ymm5, ymm4 +vpshufd ymm4, ymm0, 245 +vpmuludq ymm0, ymm0, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm0, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm0, ymm0, ymm2 +vpsrlq ymm0, ymm0, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm0, ymm0, ymm4 +vpsubd ymm0, ymm0, ymm15 +vpsrad ymm2, ymm0, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm0, ymm0, ymm2 +vpshufd ymm4, ymm3, 245 +vpmuludq ymm3, ymm3, ymm10 +vpmuludq ymm4, ymm4, ymm10 +vpmuludq ymm2, ymm3, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm3, ymm3, ymm2 +vpsrlq ymm3, ymm3, 32 +vpmuludq ymm2, ymm4, ymm14 +vpmuludq ymm2, ymm2, ymm15 +vpaddq ymm4, ymm4, ymm2 +vpor ymm3, ymm3, ymm4 +vpsubd ymm3, ymm3, ymm15 +vpsrad ymm2, ymm3, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm3, ymm3, ymm2 +vpsubd ymm6, ymm6, ymm15 +vpsrad ymm2, ymm6, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm6, ymm6, ymm2 +vpsubd ymm5, ymm5, ymm15 +vpsrad ymm2, ymm5, 31 +vpand ymm2, ymm2, ymm15 +vpaddd ymm5, ymm5, ymm2 +vmovdqu YMMWORD PTR [rdx], ymm0 +vmovdqu YMMWORD PTR [rdx+256], ymm3 +vmovdqu YMMWORD PTR [rdx+512], ymm6 +vmovdqu YMMWORD PTR [rdx+768], ymm5 +add rdx, 32 +sub rcx, 1 +jne 210b +vzeroupper +lfence +mov DWORD PTR [rsi+768], r11d +ldmxcsr DWORD PTR [rsi+768] +ret +.p2align 6", + ) +} + /// `SimpleBitPack(f, b)` (FIPS 204 Algorithm 16): writes the 256 coefficients of `*f` to `out` as `bitlen b`-bit little-endian fields. /// /// Contract: `VG.Spec.MlDsa.simpleBitPackContract`. Constant time: only the pointers, `b` and `len` may affect timing, not the data. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 7e1aa03d0e..1d45173790 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -1531,132 +1531,36 @@ mov rsi, rbx add rsi, 38912 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 39936 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 40960 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 41984 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4_avx2} mov rdi, rbx add rdi, 23552 mov rsi, rbx add rsi, 43008 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 44032 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 45056 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 46080 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4_avx2} mov rdi, rbx add rdi, 24576 mov rsi, rbx add rsi, 47104 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 48128 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 49152 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 50176 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4_avx2} mov rdi, rbx add rdi, 25600 mov rsi, rbx add rsi, 51200 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 52224 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 53248 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 54272 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4_avx2} mov rdi, rbx add rdi, 22528 mov esi, 95232 @@ -2300,12 +2204,12 @@ ret vg_keccak_pad_scratch = sym super::sha3::vg_keccak_pad_scratch, vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, - vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, - vg_mldsa_montgomery_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_add_ntt_avx2, - vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, + vg_mldsa_dot_inverse4_avx2 = sym super::mldsa::vg_mldsa_dot_inverse4_avx2, vg_mldsa_high_bits_avx2 = sym super::mldsa::vg_mldsa_high_bits_avx2, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, + vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, @@ -4948,132 +4852,36 @@ mov rsi, rbx add rsi, 38912 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 39936 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 40960 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 41984 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4} mov rdi, rbx add rdi, 23552 mov rsi, rbx add rsi, 43008 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 44032 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 45056 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 46080 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4} mov rdi, rbx add rdi, 24576 mov rsi, rbx add rsi, 47104 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 48128 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 49152 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 50176 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4} mov rdi, rbx add rdi, 25600 mov rsi, rbx add rsi, 51200 mov rdx, rbx add rdx, 18432 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 52224 -mov rdx, rbx -add rdx, 19456 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 53248 -mov rdx, rbx -add rdx, 20480 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 54272 -mov rdx, rbx -add rdx, 21504 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 25600 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse4} mov rdi, rbx add rdi, 22528 mov esi, 95232 @@ -5717,12 +5525,12 @@ ret vg_keccak_pad_scratch = sym super::sha3::vg_keccak_pad_scratch, vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, - vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, - vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, - vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, + vg_mldsa_dot_inverse4 = sym super::mldsa::vg_mldsa_dot_inverse4, vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, + vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, vg_mldsa_add = sym super::mldsa::vg_mldsa_add, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, vg_mldsa_sub = sym super::mldsa::vg_mldsa_sub, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 8f707f596b..987b7e8581 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -2139,240 +2139,54 @@ mov rsi, rbx add rsi, 50176 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 51200 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 52224 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 53248 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 54272 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 27648 mov rsi, rbx add rsi, 55296 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 56320 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 57344 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 58368 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 59392 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 28672 mov rsi, rbx add rsi, 60416 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 61440 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 62464 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 63488 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 64512 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 29696 mov rsi, rbx add rsi, 65536 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 66560 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 67584 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 68608 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 69632 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 30720 mov rsi, rbx add rsi, 70656 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 71680 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 72704 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 73728 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 74752 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 31744 mov rsi, rbx add rsi, 75776 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 76800 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 77824 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 78848 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 79872 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5_avx2} mov rdi, rbx add rdi, 26624 mov esi, 261888 @@ -3230,12 +3044,12 @@ ret vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, - vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, - vg_mldsa_montgomery_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_add_ntt_avx2, - vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, + vg_mldsa_dot_inverse5_avx2 = sym super::mldsa::vg_mldsa_dot_inverse5_avx2, vg_mldsa_high_bits_avx2 = sym super::mldsa::vg_mldsa_high_bits_avx2, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, + vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, @@ -6853,240 +6667,54 @@ mov rsi, rbx add rsi, 50176 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 51200 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 52224 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 53248 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 54272 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 26624 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 27648 mov rsi, rbx add rsi, 55296 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 56320 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 57344 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 58368 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 59392 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 27648 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 28672 mov rsi, rbx add rsi, 60416 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 61440 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 62464 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 63488 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 64512 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 29696 mov rsi, rbx add rsi, 65536 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 66560 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 67584 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 68608 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 69632 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 30720 mov rsi, rbx add rsi, 70656 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 71680 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 72704 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 73728 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 74752 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 31744 mov rsi, rbx add rsi, 75776 mov rdx, rbx add rdx, 21504 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 76800 -mov rdx, rbx -add rdx, 22528 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 77824 -mov rdx, rbx -add rdx, 23552 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 78848 -mov rdx, rbx -add rdx, 24576 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 79872 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse5} mov rdi, rbx add rdi, 26624 mov esi, 261888 @@ -7944,12 +7572,12 @@ ret vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, vg_mldsa_expand_mask_poly = sym super::mldsa::vg_mldsa_expand_mask_poly, - vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, - vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, - vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, + vg_mldsa_dot_inverse5 = sym super::mldsa::vg_mldsa_dot_inverse5, vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, + vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, vg_mldsa_add = sym super::mldsa::vg_mldsa_add, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, vg_mldsa_sub = sym super::mldsa::vg_mldsa_sub, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 7b2b354b12..559606143a 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -3077,529 +3077,169 @@ mov rsi, rbx add rsi, 64512 mov rdx, rbx add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 65536 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 66560 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 67584 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 68608 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 69632 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 70656 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx add rdi, 33792 mov rsi, rbx add rsi, 71680 mov rdx, rbx add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 72704 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 73728 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 74752 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 75776 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 76800 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 77824 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx add rdi, 34816 mov rsi, rbx add rsi, 78848 mov rdx, rbx add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 +add rdi, 35840 mov rsi, rbx -add rsi, 79872 +add rsi, 86016 mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 +add rdi, 36864 mov rsi, rbx -add rsi, 80896 +add rsi, 93184 mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 +add rdi, 37888 mov rsi, rbx -add rsi, 81920 +add rsi, 100352 mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 +add rdi, 38912 mov rsi, rbx -add rsi, 82944 +add rsi, 107520 mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 +add rdi, 39936 mov rsi, rbx -add rsi, 83968 +add rsi, 114688 mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7_avx2} mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 84992 +add rdi, 32768 +mov esi, 261888 mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 86016 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} +add rdx, 2048 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 87040 +add rdi, 33792 +mov esi, 261888 mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 88064 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 2176 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 89088 +add rdi, 34816 +mov esi, 261888 mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 90112 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 2304 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx add rdi, 35840 -mov rsi, rbx -add rsi, 91136 +mov esi, 261888 mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 92160 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} +add rdx, 2432 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx add rdi, 36864 -mov rsi, rbx -add rsi, 93184 +mov esi, 261888 mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 94208 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 2560 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 95232 +add rdi, 37888 +mov esi, 261888 mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 96256 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 2688 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 97280 +add rdi, 38912 +mov esi, 261888 mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 98304 +add rdi, 6144 +mov esi, 15 mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} +add rdx, 2816 +mov ecx, 128 +call {vg_mldsa_simple_bit_pack} mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 99328 +add rdi, 39936 +mov esi, 261888 mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 100352 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 101376 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 102400 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 103424 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 104448 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 105472 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 106496 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 107520 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 108544 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 109568 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 110592 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 111616 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 112640 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 113664 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 38912 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 114688 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_montgomery_multiply_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 115712 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 116736 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 117760 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 118784 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 119808 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 120832 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_montgomery_multiply_add_ntt_avx2} -mov rdi, rbx -add rdi, 39936 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_montgomery_inv_ntt_avx2} -mov rdi, rbx -add rdi, 32768 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2048 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 33792 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2176 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 34816 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2304 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 35840 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2432 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 36864 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2560 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 37888 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2688 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 38912 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} -mov rdi, rbx -add rdi, 6144 -mov esi, 15 -mov rdx, rbx -add rdx, 2816 -mov ecx, 128 -call {vg_mldsa_simple_bit_pack} -mov rdi, rbx -add rdi, 39936 -mov esi, 261888 -mov rdx, rbx -add rdx, 6144 -call {vg_mldsa_high_bits_avx2} +add rdx, 6144 +call {vg_mldsa_high_bits_avx2} mov rdi, rbx add rdi, 6144 mov esi, 15 @@ -4600,12 +4240,12 @@ ret vg_keccak_pad_scratch = sym super::sha3::vg_keccak_pad_scratch, vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4_avx2 = sym super::mldsa::vg_mldsa_expand_mask_poly4_avx2, - vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, - vg_mldsa_montgomery_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_add_ntt_avx2, - vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, + vg_mldsa_dot_inverse7_avx2 = sym super::mldsa::vg_mldsa_dot_inverse7_avx2, vg_mldsa_high_bits_avx2 = sym super::mldsa::vg_mldsa_high_bits_avx2, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_montgomery_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_multiply_ntt_avx2, + vg_mldsa_montgomery_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_montgomery_inv_ntt_avx2, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, @@ -9491,617 +9131,257 @@ mov rsi, rbx add rsi, 3072 call {vg_mldsa_ntt} mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 21504 -mov ecx, 128 -215: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 215b -mov rdi, rbx -add rdi, 28672 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_ntt} -mov rdi, rbx -add rdi, 1296 -mov rsi, rbx -add rsi, 960 -mov ecx, 8 -216: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 216b -mov rax, QWORD PTR [rbx+896] -add rax, 4 -mov BYTE PTR [rbx+1360], al -shr rax, 8 -mov BYTE PTR [rbx+1361], al -mov rdi, rbx -add rdi, 1362 -mov rsi, rbx -add rsi, 960 -mov ecx, 8 -217: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 217b -mov rax, QWORD PTR [rbx+896] -add rax, 5 -mov BYTE PTR [rbx+1426], al -shr rax, 8 -mov BYTE PTR [rbx+1427], al -mov rdi, rbx -add rdi, 1428 -mov rsi, rbx -add rsi, 960 -mov ecx, 8 -218: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 218b -mov rax, QWORD PTR [rbx+896] -add rax, 6 -mov BYTE PTR [rbx+1492], al -shr rax, 8 -mov BYTE PTR [rbx+1493], al -mov rdi, rbx -add rdi, 1494 -mov rsi, rbx -add rsi, 960 -mov ecx, 8 -219: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 219b -mov rax, QWORD PTR [rbx+896] -add rax, 7 -mov BYTE PTR [rbx+1558], al -shr rax, 8 -mov BYTE PTR [rbx+1559], al -mov rdi, rbx -add rdi, 1296 -mov esi, 524288 -mov rdx, rbx -add rdx, 6144 -mov rcx, rbx -add rcx, 121856 -call {vg_mldsa_expand_mask_poly4} -mov rdi, rbx -add rdi, 22528 -mov rsi, rbx -add rsi, 6144 -mov ecx, 128 -220: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 220b -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 22528 -mov ecx, 128 -221: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 221b -mov rdi, rbx -add rdi, 29696 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_ntt} -mov rdi, rbx -add rdi, 23552 -mov rsi, rbx -add rsi, 7168 -mov ecx, 128 -222: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 222b -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 23552 -mov ecx, 128 -223: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 223b -mov rdi, rbx -add rdi, 30720 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_ntt} -mov rdi, rbx -add rdi, 24576 -mov rsi, rbx -add rsi, 8192 -mov ecx, 128 -224: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 224b -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 24576 -mov ecx, 128 -225: -mov rax, QWORD PTR [rsi] -mov QWORD PTR [rdi], rax -add rdi, 8 -add rsi, 8 -sub rcx, 1 -jne 225b -mov rdi, rbx -add rdi, 31744 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 64512 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 65536 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 66560 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 67584 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 68608 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 69632 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 70656 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 32768 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 71680 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 72704 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 73728 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 74752 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 75776 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 76800 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 77824 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 33792 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 78848 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 79872 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 80896 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 81920 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 82944 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 83968 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 84992 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 34816 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 86016 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 87040 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 88064 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 89088 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 90112 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 91136 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 92160 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 35840 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 93184 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 94208 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 95232 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 96256 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 97280 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 98304 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 99328 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} -mov rdi, rbx -add rdi, 36864 -mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} -mov rdi, rbx -add rdi, 37888 +add rdi, 28672 mov rsi, rbx -add rsi, 100352 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} +add rsi, 21504 +mov ecx, 128 +215: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 215b mov rdi, rbx -add rdi, 37888 +add rdi, 28672 mov rsi, rbx -add rsi, 101376 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} +add rsi, 3072 +call {vg_mldsa_ntt} mov rdi, rbx -add rdi, 37888 +add rdi, 1296 mov rsi, rbx -add rsi, 102400 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} +add rsi, 960 +mov ecx, 8 +216: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 216b +mov rax, QWORD PTR [rbx+896] +add rax, 4 +mov BYTE PTR [rbx+1360], al +shr rax, 8 +mov BYTE PTR [rbx+1361], al mov rdi, rbx -add rdi, 37888 +add rdi, 1362 mov rsi, rbx -add rsi, 103424 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} +add rsi, 960 +mov ecx, 8 +217: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 217b +mov rax, QWORD PTR [rbx+896] +add rax, 5 +mov BYTE PTR [rbx+1426], al +shr rax, 8 +mov BYTE PTR [rbx+1427], al mov rdi, rbx -add rdi, 37888 +add rdi, 1428 mov rsi, rbx -add rsi, 104448 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} +add rsi, 960 +mov ecx, 8 +218: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 218b +mov rax, QWORD PTR [rbx+896] +add rax, 6 +mov BYTE PTR [rbx+1492], al +shr rax, 8 +mov BYTE PTR [rbx+1493], al mov rdi, rbx -add rdi, 37888 +add rdi, 1494 mov rsi, rbx -add rsi, 105472 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} +add rsi, 960 +mov ecx, 8 +219: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 219b +mov rax, QWORD PTR [rbx+896] +add rax, 7 +mov BYTE PTR [rbx+1558], al +shr rax, 8 +mov BYTE PTR [rbx+1559], al mov rdi, rbx -add rdi, 37888 -mov rsi, rbx -add rsi, 106496 +add rdi, 1296 +mov esi, 524288 mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} +add rdx, 6144 +mov rcx, rbx +add rcx, 121856 +call {vg_mldsa_expand_mask_poly4} mov rdi, rbx -add rdi, 37888 +add rdi, 22528 mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +add rsi, 6144 +mov ecx, 128 +220: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 220b mov rdi, rbx -add rdi, 38912 +add rdi, 29696 mov rsi, rbx -add rsi, 107520 -mov rdx, rbx -add rdx, 25600 -call {vg_mldsa_multiply_ntt} +add rsi, 22528 +mov ecx, 128 +221: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 221b mov rdi, rbx -add rdi, 38912 +add rdi, 29696 mov rsi, rbx -add rsi, 108544 -mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} +add rsi, 3072 +call {vg_mldsa_ntt} mov rdi, rbx -add rdi, 38912 +add rdi, 23552 mov rsi, rbx -add rsi, 109568 -mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} +add rsi, 7168 +mov ecx, 128 +222: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 222b mov rdi, rbx -add rdi, 38912 +add rdi, 30720 mov rsi, rbx -add rsi, 110592 -mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} +add rsi, 23552 +mov ecx, 128 +223: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 223b mov rdi, rbx -add rdi, 38912 +add rdi, 30720 mov rsi, rbx -add rsi, 111616 -mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} +add rsi, 3072 +call {vg_mldsa_ntt} mov rdi, rbx -add rdi, 38912 +add rdi, 24576 mov rsi, rbx -add rsi, 112640 -mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} +add rsi, 8192 +mov ecx, 128 +224: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 224b mov rdi, rbx -add rdi, 38912 +add rdi, 31744 mov rsi, rbx -add rsi, 113664 -mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} +add rsi, 24576 +mov ecx, 128 +225: +mov rax, QWORD PTR [rsi] +mov QWORD PTR [rdi], rax +add rdi, 8 +add rsi, 8 +sub rcx, 1 +jne 225b mov rdi, rbx -add rdi, 38912 +add rdi, 31744 mov rsi, rbx add rsi, 3072 -call {vg_mldsa_inv_ntt} +call {vg_mldsa_ntt} mov rdi, rbx -add rdi, 39936 +add rdi, 32768 mov rsi, rbx -add rsi, 114688 +add rsi, 64512 mov rdx, rbx add rdx, 25600 -call {vg_mldsa_multiply_ntt} +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 33792 mov rsi, rbx -add rsi, 115712 +add rsi, 71680 mov rdx, rbx -add rdx, 26624 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 34816 mov rsi, rbx -add rsi, 116736 +add rsi, 78848 mov rdx, rbx -add rdx, 27648 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 35840 mov rsi, rbx -add rsi, 117760 +add rsi, 86016 mov rdx, rbx -add rdx, 28672 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 36864 mov rsi, rbx -add rsi, 118784 +add rsi, 93184 mov rdx, rbx -add rdx, 29696 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 37888 mov rsi, rbx -add rsi, 119808 +add rsi, 100352 mov rdx, rbx -add rdx, 30720 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx -add rdi, 39936 +add rdi, 38912 mov rsi, rbx -add rsi, 120832 +add rsi, 107520 mov rdx, rbx -add rdx, 31744 -call {vg_mldsa_multiply_add_ntt} +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx add rdi, 39936 mov rsi, rbx -add rsi, 3072 -call {vg_mldsa_inv_ntt} +add rsi, 114688 +mov rdx, rbx +add rdx, 25600 +mov rcx, rbx +add rcx, 3072 +call {vg_mldsa_dot_inverse7} mov rdi, rbx add rdi, 32768 mov esi, 261888 @@ -11199,12 +10479,12 @@ ret vg_keccak_pad_scratch = sym super::sha3::vg_keccak_pad_scratch, vg_keccak_squeeze_scratch = sym super::sha3::vg_keccak_squeeze_scratch, vg_mldsa_expand_mask_poly4 = sym super::mldsa::vg_mldsa_expand_mask_poly4, - vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, - vg_mldsa_multiply_add_ntt = sym super::mldsa::vg_mldsa_multiply_add_ntt, - vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, + vg_mldsa_dot_inverse7 = sym super::mldsa::vg_mldsa_dot_inverse7, vg_mldsa_high_bits = sym super::mldsa::vg_mldsa_high_bits, vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, + vg_mldsa_inv_ntt = sym super::mldsa::vg_mldsa_inv_ntt, vg_mldsa_add = sym super::mldsa::vg_mldsa_add, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, vg_mldsa_sub = sym super::mldsa::vg_mldsa_sub,