From 49840a56e6e0b9f48bef3d3de95ecb9639c68b16 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 21:24:06 +0000 Subject: [PATCH 1/2] ML-DSA on x86-64: the norm check and MakeHint with AVX2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vg_mldsa_norm_lt_avx2 and vg_mldsa_make_hint_avx2 compute on eight coefficients at a time, in the VEX.256 form of SSE2 code on the four doublewords of each 128-bit lane, as HighBits and LowBits do. norm_lt clamps the public bound to q (which changes no result), so that a - b and (q - b) - a fit in 32 bits and one of them is negative exactly when a coefficient is out of bounds; it ANDs their ORs into an accumulator, spreads its sign bits and returns (vpmovmskb + 1) >> 32. There is no branch on data. make_hint computes r₁ of r and of (r + z) mod q with hbX, the hint as (r₁ ^ r₁' + 63) >> 6, and counts the eight hints of an iteration from the byte mask of the hints shifted to bit 7 (vpmovmskb), summing its nibbles with three shifts and additions. Both join the polynomial arithmetic's backend, as variants of vg_mldsa_norm_lt and vg_mldsa_make_hint, so signing (both) and verification (norm_lt) with AVX2 call them. The proofs: YNorm (the differences, the mask and the result) and YHint (the hint in a doubleword, the count, the loop, and the count is hintOnes). Instructions (callgrind, AVX2), 20 deterministic signatures: ML-DSA-44 58.2M → 53.5M (−8.0%), ML-DSA-65 82.2M → 75.5M (−8.2%), ML-DSA-87 113.8M → 106.1M (−6.8%); 20 verifications: 23.5M → 23.0M (−2.0%), 35.7M → 35.4M (−0.8%), 72.0M → 70.2M (−2.4%). The baseline is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- README.md | 6 +- docs/algorithms/ml-dsa-44.toml | 2 +- docs/algorithms/ml-dsa-65.toml | 2 +- docs/algorithms/ml-dsa-87.toml | 2 +- .../Artifacts/MlDsaRound/X86_64.lean | 21 + .../Impl/MlDsa/X86_64/Arith/Avx2.lean | 2 +- .../Impl/MlDsa/X86_64/Arith/Backend.lean | 10 +- .../Impl/MlDsa/X86_64/Round/Avx2.lean | 85 ++- .../Impl/MlDsa/X86_64/Sign/Frag.lean | 4 +- .../Impl/MlDsa/X86_64/Verify/Frag.lean | 2 +- .../Proof/MlDsa/X86_64/Arith/Backend.lean | 8 + .../Proof/MlDsa/X86_64/Arith/BackendAvx2.lean | 6 +- .../Proof/MlDsa/X86_64/Round/YHint.lean | 508 ++++++++++++++++++ .../Proof/MlDsa/X86_64/Round/YNorm.lean | 403 ++++++++++++++ .../Proof/MlDsa/X86_64/Sign/Inst.lean | 8 +- .../Proof/MlDsa/X86_64/Sign/PrimsC.lean | 16 +- .../Proof/MlDsa/X86_64/Sign/Verified.lean | 8 +- .../Proof/MlDsa/X86_64/Verify/Prims.lean | 6 +- .../Variants/MlDsaArith/X86_64/Avx2.lean | 6 +- .../Variants/MlDsaArith/X86_64/Sse2.lean | 6 +- src/asm/x86_64/mldsa.rs | 243 +++++++++ src/asm/x86_64/mldsa44.rs | 46 +- src/asm/x86_64/mldsa65.rs | 62 +-- src/asm/x86_64/mldsa87.rs | 82 +-- 24 files changed, 1407 insertions(+), 137 deletions(-) create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YHint.lean create mode 100644 lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YNorm.lean diff --git a/README.md b/README.md index 6251fbe6e..3a960afd9 100644 --- a/README.md +++ b/README.md @@ -827,7 +827,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2 +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2 ✅ SHA extensions @@ -843,7 +843,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2 +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2 ✅ SHA extensions @@ -859,7 +859,7 @@ yours to keep: ✅ -✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2 +✅ AVX2; SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2 ✅ SHA extensions diff --git a/docs/algorithms/ml-dsa-44.toml b/docs/algorithms/ml-dsa-44.toml index d276b7c87..a995cbda1 100644 --- a/docs/algorithms/ml-dsa-44.toml +++ b/docs/algorithms/ml-dsa-44.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa44.rs"] asm = ["mldsa44", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2" } diff --git a/docs/algorithms/ml-dsa-65.toml b/docs/algorithms/ml-dsa-65.toml index 2c2542ec7..f9038a34b 100644 --- a/docs/algorithms/ml-dsa-65.toml +++ b/docs/algorithms/ml-dsa-65.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa65.rs"] asm = ["mldsa65", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2" } diff --git a/docs/algorithms/ml-dsa-87.toml b/docs/algorithms/ml-dsa-87.toml index b8e5b55c8..287e096e5 100644 --- a/docs/algorithms/ml-dsa-87.toml +++ b/docs/algorithms/ml-dsa-87.toml @@ -3,4 +3,4 @@ family = "Signatures" specs = ["MlDsa"] modules = ["src/mldsa87.rs"] asm = ["mldsa87", "mldsa"] -optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits and LowBits with AVX2" } +optimized = { x86_64 = "SSE2 and AVX2 polynomial arithmetic; HighBits, LowBits, MakeHint and the norm check with AVX2" } diff --git a/lean/VerifiedGarbage/Artifacts/MlDsaRound/X86_64.lean b/lean/VerifiedGarbage/Artifacts/MlDsaRound/X86_64.lean index 65293f171..69e4ffd2e 100644 --- a/lean/VerifiedGarbage/Artifacts/MlDsaRound/X86_64.lean +++ b/lean/VerifiedGarbage/Artifacts/MlDsaRound/X86_64.lean @@ -5,6 +5,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Round.NormLt import VerifiedGarbage.Proof.MlDsa.X86_64.Round.UseHint import VerifiedGarbage.Proof.MlDsa.X86_64.Round.MakeHint import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YBits +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YHint /-! # ML-DSA (FIPS 204) on x86-64: rounding and hints @@ -78,6 +79,26 @@ def artifacts : List Artifact := [ contract := Spec.MlDsa.makeHintContract X86_64.abi verified := makeHint_verified spSafe := Code.all_of_allInstrs (by decide +kernel) }, + { Spec.MlDsa.normLtApi with + name := Spec.MlDsa.normLtApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.normLtApi.doc (notes := ["The function compares eight coefficients at a time in AVX2 \ + registers; it needs AVX and AVX2."]) + code := Impl.MlDsa.X86_64.Round.normLtAvx2 + contract := Spec.MlDsa.normLtContract X86_64.abi + verified := normLtY_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) + features := ["avx", "avx2"] }, + { Spec.MlDsa.makeHintApi with + name := Spec.MlDsa.makeHintApi.name ++ "_avx2" + target := X86_64.target + doc := Spec.MlDsa.makeHintApi.doc (notes := ["The function computes eight hints at a time in AVX2 \ + registers, multiplying by shifts and additions; it needs AVX and AVX2."]) + code := Impl.MlDsa.X86_64.Round.makeHintAvx2 + contract := Spec.MlDsa.makeHintContract X86_64.abi + verified := makeHintY_verified + spSafe := Code.all_of_allInstrs (by decide +kernel) + features := ["avx", "avx2"] }, { Spec.MlDsa.useHintApi with target := X86_64.target doc := Spec.MlDsa.useHintApi.doc diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean index 38325db62..d8b3f92f7 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Avx2.lean @@ -201,6 +201,6 @@ def subAvx2 : Prog isa := /-- The AVX2 code. -/ def Backend.avx2 : Backend := ⟨nttAvx2, nttInvAvx2, mulAvx2, mulAddAvx2, addAvx2, subAvx2, Round.highBitsAvx2, - Round.lowBitsAvx2, "_avx2"⟩ + Round.lowBitsAvx2, Round.normLtAvx2, Round.makeHintAvx2, "_avx2"⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean index b56611b63..7b7af429d 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Arith/Backend.lean @@ -10,7 +10,8 @@ Key generation, signing and verification call the polynomial arithmetic of one implementation, a `Backend`: the code of `vg_mldsa_ntt`, `vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, `vg_mldsa_add` and `vg_mldsa_sub`, and of -`vg_mldsa_high_bits` and `vg_mldsa_low_bits`, whose names end with `sfx` (e.g. +`vg_mldsa_high_bits`, `vg_mldsa_low_bits`, `vg_mldsa_norm_lt` and +`vg_mldsa_make_hint`, whose names end with `sfx` (e.g. `_avx2`; nothing for the SSE2 code, `sse2`). Each is a variant of the interface `MlDsaArith` on x86-64 (`Variants/MlDsaArith/X86_64/`), and the functions that call them are emitted once for each @@ -31,18 +32,21 @@ structure Backend where sub : Prog isa highBits : Prog isa lowBits : Prog isa + normLt : Prog isa + makeHint : Prog isa /-- What the names of its functions, and of those calling them, end with. -/ sfx : String /-- The SSE2 code. -/ def Backend.sse2 : Backend := ⟨Arith.ntt, Arith.nttInv, Arith.mul, Arith.mulAdd, Arith.add, Arith.sub, Round.highBits, - Round.lowBits, ""⟩ + Round.lowBits, Round.normLt, Round.makeHint, ""⟩ /-- Every function empty, which the proofs that the functions calling a backend never write `rsp` (and load MXCSR only to restore it) evaluate in its place. -/ def Backend.empty : Backend := - ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], ""⟩ + ⟨.block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], .block [], + .block [], ""⟩ end VG.Impl.MlDsa.X86_64.Arith diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Round/Avx2.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Round/Avx2.lean index 428d9729d..79292bcbc 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Round/Avx2.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Round/Avx2.lean @@ -5,11 +5,13 @@ import VerifiedGarbage.Impl.MlKem.X86_64.Avx /-! # ML-DSA on x86-64: rounding with AVX2 -`vg_mldsa_high_bits_avx2` and `vg_mldsa_low_bits_avx2` are -`vg_mldsa_high_bits` and `vg_mldsa_low_bits` (`Round.lean`) on eight +`vg_mldsa_high_bits_avx2`, `vg_mldsa_low_bits_avx2`, +`vg_mldsa_norm_lt_avx2` and `vg_mldsa_make_hint_avx2` are +`vg_mldsa_high_bits`, `vg_mldsa_low_bits`, `vg_mldsa_norm_lt` and +`vg_mldsa_make_hint` (`Round.lean`) on eight coefficients at a time, in the doublewords of `ymm` registers: in each 128-bit lane, the VEX.256 form (`toY`) of SSE2 code on the four doublewords -of an `xmm` register (`hbX`, `lbX`), with `rdi` and `r10` at the eight +of an `xmm` register (`hbX`, `lbX`, `nlX`, `mhX`), with `rdi` and `r10` at the eight coefficients of `r` and `out` and `rcx` counting down the 32 vectors. `Decompose` is the reference implementation's, as in `Round.lean`: @@ -29,7 +31,7 @@ namespace VG.Impl.MlDsa.X86_64.Round open VG.X86_64 open VG.Impl.MlKem.X86_64 (xb xmov rcxLoop toY yconst at_) -open VG.Impl.MlDsa.X86_64.Arith (vcadd) +open VG.Impl.MlDsa.X86_64.Arith (vcadd vcsub) /-- The shifts whose sum, with 1, is `M`. -/ def dSh (g : Nat) : List Nat := if g = 261888 then [10] else [1, 3, 10, 11, 13] @@ -78,4 +80,79 @@ def highBitsAvx2 : Prog isa := bitsY hbX .xmm0 def lowBitsAvx2 : Prog isa := bitsY lbX .xmm3 +/-! ## `vg_mldsa_norm_lt_avx2` + +The bound is first clamped to `q` (a branch on the public bound), which +changes no result, as every reduced coefficient is less than `q`; then the +differences `a - b` and `(q - b) - a` fit in 32 bits, and one of them is +negative exactly when `a < b` or `q - a < b`. Each doubleword of `ymm10` +ANDs the ORs of the differences of its coefficients; its top bit stays set +while all of them are good. At the end the top bits are spread over their +doublewords (`vpsrad` by 31), and `vpmovmskb` gathers the top bits of the +32 bytes: the result is `(mask + 1) >> 32`, 1 exactly when every bit is set. -/ + +/-- `ymm10 ← ymm10 & ((a - b) | ((q - b) - a))` for `a` in `xmm0`, `b` in `xmm8`, `q - b` in `xmm9`. -/ +def nlX : List Instr := + [xmov .xmm1 .xmm0, xb .psubd .xmm1 .xmm8, xmov .xmm2 .xmm9, xb .psubd .xmm2 .xmm0, xb .por .xmm1 .xmm2, + xb .pand .xmm10 .xmm1] + +def nlBodyY : List Instr := [.vmovdquLoad .l256 .xmm0 (at_ .rdi 0)] ++ toY nlX ++ [.alu .add .rdi (.imm 32)] + +/-- The low doubleword of `rax` in each doubleword of `ymm r`. -/ +def ybcast (r : XReg) : List Instr := [.vop (.vmovq r .rax), .vop (.vpbroadcastd .l256 r r)] + +/-- `b` in `ymm8`, `q - b` in `ymm9` and all ones in `ymm10`. -/ +def nlConsts : List Instr := + [.mov32 .rax (.reg .rsi)] ++ ybcast .xmm8 ++ [.mov32 .rax (.imm qImm), .alu32 .sub .rax (.reg .rsi)] ++ + ybcast .xmm9 ++ yconst .xmm10 0xFFFFFFFF + +def nlEnd : List Instr := + toY [.xop (.shift .psrad .xmm10 31)] ++ [.vpmovmskb .l256 .rax .xmm10, .vop .vzeroupper] ++ + [.alu .add .rax (.imm 1), .shift .shr .rax 32] + +/-- The bound, clamped to `q`. -/ +def nlPro : Prog isa := + .seq (.block [.mov32 .rsi (.reg .rsi), .alu32 .cmp .rsi (.imm qImm)]) (.ite .b (.block []) (.block [.mov32 .rsi (.imm qImm)])) + +def normLtAvx2 : Prog isa := .seq nlPro (.seq (.block nlConsts) (.seq (rcxLoop 32 nlBodyY) (.block nlEnd))) + +/-! ## `vg_mldsa_make_hint_avx2` + +In each lane, `r₁` of `r` and of `(r + z) mod q` (`hbX` twice, with `vcsub` +between), XORed, is nonzero (`(x + 63) >> 6`, as `x < 64`) exactly when the +hint is 1 (`mhX`); the eight hints are stored. Their count is the sum of the +nibbles of the byte mask of the hints shifted to the top bit of their low +bytes (`vpmovmskb`, which has hint `i` at bit `4i`): three shifts and +additions put it in the low nibble (`cntH`), which is added to `r9`. -/ + +/-- `r₁` of `r` to `xmm3`, and `(r + z) mod q` to `xmm0`, with `r` in `xmm4` and `z` in `xmm5`. -/ +def mhMid : List Instr := [xmov .xmm3 .xmm0, xmov .xmm0 .xmm4, xb .paddd .xmm0 .xmm5] ++ vcsub .xmm0 .xmm1 + +/-- The hint from the two `r₁`, to `xmm0`, and shifted to bit 7, to `xmm1`. -/ +def mhTail : List Instr := + [xb .pxor .xmm0 .xmm3, xb .paddd .xmm0 .xmm11, .xop (.shift .psrld .xmm0 6), xmov .xmm1 .xmm0, + .xop (.shift .pslld .xmm1 7)] + +/-- `xmm0 ← the hints` of `r` in `xmm0` and `z` in `xmm5`, and `xmm1 ← xmm0 << 7`, with `63` in `xmm11`. -/ +def mhX (g : Nat) : List Instr := [xmov .xmm4 .xmm0] ++ hbX g ++ mhMid ++ hbX g ++ mhTail + +/-- `r9 ← r9 +` the sum of the nibbles of `eax`, through `rdx`. -/ +def cntH : List Instr := + [.mov32 .rdx (.reg .rax), .shift32 .shr .rdx 4, .alu32 .add .rax (.reg .rdx), + .mov32 .rdx (.reg .rax), .shift32 .shr .rdx 8, .alu32 .add .rax (.reg .rdx), + .mov32 .rdx (.reg .rax), .shift32 .shr .rdx 16, .alu32 .add .rax (.reg .rdx), + .alu32 .and .rax (.imm 15), .alu .add .r9 (.reg .rax)] + +/-- Eight coefficients of `r` (at `rsi`) and `z` (at `rdi`) to the hints at `r10`, and their count added to `r9`. -/ +def mhBodyY (g : Nat) : List Instr := + [.vmovdquLoad .l256 .xmm0 (at_ .rsi 0), .vmovdquLoad .l256 .xmm5 (at_ .rdi 0)] ++ toY (mhX g) ++ + [.vmovdquStore .l256 (at_ .r10 0) .xmm0, .vpmovmskb .l256 .rax .xmm1] ++ cntH ++ + [.alu .add .rdi (.imm 32), .alu .add .rsi (.imm 32), .alu .add .r10 (.imm 32)] + +def mhY (g : Nat) : Prog isa := .seq (.block (yC g ++ yconst .xmm11 63)) (rcxLoop 32 (mhBodyY g)) + +def makeHintAvx2 : Prog isa := + .seq (.block (gammaCmp .rdx ++ [.mov .r10 (.reg .rcx), .mov32 .r9 (.imm 0)])) + (.seq (.ite .e (mhY g32) (mhY g88)) (.block [.mov .rax (.reg .r9), .vop .vzeroupper])) + end VG.Impl.MlDsa.X86_64.Round diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean index 10302896a..90f290c61 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Sign/Frag.lean @@ -200,11 +200,11 @@ def lowBitsAt (r : Ptr) (gamma2 : Nat) (out : Ptr) : Prog isa := /-- `‖f‖∞ < bound`, and `r15 ← r15 ∧ result`. -/ def normAt (f : Ptr) (bound : Nat) : Prog isa := - .seq (callP "vg_mldsa_norm_lt" P.normLt [.ptr f, .imm bound]) (.block [.alu32 .and .r15 (.reg .rax)]) + .seq (callP ("vg_mldsa_norm_lt" ++ P.sfx) P.normLt [.ptr f, .imm bound]) (.block [.alu32 .and .r15 (.reg .rax)]) /-- `MakeHint` of `z` and `r` to `h`, and the number of 1s added to `ONES`. -/ def makeHintAt (z r : Ptr) (gamma2 : Nat) (h : Ptr) : Prog isa := - .seq (callP "vg_mldsa_make_hint" P.makeHint [.ptr z, .ptr r, .imm gamma2, .ptr h]) + .seq (callP ("vg_mldsa_make_hint" ++ P.sfx) P.makeHint [.ptr z, .ptr r, .imm gamma2, .ptr h]) (.block [.mov32 .rcx (.mem (at_ .rbx oONES)), .alu32 .add .rcx (.reg .rax), .store (at_ .rbx oONES) .rcx]) def simpleBitPackAt (f : Ptr) (b : Nat) (out : Ptr) (len : Nat) : Prog isa := diff --git a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean index 5b4e5e24f..a13922381 100644 --- a/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean +++ b/lean/VerifiedGarbage/Impl/MlDsa/X86_64/Verify/Frag.lean @@ -182,7 +182,7 @@ def hintUnpackAt (y : Ptr) (len omega : Nat) (h : Ptr) (hlen : Nat) : Prog isa : [(.rdi, .ptr y), (.rsi, .imm len), (.rdx, .imm omega), (.rcx, .ptr h), (.r8, .imm hlen)] def normLtAt (f : Ptr) (bound : Nat) : Prog isa := - callAt "vg_mldsa_norm_lt" P.normLt [(.rdi, .ptr f), (.rsi, .imm bound)] + callAt ("vg_mldsa_norm_lt" ++ P.sfx) P.normLt [(.rdi, .ptr f), (.rsi, .imm bound)] end diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean index 74990f6c3..db7d02bf8 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/Backend.lean @@ -5,6 +5,8 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Mul import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.AddSub import VerifiedGarbage.Proof.MlKem.X86_64.ArithOk import VerifiedGarbage.Proof.MlDsa.X86_64.Round.Bits +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.MakeHint +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.NormLt /-! # ML-DSA on x86-64: what the callers of the polynomial arithmetic need of it @@ -42,6 +44,8 @@ structure BackendOk (B : Backend) : Prop where sub : FnOk (fun S => Spec.MlDsa.subContract X86_64.abi S) B.sub highBits : FnOk (fun S => Spec.MlDsa.highBitsContract X86_64.abi S) B.highBits lowBits : FnOk (fun S => Spec.MlDsa.lowBitsContract X86_64.abi S) B.lowBits + normLt : FnOk (fun S => Spec.MlDsa.normLtContract X86_64.abi S) B.normLt + makeHint : FnOk (fun S => Spec.MlDsa.makeHintContract X86_64.abi S) B.makeHint /-- An implementation of the polynomial arithmetic on x86-64. -/ structure ArithImpl where @@ -75,6 +79,10 @@ def ArithImpl.sse2 : ArithImpl where highBits := FnOk.of Round.highBits_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) lowBits := FnOk.of Round.lowBits_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + normLt := FnOk.of Round.normLt_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + makeHint := FnOk.of Round.makeHint_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) } features := [] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean index 7dbf84d0f..96d346ca9 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Arith/BackendAvx2.lean @@ -2,7 +2,7 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YNtt import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YMul import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.YAddSub -import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YBits +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YHint /-! # ML-DSA on x86-64: the polynomial arithmetic with AVX2, as an `ArithImpl` @@ -35,6 +35,10 @@ def ArithImpl.avx2 : ArithImpl where highBits := FnOk.of Round.highBitsY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) lowBits := FnOk.of Round.lowBitsY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + normLt := FnOk.of Round.normLtY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) + (by decide +kernel) + makeHint := FnOk.of Round.makeHintY_verified (by decide +kernel) (by decide +kernel) (by decide +kernel) (by decide +kernel) } features := ["avx", "avx2"] diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YHint.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YHint.lean new file mode 100644 index 000000000..9a5d3d831 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YHint.lean @@ -0,0 +1,508 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YNorm +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.MakeHint + +/-! +# ML-DSA on x86-64: `vg_mldsa_make_hint_avx2` + +Untrusted: everything here is checked by Lean. In each doubleword, `mhX` +computes the hint bit (`mhL`, `mhL_toNat`); the loop stores the eight +hints of an iteration and adds their count, the sum of the nibbles of the +byte mask of the hints shifted to bit 7 (`nib_count`), to `r9`. +-/ + +namespace VG.Proof.MlDsa.X86_64.Round + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round VG.Proof.MlDsa.Round +open VG.Spec.MlDsa +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes ifp ifn WP.keep) +open VG.Impl.MlKem.X86_64 (xb xmov toY) +open VG.Proof.MlDsa.X86_64.Arith (qV csubL csubL_toNat dword_csubV bc) + +/-! ## A doubleword -/ + +/-- The hint bit `mhX` computes from `z` and `r`. -/ +def mhL (g : Nat) (z r : BitVec 32) : BitVec 32 := + ((hbL g (csubL (r + z)) ^^^ hbL g r) + BitVec.ofNat 32 63) >>> 6 + +theorem mhL_toNat {g : Nat} (h : g ∈ gamma2s) {z r : BitVec 32} (hz : z.toNat < q) (hr : r.toNat < q) : + (mhL g z r).toNat = (makeHint g (Fin.ofNat q z.toNat) (Fin.ofNat q r.toNat)).toNat := by + have e1 : (r + z).toNat = r.toNat + z.toNat := by + rw [BitVec.toNat_add]; rw [q_eq] at hz hr; omega + have e2 : (csubL (r + z)).toNat = (r.toNat + z.toNat) % q := by + rw [csubL_toNat (by rw [e1]; omega), e1, VG.Proof.MlDsa.Arith.condSub] + split <;> rw [q_eq] at * <;> omega + have hs : (csubL (r + z)).toNat < q := by rw [e2]; exact Nat.mod_lt _ (by decide) + have hM : hbM g ≤ 44 ∧ 0 < hbM g := by rcases mem_gamma2s h with rfl | rfl <;> decide + have hu := Nat.mod_lt (hbF g ((r.toNat + z.toNat) % q)) hM.2 + have hv := Nat.mod_lt (hbF g r.toNat) hM.2 + have hx : hbF g ((r.toNat + z.toNat) % q) % hbM g ^^^ hbF g r.toNat % hbM g < 2 ^ 6 := + Nat.xor_lt_two_pow (by omega) (by omega) + have hvz : (Fin.ofNat q z.toNat).val = z.toNat := Nat.mod_eq_of_lt hz + have hvr : (Fin.ofNat q r.toNat).val = r.toNat := Nat.mod_eq_of_lt hr + rw [makeHint_eq h, hvz, hvr, mhL, BitVec.toNat_ushiftRight, BitVec.toNat_add, BitVec.toNat_xor, + hbL_toNat h hs, hbL_toNat h hr, e2, BitVec.toNat_ofNat, Nat.shiftRight_eq_div_pow] + by_cases e : hbF g r.toNat % hbM g = hbF g ((r.toNat + z.toNat) % q) % hbM g + · rw [decide_eq_false (fun h' => h' e), e, Nat.xor_self]; rfl + · rw [decide_eq_true e] + have : hbF g ((r.toNat + z.toNat) % q) % hbM g ^^^ hbF g r.toNat % hbM g ≠ 0 := + fun h' => e (xor_eq_zero h').symm + show _ = 1 + omega + +theorem mhL_le {g : Nat} (h : g ∈ gamma2s) {z r : BitVec 32} (hz : z.toNat < q) (hr : r.toNat < q) : + (mhL g z r).toNat ≤ 1 := by + rw [mhL_toNat h hz hr]; exact Bool.toNat_le _ + +/-! ## The code on a register -/ + +theorem mhMid_ok (s : State) (hq : s.xmm .xmm15 = qV) : + WP isa (.block mhMid) s fun s' => s'.xmm .xmm3 = s.xmm .xmm0 ∧ + (∀ e < 4, dword (s'.xmm .xmm0) e = csubL (dword (s.xmm .xmm4) e + dword (s.xmm .xmm5) e)) ∧ + XOnly [.xmm0, .xmm1, .xmm3] s s' := by + simp only [mhMid, VG.Impl.MlDsa.X86_64.Arith.vcsub, VG.Impl.MlDsa.X86_64.Arith.vcadd, xmov, xb, List.cons_append, + List.nil_append] + vrun [VG.X86_64.eval_movdqa] + refine ⟨trivial, fun e he => ?_, by xonly⟩ + rw [hq, ← dword_paddd _ _ he, ← dword_csubV _ he] + rfl + +theorem mhTail_ok (s : State) (h11 : Bc (s.xmm .xmm11) (BitVec.ofNat 32 63)) : + WP isa (.block mhTail) s fun s' => (∀ e < 4, + dword (s'.xmm .xmm0) e = ((dword (s.xmm .xmm0) e ^^^ dword (s.xmm .xmm3) e) + BitVec.ofNat 32 63) >>> 6 ∧ + dword (s'.xmm .xmm1) e = (((dword (s.xmm .xmm0) e ^^^ dword (s.xmm .xmm3) e) + BitVec.ofNat 32 63) >>> 6) <<< 7) ∧ + XOnly [.xmm0, .xmm1] s s' := by + simp only [mhTail, xmov, xb] + vrun [VG.X86_64.eval_movdqa] + refine ⟨fun e he => ?_, by xonly⟩ + simp (disch := first | decide | assumption) only [dword_pxor, dword_paddd, dword_psrld, dword_pslld, h11 e he] + exact ⟨rfl, rfl⟩ + +theorem mhX_ok {g : Nat} (hg : g = g32 ∨ g = g88) (s : State) (hc : HbC g s) (hq : s.xmm .xmm15 = qV) + (h11 : Bc (s.xmm .xmm11) (BitVec.ofNat 32 63)) : + WP isa (.block (mhX g)) s fun s' => (∀ e < 4, + dword (s'.xmm .xmm0) e = mhL g (dword (s.xmm .xmm5) e) (dword (s.xmm .xmm0) e) ∧ + dword (s'.xmm .xmm1) e = mhL g (dword (s.xmm .xmm5) e) (dword (s.xmm .xmm0) e) <<< 7) ∧ + XOnly [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4] s s' := by + rw [mhX, List.append_assoc, List.append_assoc, List.append_assoc, WP.block_append_iff] + refine WP.mono (Q := fun (s1 : State) => s1.xmm .xmm4 = s.xmm .xmm0 ∧ XOnly [.xmm4] s s1) + (by simp only [xmov, xb]; vrun [VG.X86_64.eval_movdqa]; exact ⟨trivial, by xonly⟩) fun s1 ⟨a1, o1⟩ => ?_ + have hc1 : HbC g s1 := ⟨by rw [o1.xmm _ (by decide)]; exact hc.c8, by rw [o1.xmm _ (by decide)]; exact hc.c9, + by rw [o1.xmm _ (by decide)]; exact hc.c10⟩ + rw [WP.block_append_iff] + refine WP.mono (hbX_ok hg s1 hc1) fun s2 ⟨a2, o2⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (mhMid_ok s2 (by rw [o2.xmm _ (by decide), o1.xmm _ (by decide), hq])) fun s3 ⟨b3, a3, o3⟩ => ?_ + have hc3 : HbC g s3 := ⟨by rw [o3.xmm _ (by decide), o2.xmm _ (by decide)]; exact hc1.c8, + by rw [o3.xmm _ (by decide), o2.xmm _ (by decide)]; exact hc1.c9, + by rw [o3.xmm _ (by decide), o2.xmm _ (by decide)]; exact hc1.c10⟩ + rw [WP.block_append_iff] + refine WP.mono (hbX_ok hg s3 hc3) fun s4 ⟨a4, o4⟩ => ?_ + refine WP.mono (mhTail_ok s4 (by + rw [o4.xmm _ (by decide), o3.xmm _ (by decide), o2.xmm _ (by decide), o1.xmm _ (by decide)]; exact h11)) + fun s5 ⟨a5, o5⟩ => ⟨fun e he => ?_, ?_⟩ + · have hz : dword (s4.xmm .xmm0) e = hbL g (csubL (dword (s.xmm .xmm0) e + dword (s.xmm .xmm5) e)) := by + rw [a4 e he, a3 e he, o2.xmm _ (by decide), a1, o2.xmm _ (by decide), o1.xmm _ (by decide)] + have hr1 : dword (s4.xmm .xmm3) e = hbL g (dword (s.xmm .xmm0) e) := by + rw [o4.xmm _ (by decide), b3, a2 e he, o1.xmm _ (by decide)] + rw [(a5 e he).1, (a5 e he).2, hz, hr1] + exact ⟨rfl, rfl⟩ + · exact ((((o1.trans o2).trans o3).trans o4).trans o5).mono (by simp) + +/-! ## The count -/ + +/-- What `cntH` leaves in `eax`: the sum of the nibbles of `x`, if it fits in one. -/ +def nib (x : BitVec 32) : BitVec 32 := + let x1 := x + x >>> 4 + let x2 := x1 + x1 >>> 8 + (x2 + x2 >>> 16) &&& 15 + +/-- The eight bits `b d` at bits `4d`. -/ +def sp (b : Nat → Bool) : Nat → Nat + | 0 => 0 + | k + 1 => sp b k + (b k).toNat * 16 ^ k + +/-- The number of the first `k` bits set. -/ +def cnt8 (b : Nat → Bool) : Nat → Nat + | 0 => 0 + | k + 1 => cnt8 b k + (b k).toNat + +theorem nib_bools : ∀ b0 b1 b2 b3 b4 b5 b6 b7 : Bool, + (nib (BitVec.ofNat 32 (b0.toNat + b1.toNat * 16 + b2.toNat * 16 ^ 2 + b3.toNat * 16 ^ 3 + b4.toNat * 16 ^ 4 + + b5.toNat * 16 ^ 5 + b6.toNat * 16 ^ 6 + b7.toNat * 16 ^ 7))).toNat = + b0.toNat + b1.toNat + b2.toNat + b3.toNat + b4.toNat + b5.toNat + b6.toNat + b7.toNat := by + decide +kernel + +theorem nib_sp (b : Nat → Bool) : (nib (BitVec.ofNat 32 (sp b 8))).toNat = cnt8 b 8 := by + have := nib_bools (b 0) (b 1) (b 2) (b 3) (b 4) (b 5) (b 6) (b 7) + simp only [sp, cnt8, Nat.pow_zero, Nat.mul_one, Nat.zero_add, Nat.pow_one] at this ⊢ + exact this + +/-- A byte mask with bit `4d` the bit `b d`, and the others clear. -/ +theorem bsum_sp {f : Nat → Bool} {b : Nat → Bool} : ∀ k, (∀ j < 4 * k, f j = (decide (j % 4 = 0) && b (j / 4))) → + VG.Proof.MlKem.X86_64.S4.bsum f (4 * k) = sp b k + | 0, _ => rfl + | k + 1, h => by + have ih := bsum_sp k fun j hj => h j (by omega) + rw [show 4 * (k + 1) = 4 * k + 1 + 1 + 1 + 1 by omega] + simp only [VG.Proof.MlKem.X86_64.S4.bsum, sp] + rw [ih, h _ (by omega), h _ (by omega), h _ (by omega), h _ (by omega)] + simp only [show (4 * k) % 4 = 0 by omega, show (4 * k + 1) % 4 = 1 by omega, show (4 * k + 2) % 4 = 2 by omega, + show (4 * k + 1 + 1 + 1) % 4 = 3 by omega, show 4 * k / 4 = k by omega, + decide_true, decide_false, Bool.true_and, Bool.false_and, Bool.toNat_false, Nat.zero_mul, Nat.add_zero, + show (1 : Nat) ≠ 0 by decide, show (2 : Nat) ≠ 0 by decide, show (3 : Nat) ≠ 0 by decide] + rw [show 2 ^ (4 * k) = 16 ^ k by rw [Nat.pow_mul]] + +theorem cntH_ok (s : State) : + WP isa (.block cntH) s fun s' => + (s'.gpr .r9 = s.gpr .r9 + BitVec.setWidth 64 (nib ((s.gpr .rax).setWidth 32)) ∧ s'.mem = s.mem ∧ + ∀ r l, s'.lane r l = s.lane r l) ∧ Keep [.rax, .rdx, .r9] s s' := by + refine WP.keep _ ?_ (by decide) + simp only [cntH] + xrun + exact ⟨rfl, fun _ _ => rfl⟩ + +end VG.Proof.MlDsa.X86_64.Round + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round +open VG.Proof.MlDsa.Arith +open VG.Proof.MlDsa.X86_64.Round (HbC Bc bc_ofDwords mhL mhL_le mhX_ok nib sp cnt8 nib_sp bsum_sp ymm_bit cntH_ok + sw3264) +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok yconst_ok wp_rcxLoopY ifp ifn ptr_step add_ofNat_zero + lane_setReg lane_setFlags sx32 State.setMem_ymm) +open VG.Impl.MlKem.X86_64 (toY) +open VG.Spec.MlDsa (q n coeffAt Reduced gamma2s) + +/-- `Σ k < m, f k`. -/ +def csum (f : Nat → Nat) : Nat → Nat + | 0 => 0 + | m + 1 => csum f m + f m + +namespace YHintL + +/-- After `i` vectors of eight. -/ +structure Inv (s₀ : State) (g : Nat) (i : Nat) (s : State) : Prop where + rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (32 * i) + rsi : s.gpr .rsi = s₀.gpr .rsi + BitVec.ofNat 64 (32 * i) + r10 : s.gpr .r10 = s₀.gpr .rcx + BitVec.ofNat 64 (32 * i) + rd : s.rd = s₀.rd + wr : s.wr = s₀.wr + yc : YC g s + c11 : ∀ l < 2, s.lane .xmm11 l = bc (BitVec.ofNat 32 63) + frame : Frame [pR (s₀.gpr .rcx)] s₀.mem s.mem + coeff : ∀ k < 256, coeffAt s.mem (s₀.gpr .rcx) k = if k < 8 * i then + mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k) else coeffAt s₀.mem (s₀.gpr .rcx) k + r9 : (s.gpr .r9).toNat = + csum (fun k => (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat) (8 * i) + +theorem lane_mhX (g : Nat) (hg : g = g32 ∨ g = g88) : laneSseBlock (toY (mhX g)) = some (mhX g) := by + rcases hg with rfl | rfl <;> decide +kernel + +/-- The constants are kept by code that writes only `xmm0` to `xmm5`. -/ +theorem keep_consts {g : Nat} {s s' : State} (hyc : YC g s) (h11 : ∀ l < 2, s.lane .xmm11 l = bc (BitVec.ofNat 32 63)) + {rs : List XReg} (hr : ∀ r ∈ rs, r ∈ [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4, .xmm5]) + (hl : ∀ r ∉ rs, ∀ l < 2, s'.lane r l = s.lane r l) : + YC g s' ∧ ∀ l < 2, s'.lane .xmm11 l = bc (BitVec.ofNat 32 63) := by + have n : ∀ r ∈ [XReg.xmm8, .xmm9, .xmm10, .xmm11, .xmm15], r ∉ rs := fun r hr' hr'' => by + have := hr r hr''; simp only [List.mem_cons, List.not_mem_nil, or_false] at hr' this + rcases hr' with rfl | rfl | rfl | rfl | rfl <;> rcases this with h | h | h | h | h | h <;> cases h + refine ⟨fun l hl' => ?_, fun l hl' => by rw [hl _ (n _ (by simp)) l hl']; exact h11 l hl'⟩ + rw [hl _ (n _ (by simp)) l hl', hl _ (n _ (by simp)) l hl', hl _ (n _ (by simp)) l hl', hl _ (n _ (by simp)) l hl'] + exact hyc l hl' + +/-- A hint shifted to bit 7: bit `8m + 7` is the hint's bit 0 if `m = 0`, and clear otherwise. -/ +theorem shl7_bit {v : BitVec 32} (hv : v.toNat ≤ 1) {m : Nat} (hm : m < 4) : + (v <<< 7).getLsbD (8 * m + 7) = (decide (m = 0) && v.getLsbD 0) := by + rw [BitVec.getLsbD_shiftLeft] + rcases (by omega : m = 0 ∨ 0 < m) with rfl | h + · simp + · have : v.getLsbD (8 * m) = false := by + rw [← BitVec.testBit_toNat, Nat.testBit_lt_two_pow (Nat.lt_of_le_of_lt hv + (Nat.one_lt_two_pow (by omega)))] + simp [show 8 * m + 7 - 7 = 8 * m by omega, this, show m ≠ 0 by omega] + +theorem bit0_toNat {v : BitVec 32} (hv : v.toNat ≤ 1) : (v.getLsbD 0).toNat = v.toNat := by + rw [← BitVec.testBit_toNat, Nat.testBit_zero] + rcases (by omega : v.toNat = 0 ∨ v.toNat = 1) with h | h <;> rw [h] <;> rfl + +theorem csum_le {f : Nat → Nat} : ∀ {m : Nat}, (∀ k < m, f k ≤ 1) → csum f m ≤ m + | 0, _ => Nat.le_refl _ + | m + 1, h => by + have := csum_le (m := m) fun k hk => h k (by omega) + have := h m (by omega) + simp only [csum]; omega + +theorem csum8 (f : Nat → Nat) (b : Nat → Bool) (m : Nat) (h : ∀ d < 8, (b d).toNat = f (m + d)) : + csum f (m + 8) = csum f m + cnt8 b 8 := by + rw [show csum f (m + 8) = csum f m + f m + f (m + 1) + f (m + 2) + f (m + 3) + f (m + 4) + f (m + 5) + + f (m + 6) + f (m + 7) from rfl, + show cnt8 b 8 = (b 0).toNat + (b 1).toNat + (b 2).toNat + (b 3).toNat + (b 4).toNat + (b 5).toNat + + (b 6).toNat + (b 7).toNat by simp [cnt8], + h 0 (by decide), h 1 (by decide), h 2 (by decide), h 3 (by decide), h 4 (by decide), h 5 (by decide), + h 6 (by decide), h 7 (by decide)] + simp only [Nat.add_zero] + omega + +section +variable {s₀ : State} (hrd : s₀.rd = [pR (s₀.gpr .rdi), pR (s₀.gpr .rsi)]) (hwr : s₀.wr = [pR (s₀.gpr .rcx)]) + (hdz : (pR (s₀.gpr .rdi)).Disjoint (pR (s₀.gpr .rcx))) (hdr : (pR (s₀.gpr .rsi)).Disjoint (pR (s₀.gpr .rcx))) + (hz : Reduced s₀.mem (s₀.gpr .rdi)) (hr : Reduced s₀.mem (s₀.gpr .rsi)) {g : Nat} (hg : g = g32 ∨ g = g88) +include hrd hwr hdz hdr hz hr hg + +theorem step {i : Nat} (hi : i < 32) {s : State} (hI : Inv s₀ g i s) : + WP isa (.block (mhBodyY g ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + Inv s₀ g (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have hg' : g ∈ gamma2s := by rcases hg with rfl | rfl <;> decide + have j0 : 8 * i + 8 ≤ 256 := by omega + have hw : pR (s₀.gpr .rcx) ∈ s.wr := by rw [hI.wr, hwr]; simp + have hrz : pR (s₀.gpr .rdi) ∈ s.rd ++ s.wr := by rw [hI.rd, hrd]; simp + have hrr : pR (s₀.gpr .rsi) ∈ s.rd ++ s.wr := by rw [hI.rd, hrd]; simp + have e1 : s.gpr .rsi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rsi) (8 * i) := by + rw [add_ofNat_zero, hI.rsi]; congr 2; omega + have e2 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rdi) (8 * i) := by + rw [add_ofNat_zero, hI.rdi]; congr 2; omega + have e3 : s.gpr .r10 = coeffAddr (s₀.gpr .rcx) (8 * i) := by rw [hI.r10]; congr 2; omega + rw [mhBodyY, List.append_assoc, List.append_assoc, List.append_assoc, List.append_assoc, + show ∀ a b : Instr, [a, b] = [a] ++ [b] from fun _ _ => rfl, List.append_assoc, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [e1]; exact f_in32 hrr j0)) fun s1 ⟨L1, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (yld_ok (by rw [o1.rd, o1.wr, o1.gpr, e2]; exact f_in32 hrz j0)) fun s2 ⟨L2, o2⟩ => ?_ + have k2 := keep_consts hI.yc hI.c11 (rs := [.xmm0, .xmm5]) (by simp) fun r hr' l hl => by + simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr' + rw [o2.lane r (by simp [hr'.2]) l hl, o1.lane r (by simp [hr'.1]) l hl] + rw [WP.block_append_iff] + refine WP.mono (ylanes (lane_mhX g hg) (P := fun l t => ∀ e < 4, + dword (t.xmm .xmm0) e = mhL g (dword ((s2.proj l).xmm .xmm5) e) (dword ((s2.proj l).xmm .xmm0) e) ∧ + dword (t.xmm .xmm1) e = mhL g (dword ((s2.proj l).xmm .xmm5) e) (dword ((s2.proj l).xmm .xmm0) e) <<< 7) + fun l hl => mhX_ok hg _ (k2.1.hbc hl) (k2.1.q hl) + (by rw [State.proj_xmm, k2.2 l hl]; exact bc_ofDwords _)) fun s3 ⟨B3, o3⟩ => ?_ + have o13 := (o1.trans o2).trans o3 + have k3 := keep_consts k2.1 k2.2 (rs := [.xmm0, .xmm1, .xmm2, .xmm3, .xmm4]) (by simp) o3.lane + have g3 : s3.gpr .r10 = coeffAddr (s₀.gpr .rcx) (8 * i) := by rw [o13.gpr, e3] + have w0 : InRegions s3.wr (s3.gpr .r10) 32 := by rw [o13.wr, g3]; exact f_in32 hw j0 + -- The values of the lanes. + have hv : ∀ l < 2, ∀ e < 4, mhL g (dword ((s2.proj l).xmm .xmm5) e) (dword ((s2.proj l).xmm .xmm0) e) = + mhL g (coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + 4 * l + e)) (coeffAt s₀.mem (s₀.gpr .rsi) (8 * i + 4 * l + e)) := + fun l hl e he => by + rw [State.proj_xmm, State.proj_xmm, L2 l hl, o2.lane _ (by decide) l hl, L1 l hl, o1.mem, o1.gpr, e1, e2, + dword_readW _ _ he, dword_readW _ _ he, lane_load, lane_load, coeffAddr_add, coeffAddr_add, ← coeffAt_eq, + ← coeffAt_eq, coeffAt_frame hI.frame (by simpa using hdz) (by rw [n_eq]; omega), + coeffAt_frame hI.frame (by simpa using hdr) (by rw [n_eq]; omega)] + have hle : ∀ k < 256, (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat ≤ 1 := + fun k hk => mhL_le hg' (hz k (by rw [n_eq]; exact hk)) (hr k (by rw [n_eq]; exact hk)) + rw [WP.block_append_iff] + refine WP.mono (Q := fun (s4 : State) => s4.mem = s3.mem.writeW (s3.gpr .r10) (s3.ymm .xmm0) ∧ + s4.gpr = (s3.setReg .rax (byteMask (s3.ymm .xmm1) 32)).gpr ∧ s4.rd = s3.rd ∧ s4.wr = s3.wr ∧ + ∀ r l, s4.lane r l = s3.lane r l) + (by + vrund [State.store256_eq, State.setMem_gpr, State.setMem_wr, State.setMem_mem, State.setMem_rd, + State.setMem_ymm, w0] + exact ⟨rfl, fun r l => by simp only [lane_setReg, State.setMem_lane]⟩) fun s4 ⟨m4, g4, r4, w4, l4⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (cntH_ok s4) fun s5 ⟨⟨h9, m5, l5⟩, k5⟩ => ?_ + have hax : s4.gpr .rax = byteMask (s3.ymm .xmm1) 32 := by rw [g4, RegUpd.gpr_setReg_self] + -- The count of the eight hints. + let b : Nat → Bool := fun d => + (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + d)) (coeffAt s₀.mem (s₀.gpr .rsi) (8 * i + d))).getLsbD 0 + have hbits : ∀ j < 4 * 8, (s3.ymm .xmm1).getLsbD (8 * j + 7) = (decide (j % 4 = 0) && b (j / 4)) := fun j hj => by + have hl : j / 16 < 2 := by omega + have he : j % 16 / 4 < 4 := by omega + rw [ymm_bit _ _ (by omega), ← State.proj_xmm, (B3 _ hl _ he).2, hv _ hl _ he, + shl7_bit (hle _ (by omega)) (by omega)] + simp only [b, show 8 * i + 4 * (j / 16) + j % 16 / 4 = 8 * i + j / 4 by omega] + have hcnt : (BitVec.setWidth 64 (nib ((s4.gpr .rax).setWidth 32))).toNat = cnt8 b 8 := by + rw [hax, VG.Proof.MlKem.X86_64.S4.byteMask_eq _ (by decide), BitVec.toNat_setWidth, + show BitVec.setWidth 32 (BitVec.ofNat 64 (VG.Proof.MlKem.X86_64.S4.bsum + (fun i => (s3.ymm .xmm1).getLsbD (8 * i + 7)) 32)) = + BitVec.ofNat 32 (VG.Proof.MlKem.X86_64.S4.bsum (fun i => (s3.ymm .xmm1).getLsbD (8 * i + 7)) 32) by + apply BitVec.eq_of_toNat_eq + have := VG.Proof.MlKem.X86_64.S4.bsum_lt (fun i => (s3.ymm .xmm1).getLsbD (8 * i + 7)) 32 + simp only [BitVec.toNat_setWidth, BitVec.toNat_ofNat]; omega, + (bsum_sp 8 hbits : VG.Proof.MlKem.X86_64.S4.bsum _ 32 = _), nib_sp] + have : cnt8 b 8 ≤ 8 := by + simp only [cnt8] + have hb := fun d => Bool.toNat_le (b d) + have := hb 0; have := hb 1; have := hb 2; have := hb 3; have := hb 4; have := hb 5 + have := hb 6; have := hb 7 + omega + have := (nib_sp b).symm ▸ this + omega + simp only [List.cons_append, List.nil_append] + xrun + have G5 : ∀ r, r ∉ [Reg.rax, .rdx, .r9] → s5.gpr r = s.gpr r := fun r hr => by + rw [k5.gpr hr, g4, RegUpd.gpr_setReg_of_ne _ _ (fun h => hr (by simp [h])), o13.gpr] + refine ⟨⟨?_, ?_, ?_, ?_, ?_, ?_, ?_, ?_, fun k hk => ?_, ?_⟩, by rw [G5 .rcx (by simp)], by rw [G5 .rcx (by simp)]⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [G5 .rdi (by simp), hI.rdi]; exact ptr_step _ i 32 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [G5 .rsi (by simp), hI.rsi]; exact ptr_step _ i 32 + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [G5 .r10 (by simp), hI.r10]; exact ptr_step _ i 32 + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags]; rw [k5.2.1, r4, o13.rd, hI.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags]; rw [k5.2.2, w4, o13.wr, hI.wr] + · exact k3.1.keep (rs := []) (by simp) fun r _ l _ => by simp only [lane_setReg, lane_setFlags]; rw [l5, l4] + · intro l hl; simp only [lane_setReg, lane_setFlags]; rw [l5, l4]; exact k3.2 l hl + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags] + rw [m5, m4, g3, o13.mem]; exact hI.frame.writeW (List.mem_singleton_self _) _ (pR_contains32 _ j0) + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags] + rw [m5, m4, g3, o13.mem, coeffAt_write256 _ _ j0 _ hk] + split + · rename_i h + rw [ifp (show k < 8 * (i + 1) by omega), State.ymm, extract_ymm _ _ (by omega)] + have hc : ∀ l < 2, ∀ e < 4, dword (s3.lane .xmm0 l) e = + mhL g (coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + 4 * l + e)) (coeffAt s₀.mem (s₀.gpr .rsi) (8 * i + 4 * l + e)) := + fun l hl e he => by rw [← State.proj_xmm, (B3 l hl e he).1, hv l hl e he] + split + · rename_i h4 + have := hc 0 (by decide) (k - 8 * i) h4 + rw [show 8 * i + 4 * 0 + (k - 8 * i) = k by omega] at this + exact this + · rename_i h4 + have := hc 1 (by decide) (k - 8 * i - 4) (by omega) + rw [show 8 * i + 4 * 1 + (k - 8 * i - 4) = k by omega] at this + exact this + · rename_i h + rw [hI.coeff k hk] + by_cases h' : k < 8 * i + · rw [ifp h', ifp (by omega)] + · rw [ifn h', ifn (by omega)] + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_false, reduceCtorEq] + have h94 : s4.gpr .r9 = s.gpr .r9 := by + rw [g4, RegUpd.gpr_setReg_of_ne _ _ (by decide), o13.gpr] + have e8 := csum8 (fun k => (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat) b + (8 * i) fun d hd => bit0_toNat (hle _ (by omega)) + have hb := csum_le (f := fun k => (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat) + (m := 8 * i + 8) fun k hk => hle k (by omega) + rw [h9, BitVec.toNat_add, hcnt, h94, hI.r9, show 8 * (i + 1) = 8 * i + 8 by omega, e8] + rw [e8] at hb + exact Nat.mod_eq_of_lt (by omega) + +/-- The constants and the loop: the hints at `h`, and their count in `r9`. -/ +theorem loop_ok {s : State} (hs0 : s.gpr .rdi = s₀.gpr .rdi) (hs1 : s.gpr .rsi = s₀.gpr .rsi) + (hs10 : s.gpr .r10 = s₀.gpr .rcx) (hs9 : s.gpr .r9 = 0) (hsrd : s.rd = s₀.rd) (hswr : s.wr = s₀.wr) + (hsm : s.mem = s₀.mem) : + WP isa (mhY g) s fun s' => Frame [pR (s₀.gpr .rcx)] s₀.mem s'.mem ∧ + (∀ k < 256, coeffAt s'.mem (s₀.gpr .rcx) k = + mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) ∧ + (s'.gpr .r9).toNat = + csum (fun k => (mhL g (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat) 256 := by + unfold mhY + refine WP.seq ?_ + rw [WP.block_append_iff] + refine WP.mono (yC_ok g s) fun w ⟨yc, k1, m1, _, _⟩ => + WP.mono (yconst_ok .xmm11 63 w) fun w2 ⟨l2, k2, m2, _, o2⟩ => ?_ + have yc2 : YC g w2 := fun l hl => by + rw [o2 .xmm8 (by decide) l hl, o2 .xmm9 (by decide) l hl, o2 .xmm10 (by decide) l hl, + o2 .xmm15 (by decide) l hl] + exact yc l hl + refine WP.mono (wp_rcxLoopY (N := 32) (by decide) (by decide) _ (fun u o hy _ => + ⟨by rw [o.keep.gpr (by decide), k2.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero, hs0], + by rw [o.keep.gpr (by decide), k2.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero, hs1], + by rw [o.keep.gpr (by decide), k2.gpr (by decide), k1.gpr (by decide), Nat.mul_zero, add_ofNat_zero, hs10], + by rw [o.keep.2.1, k2.2.1, k1.2.1, hsrd], by rw [o.keep.2.2, k2.2.2, k1.2.2, hswr], + fun l hl => by simp only [State.lane]; rw [o.xmm, hy]; exact yc2 l hl, + fun l hl => by simp only [State.lane]; rw [o.xmm, hy]; exact l2 l hl, + by rw [o.mem, m2, m1, hsm]; exact Frame.refl _ _, fun k _ => by rw [o.mem, m2, m1, hsm, ifn (by omega)], + by rw [o.keep.gpr (by decide), k2.gpr (by decide), k1.gpr (by decide), hs9]; rfl⟩) + fun i hi u hI => step hrd hwr hdz hdr hz hr hg hi hI) fun u hI => ⟨hI.frame, fun k hk => by + rw [hI.coeff k hk, ifp (by omega)], hI.r9⟩ + +end + +end YHintL + +end VG.Proof.MlDsa.X86_64.Arith + +namespace VG.Proof.MlDsa.X86_64.Round + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round VG.Proof.MlDsa.Round +open VG.Spec.MlDsa +open VG.Proof.MlKem.X86_64 (Keep Keep.gpr WP.keep gprPreserved_of) +open VG.Proof.MlDsa.X86_64.Arith (csum) + +/-- The count of the hints is `onesFrom` them. -/ +theorem csum_onesFrom (v : Vector Bool n) (f : Nat → Nat) (hf : ∀ k < 256, f k = v[k]!.toNat) : + ∀ m ≤ 256, csum f m + onesFrom v m = onesFrom v 0 + | 0, _ => Nat.zero_add _ + | m + 1, hm => by + have ih := csum_onesFrom v f hf m (by omega) + rw [onesFrom_step v (by rw [n_eq]; omega), ← hf m (by omega)] at ih + simp only [csum]; omega + +section +variable {s₀ : State} (hp : makeHintK.pre s₀) +include hp + +theorem makeHintY_correct : ∃ t s', Exec isa makeHintAvx2 s₀ t s' ∧ abiPreserved s₀ s' ∧ makeHintK.post s₀ s' := by + have hg : arg32 s₀ .rdx ∈ gamma2s := hp.2.2.2.2.2.2.2.1 + have hz : Reduced s₀.mem (s₀.gpr .rdi) := hp.2.2.2.2.2.2.2.2.1 + have hr : Reduced s₀.mem (s₀.gpr .rsi) := hp.2.2.2.2.2.2.2.2.2 + let f : Nat → Nat := fun k => + (mhL (arg32 s₀ .rdx) (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)).toNat + have wp : WP isa makeHintAvx2 s₀ fun s' => Frame [pR (s₀.gpr .rcx)] s₀.mem s'.mem ∧ + (∀ k < 256, coeffAt s'.mem (s₀.gpr .rcx) k = + mhL (arg32 s₀ .rdx) (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) ∧ + (s'.gpr .rax).toNat = csum f 256 := by + unfold makeHintAvx2 + refine WP.seq (WP.mono (mhPrologue_ok s₀) fun s₁ ⟨⟨h10, h9, hzf, hm⟩, hk⟩ => ?_) + have go : ∀ g, arg32 s₀ .rdx = g → (g = g32 ∨ g = g88) → WP isa (mhY g) s₁ fun s' => + Frame [pR (s₀.gpr .rcx)] s₀.mem s'.mem ∧ + (∀ k < 256, coeffAt s'.mem (s₀.gpr .rcx) k = + mhL (arg32 s₀ .rdx) (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) ∧ + (s'.gpr .r9).toNat = csum f 256 := fun g hge hg' => by + subst hge + exact Arith.YHintL.loop_ok hp.1 hp.2.1 hp.2.2.1 hp.2.2.2.1 hz hr hg' (hk.gpr (by decide)) + (hk.gpr (by decide)) h10 h9 hk.2.1 hk.2.2 hm + have hite : WP isa (.ite .e (mhY g32) (mhY g88)) s₁ fun s' => Frame [pR (s₀.gpr .rcx)] s₀.mem s'.mem ∧ + (∀ k < 256, coeffAt s'.mem (s₀.gpr .rcx) k = + mhL (arg32 s₀ .rdx) (coeffAt s₀.mem (s₀.gpr .rdi) k) (coeffAt s₀.mem (s₀.gpr .rsi) k)) ∧ + (s'.gpr .r9).toNat = csum f 256 := by + refine WP.ite (M := isa) _ (show isa.eval .e s₁ = _ from hzf) (fun h => ?_) (fun h => ?_) + · rw [sub_beq_zero32, decide_eq_true_eq] at h + exact go _ ((gamma_cases hg).1 h) (.inl rfl) + · rw [sub_beq_zero32, decide_eq_false_iff_not] at h + exact go _ ((gamma_cases hg).2 h) (.inr rfl) + refine WP.seq (WP.mono hite fun u ⟨hf, hc, h9'⟩ => ?_) + refine WP.mono (Q := fun (u' : State) => u'.mem = u.mem ∧ u'.gpr .rax = u.gpr .r9) (by vrund; exact ⟨rfl, rfl⟩) + fun u' ⟨hm', hax⟩ => ?_ + rw [hm', hax] + exact ⟨hf, hc, h9'⟩ + obtain ⟨t, s', he, ⟨hf, hv, hax⟩, hk⟩ := WP.keep [.rax, .rcx, .rdx, .rsi, .rdi, .r9, .r10] wp (by decide +kernel) + refine ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he (gprPreserved_of hk (by decide) hf ?_), ?_, ?_⟩ + · simpa using hp.2.2.2.2.2.2.1 + · refine hintIs_of_toNat fun k hk' => ?_ + apply BitVec.eq_of_toNat_eq + rw [hv k hk', mhL_toNat hg (hz k hk') (hr k hk'), zipWith_get _ _ _ hk', polyAt_get _ _ hk', polyAt_get _ _ hk'] + simp only [BitVec.natCast_eq_ofNat, BitVec.toNat_ofNat] + exact (Nat.mod_eq_of_lt (Nat.lt_of_le_of_lt (Bool.toNat_le _) (by decide))).symm + · have e := csum_onesFrom (Vector.zipWith (makeHint (arg32 s₀ .rdx)) (polyAt s₀.mem (s₀.gpr .rdi)) + (polyAt s₀.mem (s₀.gpr .rsi))) f (fun k hk' => by + rw [zipWith_get _ _ _ (by rw [n_eq]; exact hk'), polyAt_get _ _ (by rw [n_eq]; exact hk'), + polyAt_get _ _ (by rw [n_eq]; exact hk')] + exact mhL_toNat hg (hz k (by rw [n_eq]; exact hk')) (hr k (by rw [n_eq]; exact hk'))) 256 (Nat.le_refl _) + have e0 : onesFrom (Vector.zipWith (makeHint (arg32 s₀ .rdx)) (polyAt s₀.mem (s₀.gpr .rdi)) + (polyAt s₀.mem (s₀.gpr .rsi))) 256 = 0 := onesFrom_n _ + have : (s'.gpr .rax).toNat ≤ 256 := by + rw [hax] + exact Arith.YHintL.csum_le fun k hk' => mhL_le hg (hz k (by rw [n_eq]; exact hk')) (hr k (by rw [n_eq]; exact hk')) + rw [BitVec.toNat_setWidth, hintOnes_single] + omega + +end + +theorem makeHintY_ct : ConstantTime isa makeHintK.pre makeHintK.pub makeHintAvx2 := + VG.Taint.constantTime (A := X86_64.taint) (regsLo [.rdi, .rsi, .rcx, .rsp] [.rdx]) + (fun _ _ _ _ hp => agree_regsLo (fun r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl | rfl | rfl + exacts [hp.1, hp.2.1, hp.2.2.1, hp.2.2.2.1]) fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact hp.2.2.2.2) + (by taint_decide) + +theorem makeHintY_verified : Verified X86_64.target makeHintAvx2 (makeHintContract X86_64.abi) := + Verified.of_correct (fun _ hp => makeHintY_correct hp) makeHintY_ct (by + round_implies [makeHintContract, makeHintSig, makeHintK, hintK, X86_64.abi, X86_64.argRegs] [hintSat] + using hintSat) + +end VG.Proof.MlDsa.X86_64.Round diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YNorm.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YNorm.lean new file mode 100644 index 000000000..a94ca1e18 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Round/YNorm.lean @@ -0,0 +1,403 @@ +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.YBits +import VerifiedGarbage.Proof.MlDsa.X86_64.Round.NormLt +import VerifiedGarbage.Proof.MlKem.X86_64.S4Vec + +/-! +# ML-DSA on x86-64: `vg_mldsa_norm_lt_avx2` + +Untrusted: everything here is checked by Lean. With the bound clamped to +`b ≤ q` (which changes no result, `good_clamp`), each doubleword of `ymm10` +keeps its top bit while every coefficient it has seen is good +(`Good b a`: `a < b` or `q - a < b`, `nlL_msb`); at the end the top bits +of the 32 bytes are all set exactly when every coefficient is good +(`allOnes_bsum`). +-/ + +namespace VG.Proof.MlDsa.X86_64.Round + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round VG.Proof.MlDsa.Round +open VG.Spec.MlDsa +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes ifp ifn) +open VG.Impl.MlKem.X86_64 (xb xmov toY) +open VG.Proof.MlDsa.X86_64.Arith (dword_psubd dword_pand dword_psrad) +open VG.Proof.MlKem.X86_64.S4 (bsum bsum_lt byteMask_eq) +open VG.Proof.MlDsa.X86_64.Arith (bc) + +/-! ## A doubleword -/ + +/-- The top bit of a difference of doublewords below `2³¹`: whether it borrows. -/ +theorem msb_sub32 {u v : BitVec 32} (hu : u.toNat < 2 ^ 31) (hv : v.toNat < 2 ^ 31) : + (u - v).msb = decide (u.toNat < v.toNat) := by + rw [BitVec.msb_eq_decide, BitVec.toNat_sub] + by_cases h : u.toNat < v.toNat + · rw [decide_eq_true h, decide_eq_true_iff]; omega + · rw [decide_eq_false h, decide_eq_false_iff_not]; omega + +/-- What `nlX` ORs into a doubleword: `(a - b) | ((q - b) - a)`. -/ +def nlL (a b c : BitVec 32) : BitVec 32 := (a - b) ||| (c - a) + +theorem nlL_msb {a b : BitVec 32} (ha : a.toNat < q) (hb : b.toNat ≤ q) : + (nlL a b (BitVec.ofNat 32 (q - b.toNat))).msb = decide (Good b.toNat a.toNat) := by + rw [q_eq] at ha hb + have hc : (BitVec.ofNat 32 (q - b.toNat)).toNat = q - b.toNat := by rw [BitVec.toNat_ofNat, q_eq]; omega + rw [nlL, BitVec.msb_or, msb_sub32 (by omega) (by omega), msb_sub32 (by rw [hc, q_eq]; omega) (by omega), hc] + simp only [Good, Bool.decide_or, q_eq] + congr 1 + apply Bool.eq_iff_iff.mpr + simp only [decide_eq_true_iff] + omega + +/-- Clamping the bound to `q` changes no coefficient's goodness. -/ +theorem good_clamp {B a : Nat} (ha : a < q) : Good (min B q) a ↔ Good B a := by + simp only [Good] + constructor + · rintro (h | h) <;> [left; right] <;> omega + · intro h + by_cases hB : B ≤ q + · rw [Nat.min_eq_left hB]; exact h + · rw [Nat.min_eq_right (by omega)]; left; exact ha + +/-! ## The mask -/ + +theorem bsum_allOnes (f : Nat → Bool) : ∀ n, bsum f n = 2 ^ n - 1 ↔ ∀ i < n, f i = true + | 0 => by simp [bsum] + | n + 1 => by + have hl := bsum_lt f n + have ih := bsum_allOnes f n + have h1 : 1 ≤ 2 ^ n := Nat.one_le_two_pow + simp only [bsum, Nat.pow_succ] + generalize 2 ^ n = N at hl ih h1 ⊢ + constructor + · intro h i hi + have hfn : f n = true := by + cases e : f n + · simp only [e, Bool.toNat_false, Nat.zero_mul, Nat.add_zero] at h; omega + · rfl + rw [hfn] at h + simp only [Bool.toNat_true, Nat.one_mul] at h + rcases (by omega : i < n ∨ i = n) with hi | rfl + · exact (ih.mp (by omega)) i hi + · exact hfn + · intro h + rw [ih.mpr fun i hi => h i (by omega), h n (by omega)] + simp only [Bool.toNat_true, Nat.one_mul] + omega + +/-- A bit of a 256-bit register, in its lanes and doublewords. -/ +theorem ymm_bit (s : State) (r : XReg) {i : Nat} (hi : i < 32) : + (s.ymm r).getLsbD (8 * i + 7) = (dword (s.lane r (i / 16)) (i % 16 / 4)).getLsbD (8 * (i % 4) + 7) := by + simp only [State.ymm, State.lane, dword, BitVec.getLsbD_append, BitVec.getLsbD_extractLsb', + show 8 * (i % 4) + 7 < 32 by omega, decide_true, Bool.true_and] + by_cases h : i < 16 + · rw [ite_eq_left (show 8 * i + 7 < 128 by omega), ite_eq_left (show i / 16 = 0 by omega), + show 32 * (i % 16 / 4) + (8 * (i % 4) + 7) = 8 * i + 7 by omega] + · rw [ite_eq_right (show ¬ 8 * i + 7 < 128 by omega), ite_eq_right (show ¬ i / 16 = 0 by omega), + show 32 * (i % 16 / 4) + (8 * (i % 4) + 7) = 8 * i + 7 - 128 by omega] + +/-- A doubleword spread from its top bit. -/ +theorem bit_of_spread {d : BitVec 32} (h : d = 0 ∨ d = BitVec.allOnes 32) {j : Nat} (hj : j < 32) : + d.getLsbD j = d.msb := by + rcases h with rfl | rfl + · simp + · rw [BitVec.msb_allOnes (by decide), BitVec.getLsbD_allOnes]; simp [hj] + +/-! ## The code on a register -/ + +theorem nlX_ok (s : State) : + WP isa (.block nlX) s fun s' => (∀ e < 4, dword (s'.xmm .xmm10) e = + dword (s.xmm .xmm10) e &&& nlL (dword (s.xmm .xmm0) e) (dword (s.xmm .xmm8) e) (dword (s.xmm .xmm9) e)) ∧ + XOnly [.xmm1, .xmm2, .xmm10] s s' := by + simp only [nlX, xmov, xb] + vrun [VG.X86_64.eval_movdqa] + refine ⟨fun e he => ?_, by xonly⟩ + simp only [dword_pand, dword_por, dword_psubd _ _ he] + rfl + +theorem lane_nlX : laneSseBlock (toY nlX) = some nlX := by decide +kernel + +/-- `rax`'s low doubleword in each doubleword of `ymm r`. -/ +theorem ybc_ok (r : XReg) (s : State) : + WP isa (.block (ybcast r)) s fun s' => + (∀ l < 2, s'.lane r l = bc ((s.gpr .rax).setWidth 32)) ∧ s'.gpr = s.gpr ∧ s'.mem = s.mem ∧ s'.rd = s.rd ∧ + s'.wr = s.wr ∧ s'.mxcsr = s.mxcsr ∧ ∀ r' ≠ r, ∀ l < 2, s'.lane r' l = s.lane r' l := by + apply WP.of_runBlock + simp only [ybcast, runBlock_cons, runStep_some, runBlock_nil, exec, VOp.exec, Option.some.injEq, + exists_eq_left'] + refine ⟨fun l hl => ?_, rfl, rfl, rfl, rfl, rfl, fun r' hr' l hl => ?_⟩ + · rw [State.lane_setV256, ifp rfl] + have : dword ((s.setV .l128 r ((0 : BitVec 64) ++ s.gpr .rax) 0).xmm r) 0 = (s.gpr .rax).setWidth 32 := by + rw [RegUpd.xmm_setV, ifp rfl] + apply BitVec.eq_of_getLsbD_eq; intro i hi + simp only [dword, BitVec.getLsbD_extractLsb', hi, decide_true, Bool.true_and] + rw [BitVec.getLsbD_append, ifp (by omega), BitVec.getLsbD_setWidth] + simp [show i < 64 by omega, hi] + rcases lane01 hl with rfl | rfl <;> simp only [ite_true, ite_false, this, Nat.one_ne_zero] <;> rfl + · rw [State.lane_setV256, ifn hr', State.lane_setV128, ifn hr'] + +end VG.Proof.MlDsa.X86_64.Round + +namespace VG.Proof.MlDsa.X86_64.Arith + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round +open VG.Proof.MlDsa.Arith +open VG.Proof.MlDsa.X86_64.Round (Good nlL nlL_msb nlX_ok lane_nlX bc_ofDwords) +open VG.Proof.MlKem.X86_64 (Keep XOnly YOnly ylanes yld_ok ifp ifn ptr_step add_ofNat_zero lane_setReg lane_setFlags) +open VG.Spec.MlDsa (q n coeffAt Reduced) + +namespace YNormL + +/-- After `i` vectors of eight: the top bit of doubleword `e` of lane `l` of +`ymm10` is whether coefficients `8j + 4l + e`, `j < i`, are good. -/ +structure Inv (s₀ : State) (b : BitVec 32) (i : Nat) (s : State) : Prop where + rdi : s.gpr .rdi = s₀.gpr .rdi + BitVec.ofNat 64 (32 * i) + rd : s.rd = s₀.rd + wr : s.wr = s₀.wr + mem : s.mem = s₀.mem + c8 : ∀ l < 2, s.lane .xmm8 l = bc b + c9 : ∀ l < 2, s.lane .xmm9 l = bc (BitVec.ofNat 32 (q - b.toNat)) + acc : ∀ l < 2, ∀ e < 4, (dword (s.lane .xmm10 l) e).msb = true ↔ + ∀ j < i, Good b.toNat (coeffAt s₀.mem (s₀.gpr .rdi) (8 * j + 4 * l + e)).toNat + +theorem step {s₀ : State} (hrd : pR (s₀.gpr .rdi) ∈ s₀.rd) (hr : Reduced s₀.mem (s₀.gpr .rdi)) {b : BitVec 32} + (hb : b.toNat ≤ q) {i : Nat} (hi : i < 32) {s : State} (hI : Inv s₀ b i s) : + WP isa (.block (nlBodyY ++ ([.alu .sub .rcx (.imm 1)] : List Instr))) s fun s' => + Inv s₀ b (i + 1) s' ∧ s'.gpr .rcx = s.gpr .rcx - 1 ∧ s'.zf = some (s.gpr .rcx - 1 == 0) := by + have j0 : 8 * i + 8 ≤ 256 := by omega + have hr' : pR (s₀.gpr .rdi) ∈ s.rd ++ s.wr := by rw [hI.rd]; exact List.mem_append_left _ hrd + have e1 : s.gpr .rdi + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .rdi) (8 * i) := by + rw [add_ofNat_zero, hI.rdi]; congr 2; omega + rw [nlBodyY, List.append_assoc, List.append_assoc, WP.block_append_iff] + refine WP.mono (yld_ok (by rw [e1]; exact f_in32 hr' j0)) fun s1 ⟨L1, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (ylanes lane_nlX (P := fun l t => ∀ e < 4, dword (t.xmm .xmm10) e = + dword ((s1.proj l).xmm .xmm10) e &&& nlL (dword ((s1.proj l).xmm .xmm0) e) (dword ((s1.proj l).xmm .xmm8) e) + (dword ((s1.proj l).xmm .xmm9) e)) fun l hl => nlX_ok _) fun s3 ⟨B3, o3⟩ => ?_ + have o13 := o1.trans o3 + vrund + refine ⟨⟨?_, ?_, ?_, ?_, fun l hl => ?_, fun l hl => ?_, fun l hl e he => ?_⟩, ?_⟩ + · simp only [RegUpd.gpr_setReg, RegUpd.gpr_setFlags, ite_true, ite_false, reduceCtorEq] + rw [o13.gpr, hI.rdi]; exact ptr_step _ i 32 + · simp only [RegUpd.rd_setReg, RegUpd.rd_setFlags]; rw [o13.rd, hI.rd] + · simp only [RegUpd.wr_setReg, RegUpd.wr_setFlags]; rw [o13.wr, hI.wr] + · simp only [RegUpd.mem_setReg, RegUpd.mem_setFlags]; rw [o13.mem, hI.mem] + · simp only [lane_setReg, lane_setFlags]; rw [o13.lane _ (by decide) l hl, hI.c8 l hl] + · simp only [lane_setReg, lane_setFlags]; rw [o13.lane _ (by decide) l hl, hI.c9 l hl] + · simp only [lane_setReg, lane_setFlags] + have hx : dword ((s1.proj l).xmm .xmm0) e = coeffAt s₀.mem (s₀.gpr .rdi) (8 * i + 4 * l + e) := by + rw [State.proj_xmm, L1 l hl, e1, dword_readW _ _ he, lane_load, coeffAddr_add, ← coeffAt_eq, hI.mem] + have h8 : dword ((s1.proj l).xmm .xmm8) e = b := by + rw [State.proj_xmm, o1.lane _ (by decide) l hl, hI.c8 l hl]; exact bc_ofDwords _ e he + have h9 : dword ((s1.proj l).xmm .xmm9) e = BitVec.ofNat 32 (q - b.toNat) := by + rw [State.proj_xmm, o1.lane _ (by decide) l hl, hI.c9 l hl]; exact bc_ofDwords _ e he + have h10 : (s1.proj l).xmm .xmm10 = s.lane .xmm10 l := by rw [State.proj_xmm, o1.lane _ (by decide) l hl] + rw [← State.proj_xmm, B3 l hl e he, BitVec.msb_and, Bool.and_eq_true, hx, h8, h9, h10, + nlL_msb (by rw [← VG.Proof.MlDsa.Round.n_eq] at *; exact hr _ (by rw [VG.Proof.MlDsa.Round.n_eq]; omega)) hb, + decide_eq_true_iff, hI.acc l hl e he] + refine ⟨fun ⟨h₁, h₂⟩ j hj => ?_, fun h => ⟨fun j hj => h j (by omega), h i (by omega)⟩⟩ + rcases (by omega : j < i ∨ j = i) with hj | rfl + exacts [h₁ j hj, h₂] + · exact ⟨by rw [o13.gpr], by rw [o13.gpr]⟩ + +end YNormL + +end VG.Proof.MlDsa.X86_64.Arith + +namespace VG.Proof.MlDsa.X86_64.Round + +open VG VG.X86_64 VG.Impl.MlDsa.X86_64.Round VG.Proof.MlDsa.Round +open VG.Spec.MlDsa +open VG.Proof.MlKem.X86_64 (Keep Keep.gpr WP.keep gprPreserved_of XOnly YOnly ylanes yconst_ok ifp ifn) +open VG.Proof.MlDsa.X86_64.Arith (bc dword_psrad sshiftRight31) +open VG.Impl.MlKem.X86_64 (toY) +open VG.Proof.MlKem.X86_64.S4 (bsum byteMask_eq) + +theorem sw3264 (y : BitVec 32) : BitVec.setWidth 32 (BitVec.setWidth 64 y) = y := by + apply BitVec.eq_of_toNat_eq; rw [BitVec.toNat_setWidth, BitVec.toNat_setWidth]; have := y.isLt; omega + +/-- The bound, clamped to `q`. -/ +theorem nlPro_ok (s₀ : State) : + WP isa nlPro s₀ fun s₂ => (s₂.mem = s₀.mem ∧ ((s₂.gpr .rsi).setWidth 32).toNat = min (arg32 s₀ .rsi) q) ∧ + Keep [.rsi] s₀ s₂ := by + unfold nlPro + refine WP.seq (WP.mono (Q := fun (s₁ : State) => (s₁.cf = some (decide (((s₀.gpr .rsi).setWidth 32).toNat < q)) ∧ + s₁.mem = s₀.mem ∧ s₁.gpr .rsi = BitVec.setWidth 64 ((s₀.gpr .rsi).setWidth 32)) ∧ Keep [.rsi] s₀ s₁) + (by refine WP.keep _ ?_ (by decide); xrun; rfl) + fun s₁ ⟨⟨hc, hm, hsi⟩, hk⟩ => ?_) + refine WP.ite (M := isa) _ (show isa.eval .b s₁ = _ from hc) (fun h => ?_) (fun h => ?_) + · rw [decide_eq_true_iff] at h + refine WP.mono (Q := fun s₂ => s₂ = s₁) (by vrund) fun s₂ e => ?_ + subst e + refine ⟨⟨hm, ?_⟩, hk⟩ + rw [hsi, sw3264, Nat.min_eq_left (by unfold arg32; omega)] + · rw [decide_eq_false_iff_not] at h + refine WP.mono (Q := fun (s₂ : State) => (s₂.gpr .rsi = BitVec.setWidth 64 qImm ∧ s₂.mem = s₁.mem) ∧ + Keep [.rsi] s₁ s₂) (by refine WP.keep _ ?_ (by decide); xrun) fun s₂ ⟨⟨h1, h2⟩, k2⟩ => ?_ + refine ⟨⟨h2.trans hm, ?_⟩, (hk.trans k2).mono (by simp)⟩ + rw [h1, Nat.min_eq_right (by unfold arg32; omega)]; rfl + +theorem qsub_eq {x : BitVec 32} (hx : x.toNat ≤ q) : qImm - x = BitVec.ofNat 32 (q - x.toNat) := by + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_sub, BitVec.toNat_ofNat, show qImm.toNat = q from rfl] + rw [q_eq] at hx ⊢; omega + +/-- The constants of the loop: `b`, `q - b` and all ones. -/ +theorem nlConsts_ok (s : State) (hb : ((s.gpr .rsi).setWidth 32).toNat ≤ q) : + WP isa (.block nlConsts) s fun s' => + (∀ l < 2, s'.lane .xmm8 l = bc ((s.gpr .rsi).setWidth 32) ∧ + s'.lane .xmm9 l = bc (BitVec.ofNat 32 (q - ((s.gpr .rsi).setWidth 32).toNat)) ∧ + s'.lane .xmm10 l = bc (BitVec.allOnes 32)) ∧ Keep [.rax] s s' ∧ s'.mem = s.mem ∧ s'.mxcsr = s.mxcsr := by + simp only [nlConsts, List.append_assoc] + rw [WP.block_append_iff] + refine WP.mono (Q := fun (s1 : State) => (s1.gpr .rax = BitVec.setWidth 64 ((s.gpr .rsi).setWidth 32) ∧ + s1.mem = s.mem ∧ s1.mxcsr = s.mxcsr ∧ ∀ r l, s1.lane r l = s.lane r l) ∧ Keep [.rax] s s1) + (by refine WP.keep _ ?_ (by decide); xrun; exact ⟨rfl, fun _ _ => rfl⟩) fun s1 ⟨⟨a1, m1, x1, l1⟩, k1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (ybc_ok .xmm8 s1) fun s2 ⟨c2, g2, m2, r2, w2, x2, o2⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (Q := fun (s3 : State) => (s3.gpr .rax = BitVec.setWidth 64 (qImm - (s2.gpr .rsi).setWidth 32) ∧ + s3.mem = s2.mem ∧ s3.mxcsr = s2.mxcsr ∧ ∀ r l, s3.lane r l = s2.lane r l) ∧ Keep [.rax] s2 s3) + (by refine WP.keep _ ?_ (by decide); xrun; exact ⟨rfl, fun _ _ => rfl⟩) fun s3 ⟨⟨a3, m3, x3, l3⟩, k3⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (ybc_ok .xmm9 s3) fun s4 ⟨c4, g4, m4, r4, w4, x4, o4⟩ => ?_ + refine WP.mono (yconst_ok .xmm10 _ s4) fun s5 ⟨c5, k5, m5, x5, o5⟩ => ?_ + have e2 : s2.gpr .rsi = s.gpr .rsi := by rw [g2, k1.gpr (by decide)] + refine ⟨fun l hl => ⟨?_, ?_, ?_⟩, ?_, by rw [m5, m4, m3, m2, m1], by rw [x5, x4, x3, x2, x1]⟩ + · rw [o5 _ (by decide) l hl, o4 _ (by decide) l hl, l3, c2 l hl, a1, sw3264] + · rw [o5 _ (by decide) l hl, c4 l hl, a3, e2, sw3264, qsub_eq hb] + · rw [c5 l hl]; rfl + · refine ⟨fun r hr => ?_, ?_, ?_⟩ + · rw [k5.gpr hr, g4, k3.gpr hr, g2, k1.gpr hr] + · rw [k5.2.1, r4, k3.2.1, r2, k1.2.1] + · rw [k5.2.2, w4, k3.2.2, w2, k1.2.2] + +theorem lane_psrad : laneSseBlock (toY [.xop (.shift .psrad .xmm10 31)]) = some [.xop (.shift .psrad .xmm10 31)] := by + decide +kernel + +/-- Whether every doubleword of `ymm10` has its top bit set. -/ +def allMsb (s : State) : Bool := (List.range 2).all fun l => (List.range 4).all fun e => (dword (s.lane .xmm10 l) e).msb + +/-- The end: 1 if every doubleword of `ymm10` has its top bit set, and 0 otherwise. -/ +theorem nlEnd_ok (s : State) : + WP isa (.block nlEnd) s fun s' => + ((s'.gpr .rax).setWidth 32 = if allMsb s = true then 1 else 0) ∧ + s'.mem = s.mem ∧ s'.rd = s.rd ∧ s'.wr = s.wr ∧ ∀ r, r ≠ .rax → s'.gpr r = s.gpr r := by + rw [nlEnd, List.append_assoc, WP.block_append_iff] + refine WP.mono (ylanes lane_psrad (rs := [.xmm10]) (P := fun l t => ∀ e < 4, + dword (t.xmm .xmm10) e = (dword ((s.proj l).xmm .xmm10) e).sshiftRight (min (31 : BitVec 8).toNat 32)) + fun l hl => by vrun; exact ⟨fun e he => dword_psrad _ _ he, by xonly⟩) fun s1 ⟨B1, o1⟩ => ?_ + rw [WP.block_append_iff] + refine WP.mono (Q := fun (s2 : State) => s2.gpr = (s1.setReg .rax (byteMask (s1.ymm .xmm10) 32)).gpr ∧ + s2.mem = s1.mem ∧ s2.rd = s1.rd ∧ s2.wr = s1.wr) (by vrund; exact ⟨rfl, rfl, rfl, rfl⟩) fun s2 ⟨g2, m2, r2, w2⟩ => ?_ + xrun + have hf : ∀ i < 32, (s1.ymm .xmm10).getLsbD (8 * i + 7) = (dword (s.lane .xmm10 (i / 16)) (i % 16 / 4)).msb := + fun i hi => by + have hl : i / 16 < 2 := by omega + have he : i % 16 / 4 < 4 := by omega + have hd := B1 (i / 16) hl (i % 16 / 4) he + rw [State.proj_xmm, State.proj_xmm, sshiftRight31] at hd + rw [ymm_bit _ _ hi, hd] + split + · rename_i h + rw [BitVec.msb_eq_decide]; simp [h] + · rename_i h + rw [show (-1 : BitVec 32) = BitVec.allOnes 32 by decide, BitVec.getLsbD_allOnes, BitVec.msb_eq_decide] + simp only [decide_eq_true (show 8 * (i % 4) + 7 < 32 by omega)] + rw [decide_eq_true (by omega)] + have hall : allMsb s = true ↔ ∀ i < 32, (s1.ymm .xmm10).getLsbD (8 * i + 7) = true := by + simp only [allMsb, List.all_eq_true, List.mem_range] + constructor + · intro h i hi; rw [hf i hi]; exact h _ (by omega) _ (by omega) + · intro h l hl e he + have := h (16 * l + 4 * e) (by omega) + rwa [hf _ (by omega), show (16 * l + 4 * e) / 16 = l by omega, show (16 * l + 4 * e) % 16 / 4 = e by omega] + at this + have hb := VG.Proof.MlKem.X86_64.S4.bsum_lt (fun i => (s1.ymm .xmm10).getLsbD (8 * i + 7)) 32 + have hax : s2.gpr .rax = BitVec.ofNat 64 (bsum (fun i => (s1.ymm .xmm10).getLsbD (8 * i + 7)) 32) := by + rw [g2, RegUpd.gpr_setReg_self, byteMask_eq _ (by decide)] + refine ⟨?_, by rw [m2, o1.mem], by rw [r2, o1.rd], by rw [w2, o1.wr], fun r hr => ?_⟩ + · rw [hax] + apply BitVec.eq_of_toNat_eq + rw [BitVec.toNat_setWidth, BitVec.toNat_ushiftRight, BitVec.toNat_add, BitVec.toNat_ofNat, + Nat.shiftRight_eq_div_pow] + have e1 : (1 : BitVec 64).toNat = 1 := rfl + rw [e1] + by_cases h : allMsb s = true + · rw [ite_eq_left h, (bsum_allOnes _ 32).mpr (hall.mp h)]; rfl + · rw [ite_eq_right h] + have : bsum (fun i => (s1.ymm .xmm10).getLsbD (8 * i + 7)) 32 ≠ 2 ^ 32 - 1 := fun e => + h (hall.mpr ((bsum_allOnes _ 32).mp e)) + show _ = 0 + omega + · rw [ite_eq_right hr, ite_eq_right hr, g2, RegUpd.gpr_setReg_of_ne _ _ hr, o1.gpr] + +section +variable {s₀ : State} (hp : normLtK.pre s₀) +include hp + +theorem normLtY_wp : WP isa normLtAvx2 s₀ fun s' => + ((s'.gpr .rax).setWidth 32 = if normRq [polyAt s₀.mem (s₀.gpr .rdi)] < arg32 s₀ .rsi then 1 else 0) ∧ + Frame [] s₀.mem s'.mem := by + have hr : Reduced s₀.mem (s₀.gpr .rdi) := hp.2.2.2 + have hrd : pR (s₀.gpr .rdi) ∈ s₀.rd := by rw [hp.1]; simp + unfold normLtAvx2 + refine WP.seq (WP.mono (nlPro_ok s₀) fun s₂ ⟨⟨hm2, hb2⟩, k2⟩ => ?_) + have hbq : ((s₂.gpr .rsi).setWidth 32).toNat ≤ q := by rw [hb2]; exact Nat.min_le_right _ _ + refine WP.seq (WP.mono (nlConsts_ok s₂ hbq) fun s₃ ⟨hc, k3, m3, _⟩ => ?_) + refine WP.seq (WP.mono (VG.Proof.MlKem.X86_64.wp_rcxLoopY (N := 32) (by decide) (by decide) + (Arith.YNormL.Inv s₀ ((s₂.gpr .rsi).setWidth 32)) (fun u o hy _ => ⟨?_, ?_, ?_, ?_, fun l hl => ?_, fun l hl => ?_, + fun l hl e he => ?_⟩) fun i hi u hI => Arith.YNormL.step hrd hr hbq hi hI) fun s₄ hI => ?_) + · rw [o.keep.gpr (by decide), k3.gpr (by decide), k2.gpr (by decide), Nat.mul_zero, + VG.Proof.MlKem.X86_64.add_ofNat_zero] + · rw [o.keep.2.1, k3.2.1, k2.2.1] + · rw [o.keep.2.2, k3.2.2, k2.2.2] + · rw [o.mem, m3, hm2] + · simp only [State.lane]; rw [o.xmm, hy]; exact (hc l hl).1 + · simp only [State.lane]; rw [o.xmm, hy]; exact (hc l hl).2.1 + · have : u.lane .xmm10 l = bc (BitVec.allOnes 32) := by simp only [State.lane]; rw [o.xmm, hy]; exact (hc l hl).2.2 + rw [this, show dword (bc (BitVec.allOnes 32)) e = BitVec.allOnes 32 from bc_ofDwords _ e he] + exact iff_of_true (by decide) fun j hj => absurd hj (Nat.not_lt_zero j) + refine WP.mono (nlEnd_ok s₄) fun s' ⟨hv, hm', _, _, _⟩ => ⟨?_, ?_⟩ + · have hnorm : normRq [polyAt s₀.mem (s₀.gpr .rdi)] < arg32 s₀ .rsi ↔ allMsb s₄ = true := by + rw [normRq_lt] + simp only [allMsb, List.all_eq_true, List.mem_range] + constructor + · intro h l hl e he + rw [(hI.acc l hl e he)] + intro j hj + have hk : 8 * j + 4 * l + e < 256 := by omega + have := h _ hk + rw [normZq_lt, polyAt_val hr hk] at this + rw [hb2] + exact (good_clamp (by rw [← VG.Proof.MlDsa.Round.n_eq] at *; exact hr _ hk)).mpr this + · intro h k hk + have hk' : k < 256 := by rw [VG.Proof.MlDsa.Round.n_eq] at hk; exact hk + have := (hI.acc (k % 8 / 4) (by omega) (k % 4) (by omega)).mp (h _ (by omega) _ (by omega)) (k / 8) (by omega) + rw [show 8 * (k / 8) + 4 * (k % 8 / 4) + k % 4 = k by omega, hb2] at this + rw [normZq_lt, polyAt_val hr hk] + exact (good_clamp (by rw [← VG.Proof.MlDsa.Round.n_eq] at *; exact hr _ hk)).mp this + rw [hv] + by_cases h : allMsb s₄ = true + · rw [ite_eq_left h, ite_eq_left (hnorm.mpr h)] + · rw [ite_eq_right h, ite_eq_right (fun h' => h (hnorm.mp h'))] + · rw [hm', hI.mem]; exact Frame.refl _ _ + +theorem normLtY_correct : ∃ t s', Exec isa normLtAvx2 s₀ t s' ∧ abiPreserved s₀ s' ∧ normLtK.post s₀ s' := by + obtain ⟨t, s', he, ⟨hv, hf⟩, hk⟩ := WP.keep [.rax, .rcx, .rsi, .rdi] (normLtY_wp hp) (by decide +kernel) + exact ⟨t, s', he, abiPreserved_of_exec (by decide +kernel) he (gprPreserved_of hk (by decide) hf (by simp)), hv⟩ + +end + +theorem normLtY_ct : ConstantTime isa normLtK.pre normLtK.pub normLtAvx2 := + VG.Taint.constantTime (A := X86_64.taint) (regsLo [.rdi, .rsp] [.rsi]) + (fun _ _ _ _ hp => agree_regsLo (fun r hr => by + simp only [List.mem_cons, List.not_mem_nil, or_false] at hr + rcases hr with rfl | rfl + exacts [hp.1, hp.2.1]) fun r hr => by + simp only [List.mem_singleton] at hr; subst hr; exact hp.2.2) + (by taint_decide) + +theorem normLtY_verified : Verified X86_64.target normLtAvx2 (normLtContract X86_64.abi) := + Verified.of_correct (fun _ hp => normLtY_correct hp) normLtY_ct (by + round_implies [normLtContract, normLtSig, normLtK, X86_64.abi, X86_64.argRegs] [normSat] using normSat) + +end VG.Proof.MlDsa.X86_64.Round diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean index df2a8de65..938cf8fa4 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Inst.lean @@ -65,6 +65,8 @@ def primsWith (B : Impl.MlDsa.X86_64.Arith.Backend) : Prims := sub := B.sub highBits := B.highBits lowBits := B.lowBits + normLt := B.normLt + makeHint := B.makeHint sfx := B.sfx } theorem nosp_of {c : Prog isa} (h : c.allInstrs (fun i => !Taint.clobbers i .rsp) = true) : NoSp c := by @@ -145,10 +147,8 @@ def prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) signStack where Callee _ signStack prims.ball) highBits := calleeOf v.ok.highBits lowBits := calleeOf v.ok.lowBits - normLt := (⟨0, by decide, Proof.MlDsa.X86_64.Round.normLt_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : - Callee _ signStack prims.normLt) - makeHint := (⟨0, by decide, Proof.MlDsa.X86_64.Round.makeHint_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : - Callee _ signStack prims.makeHint) + normLt := calleeOf v.ok.normLt + makeHint := calleeOf v.ok.makeHint simpleBitPack := (⟨0, by decide, Proof.MlDsa.X86_64.Pack.simpleBitPack_verified, nosp_of (by decide +kernel), by decide +kernel⟩ : Callee _ signStack prims.simpleBitPack) diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsC.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsC.lean index 8560f2aa7..450aeefea 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsC.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/PrimsC.lean @@ -160,9 +160,9 @@ theorem normArgs_ok {bs : List (Reg × Nat)} {f : Ptr} {B : Nat} (hB : B < 2 ^ 3 simp only [normChk, Bool.and_eq_true, decide_eq_true_eq] at hc simp only [List.all_cons, List.all_nil, Arg.ok, hc.1.2, hc.2, decide_true, Bool.and_true, decide_eq_true hB] -theorem normCall_ok {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wbs s) {f : Ptr} {B : Nat} +theorem normCall_ok {nm : String} {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wbs s) {f : Ptr} {B : Nat} (hB : B < 2 ^ 32) (hc : normChk (rbs ++ wbs) f = true) (hr : Reduced s.mem (pa s f)) : - WP isa (callP "vg_mldsa_norm_lt" P.normLt [.ptr f, .imm B]) s fun s' => PPostB D s s' [] ∧ + WP isa (callP nm P.normLt [.ptr f, .imm B]) s fun s' => PPostB D s s' [] ∧ (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ (s'.gpr .rax).setWidth 32 = if normRq [polyAt s.mem (pa s f)] < B then 1 else 0 := by have i1 : inB (rbs ++ wbs) f 1024 = true := by @@ -179,10 +179,10 @@ theorem normCall_ok {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wb simp only [e1, e2, Arg.val, er, A.poly' i1 hD, sw32_ofNat hB] at hq exact hq -theorem normCall_tr {P : Prims} (hP : PrimsOk P D) {f : Ptr} {B : Nat} (hB : B < 2 ^ 32) +theorem normCall_tr {nm : String} {P : Prims} (hP : PrimsOk P D) {f : Ptr} {B : Nat} (hB : B < 2 ^ 32) (hc : normChk (rbs ++ wbs) f = true) : RelCT isa (fun x y => LRel D rbs wbs x y ∧ Reduced x.mem (pa x f) ∧ Reduced y.mem (pa y f)) - (callP "vg_mldsa_norm_lt" P.normLt [.ptr f, .imm B]) fun _ _ => True := by + (callP nm P.normLt [.ptr f, .imm B]) fun _ _ => True := by have i1 : inB (rbs ++ wbs) f 1024 = true := by simp only [normChk, Bool.and_eq_true] at hc; exact hc.1.1 refine callP_tr hP.normLt.ver.1 hP.normLt.ver.2.1 (normArgs_ok hB hc) @@ -238,10 +238,10 @@ theorem hintPre {S : Nat} (hS : S + 8 ≤ D) {s s1 : State} (A : At D rbs wbs s simp only [List.mem_cons, List.mem_nil_iff, or_false, forall_eq_or_imp, forall_eq] exact ⟨A.stk hS i1, A.stk hS i2, A.stk hS i3⟩ -theorem hintCall_ok {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wbs s) {z r h : Ptr} {γ : Nat} +theorem hintCall_ok {nm : String} {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wbs s) {z r h : Ptr} {γ : Nat} (hγ : γ ∈ gamma2s) (hc : hintChk (rbs ++ wbs) wbs z r h = true) (rz : Reduced s.mem (pa s z)) (rr : Reduced s.mem (pa s r)) : - WP isa (callP "vg_mldsa_make_hint" P.makeHint [.ptr z, .ptr r, .imm γ, .ptr h]) s fun s' => + WP isa (callP nm P.makeHint [.ptr z, .ptr r, .imm γ, .ptr h]) s fun s' => PPostB D s s' [(h, 1024)] ∧ (∀ r ∈ calleeSaved, s'.gpr r = s.gpr r) ∧ HintIs s'.mem (pa s h) 1 [Vector.zipWith (makeHint γ) (polyAt s.mem (pa s z)) (polyAt s.mem (pa s r))] ∧ ((s'.gpr .rax).setWidth 32).toNat = @@ -259,11 +259,11 @@ theorem hintCall_ok {P : Prims} (hP : PrimsOk P D) {s : State} (L : Lay D rbs wb simp only [e1, e2, e3, e4, Arg.val, hm₂, er, A.poly' i1 hD, A.poly' i2 hD, sw32_ofNat (gamma2_lt hγ)] at hq exact hq -theorem hintCall_tr {P : Prims} (hP : PrimsOk P D) {z r h : Ptr} {γ : Nat} (hγ : γ ∈ gamma2s) +theorem hintCall_tr {nm : String} {P : Prims} (hP : PrimsOk P D) {z r h : Ptr} {γ : Nat} (hγ : γ ∈ gamma2s) (hc : hintChk (rbs ++ wbs) wbs z r h = true) : RelCT isa (fun x y => LRel D rbs wbs x y ∧ (Reduced x.mem (pa x z) ∧ Reduced x.mem (pa x r)) ∧ (Reduced y.mem (pa y z) ∧ Reduced y.mem (pa y r))) - (callP "vg_mldsa_make_hint" P.makeHint [.ptr z, .ptr r, .imm γ, .ptr h]) fun _ _ => True := by + (callP nm P.makeHint [.ptr z, .ptr r, .imm γ, .ptr h]) fun _ _ => True := by obtain ⟨w1, i1, i2, i3, _⟩ := hintChk_spec hc refine callP_tr hP.makeHint.ver.1 hP.makeHint.ver.2.1 (hintArgs_ok hγ hc) fun x y x1 y1 ⟨R, ⟨rzx, rrx⟩, ⟨rzy, rry⟩⟩ ⟨⟨hAx, hmx⟩, kx⟩ ⟨⟨hAy, hmy⟩, ky⟩ => diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean index b04fff664..8d11355f3 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Sign/Verified.lean @@ -76,7 +76,8 @@ with no implementation of it. -/ theorem sign_same {m mc : Prog isa → Bool} (hm : Comp m mc) {B : Backend} (h1 : mc B.ntt = true) (h2 : mc B.invNtt = true) (h3 : mc B.mul = true) (h4 : mc B.mulAdd = true) (h5 : mc B.add = true) - (h6 : mc B.sub = true) (h8 : mc B.highBits = true) (h9 : mc B.lowBits = true) (p : Params) : + (h6 : mc B.sub = true) (h8 : mc B.highBits = true) (h9 : mc B.lowBits = true) + (h10 : mc B.normLt = true) (h11 : mc B.makeHint = true) (p : Params) : Same m (Impl.MlDsa.X86_64.Sign.sign (primsWith B) p) (Impl.MlDsa.X86_64.Sign.sign (primsWith .empty) p) := by unfold Impl.MlDsa.X86_64.Sign.sign same_tac hm @@ -93,14 +94,15 @@ include h3 theorem sign_ctl : ctlOk (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) = true := ctlOk_of_ctlC (Same.ok (sign_same Comp.ctlC v.ok.ntt.ctl v.ok.invNtt.ctl v.ok.mul.ctl v.ok.mulAdd.ctl - v.ok.add.ctl v.ok.sub.ctl v.ok.highBits.ctl v.ok.lowBits.ctl p) (sign0_ctlC h3)) + v.ok.add.ctl v.ok.sub.ctl v.ok.highBits.ctl v.ok.lowBits.ctl v.ok.normLt.ctl v.ok.makeHint.ctl p) (sign0_ctlC h3)) theorem sign_spSafe : (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p).all (fun i => !isa.writesSp i) = true := Code.all_of_allInstrs (Same.ok (sign_same (Comp.all _) (Code.allInstrs_of_all v.ok.ntt.sp) (Code.allInstrs_of_all v.ok.invNtt.sp) (Code.allInstrs_of_all v.ok.mul.sp) (Code.allInstrs_of_all v.ok.mulAdd.sp) (Code.allInstrs_of_all v.ok.add.sp) (Code.allInstrs_of_all v.ok.sub.sp) - (Code.allInstrs_of_all v.ok.highBits.sp) (Code.allInstrs_of_all v.ok.lowBits.sp) p) (sign0_sp h3)) + (Code.allInstrs_of_all v.ok.highBits.sp) (Code.allInstrs_of_all v.ok.lowBits.sp) + (Code.allInstrs_of_all v.ok.normLt.sp) (Code.allInstrs_of_all v.ok.makeHint.sp) p) (sign0_sp h3)) theorem sign_verified : Verified X86_64.target (Impl.MlDsa.X86_64.Sign.sign (primsWith v.code) p) (signContractT p X86_64.abi signStack) := diff --git a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean index 8d87a8c4d..0646b09fd 100644 --- a/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean +++ b/lean/VerifiedGarbage/Proof/MlDsa/X86_64/Verify/Prims.lean @@ -53,6 +53,7 @@ def primsWith (B : Arith.Backend) : Prims := mul := B.mul mulAdd := B.mulAdd sub := B.sub + normLt := B.normLt sfx := B.sfx } /-- A function of the polynomial arithmetic satisfies what the proofs of verification need of it. -/ @@ -102,10 +103,7 @@ theorem prims_okWith (v : ArithImpl) : PrimsOk (primsWith v.code) where (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) (Code.all_of_allInstrs (by lit_decide)) : CalleeOk prims.hintUnpack _) - normLt := (CalleeOk.of_verified Proof.MlDsa.X86_64.Round.normLt_verified (by decide) - (Proof.MlKem.X86_64.nosp_of (by lit_decide)) (by lit_decide) (by lit_decide) - (Code.all_of_allInstrs (by lit_decide)) : - CalleeOk prims.normLt _) + normLt := calleeOf v.ok.normLt /-- `vg_mldsa*_verify` for the parameter set `p`, calling the x86-64 primitives, with the polynomial arithmetic of `v`. -/ diff --git a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean index 23c91e559..5f57b2be7 100644 --- a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean +++ b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Avx2.lean @@ -6,8 +6,10 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.BackendAvx2 A variant of `MlDsaArith` on x86-64 (see `TCB/Emit.lean`): `vg_mldsa_ntt_avx2`, `vg_mldsa_inv_ntt_avx2`, `vg_mldsa_multiply_ntt_avx2`, `vg_mldsa_multiply_add_ntt_avx2`, -`vg_mldsa_add_avx2`, `vg_mldsa_sub_avx2`, `vg_mldsa_high_bits_avx2` and -`vg_mldsa_low_bits_avx2`, on eight coefficients at a time in AVX2 registers, which need AVX and AVX2; key generation, signing and +`vg_mldsa_add_avx2`, `vg_mldsa_sub_avx2`, `vg_mldsa_high_bits_avx2`, +`vg_mldsa_low_bits_avx2`, `vg_mldsa_norm_lt_avx2` and +`vg_mldsa_make_hint_avx2`, on eight coefficients at a time in AVX2 +registers, which need AVX and AVX2; key generation, signing and verification calling them need them too. -/ diff --git a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean index ae7ce4706..fd2c251ab 100644 --- a/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean +++ b/lean/VerifiedGarbage/Variants/MlDsaArith/X86_64/Sse2.lean @@ -5,9 +5,9 @@ import VerifiedGarbage.Proof.MlDsa.X86_64.Arith.Backend A variant of `MlDsaArith` on x86-64 (see `TCB/Emit.lean`): `vg_mldsa_ntt`, `vg_mldsa_inv_ntt`, `vg_mldsa_multiply_ntt`, `vg_mldsa_multiply_add_ntt`, -`vg_mldsa_add`, `vg_mldsa_sub`, `vg_mldsa_high_bits` and -`vg_mldsa_low_bits`, in the baseline ISA (SSE2), -which key generation, signing and verification call. +`vg_mldsa_add`, `vg_mldsa_sub`, `vg_mldsa_high_bits`, +`vg_mldsa_low_bits`, `vg_mldsa_norm_lt` and `vg_mldsa_make_hint`, in the +baseline ISA (SSE2), which key generation, signing and verification call. -/ namespace VG.Variants.MlDsaArith.X86_64.Sse2 diff --git a/src/asm/x86_64/mldsa.rs b/src/asm/x86_64/mldsa.rs index 480c88a58..7d624015f 100644 --- a/src/asm/x86_64/mldsa.rs +++ b/src/asm/x86_64/mldsa.rs @@ -4947,6 +4947,249 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa_make_hint(z: *const [u32; 256], r: ) } +/// The CPU features `vg_mldsa_norm_lt_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_NORM_LT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// Returns 1 if the infinity norm of the polynomial `*f` (FIPS 204 §2.3: the largest `|fᵢ mod± q|`) is less than `bound`, and 0 otherwise. +/// +/// Contract: `VG.Spec.MlDsa.normLtContract`. Constant time: only the pointer and `bound` may affect timing, not the data. +/// +/// The function compares eight coefficients at a time in AVX2 registers; it needs AVX and AVX2. +/// +/// # Safety +/// +/// * `f` must be valid for reads of 1024 bytes. +/// * Each of the 256 `u32`s of `f` must be less than `q` = 8380417. +/// * `f` must not overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_norm_lt_avx2(f: *const [u32; 256], bound: u32) -> u32 { + core::arch::naked_asm!( + "mov esi, esi", + "cmp esi, 8380417", + "jb 20f", + "mov esi, 8380417", + "jmp 21f", + "20:", + "21:", + "mov eax, esi", + "vmovq xmm8, rax", + "vpbroadcastd ymm8, xmm8", + "mov eax, 8380417", + "sub eax, esi", + "vmovq xmm9, rax", + "vpbroadcastd ymm9, xmm9", + "mov eax, -1", + "vmovq xmm10, rax", + "vpbroadcastd ymm10, xmm10", + "mov ecx, 32", + "22:", + "vmovdqu ymm0, YMMWORD PTR [rdi]", + "vpsubd ymm1, ymm0, ymm8", + "vpsubd ymm2, ymm9, ymm0", + "vpor ymm1, ymm1, ymm2", + "vpand ymm10, ymm10, ymm1", + "add rdi, 32", + "sub rcx, 1", + "jne 22b", + "vpsrad ymm10, ymm10, 31", + "vpmovmskb eax, ymm10", + "vzeroupper", + "add rax, 1", + "shr rax, 32", + "ret", + ) +} + +/// The CPU features `vg_mldsa_make_hint_avx2` requires (`Artifact.features`). +pub(crate) const VG_MLDSA_MAKE_HINT_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; + +/// `MakeHint` (FIPS 204 Algorithm 39) of each pair of coefficients of `*z` and `*r`, with `gamma2` = `γ₂`: writes 1 for true and 0 for false to `*h`, and returns the number of 1s. +/// +/// Contract: `VG.Spec.MlDsa.makeHintContract`. Constant time: only the pointers and `gamma2` may affect timing, not the data. +/// +/// The function computes eight hints at a time in AVX2 registers, multiplying by shifts and additions; it needs AVX and AVX2. +/// +/// # Safety +/// +/// * `z` must be valid for reads of 1024 bytes. +/// * `r` must be valid for reads of 1024 bytes. +/// * `h` must be valid for reads and writes of 1024 bytes. +/// * `gamma2` must be (q - 1)/88 = 95232 or (q - 1)/32 = 261888. +/// * Each of the 256 `u32`s of `z` must be less than `q` = 8380417. +/// * Each of the 256 `u32`s of `r` must be less than `q` = 8380417. +/// * `h` must not overlap `z` or `r` (distinct Rust objects never do). +/// * None of `z`, `r` and `h` may overlap the return address on the stack, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `avx` and `avx2` target features. +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa_make_hint_avx2(z: *const [u32; 256], r: *const [u32; 256], gamma2: u32, h: *mut [u32; 256]) -> u32 { + core::arch::naked_asm!( + "mov edx, edx", + "cmp edx, 261888", + "mov r10, rcx", + "mov r9d, 0", + "je 20f", + "mov eax, 127", + "vmovq xmm8, rax", + "vpbroadcastd ymm8, xmm8", + "mov eax, 8388608", + "vmovq xmm9, rax", + "vpbroadcastd ymm9, xmm9", + "mov eax, 44", + "vmovq xmm10, rax", + "vpbroadcastd ymm10, xmm10", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, 63", + "vmovq xmm11, rax", + "vpbroadcastd ymm11, xmm11", + "mov ecx, 32", + "22:", + "vmovdqu ymm0, YMMWORD PTR [rsi]", + "vmovdqu ymm5, YMMWORD PTR [rdi]", + "vmovdqa ymm4, ymm0", + "vpaddd ymm0, ymm0, ymm8", + "vpsrld ymm0, ymm0, 7", + "vmovdqa ymm1, ymm0", + "vpslld ymm2, ymm1, 1", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 3", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 10", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 11", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 13", + "vpaddd ymm0, ymm0, ymm2", + "vpaddd ymm0, ymm0, ymm9", + "vpsrld ymm0, ymm0, 24", + "vpsubd ymm1, ymm0, ymm10", + "vpsrad ymm1, ymm1, 31", + "vpand ymm0, ymm0, ymm1", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm4, ymm5", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm1, ymm0, 31", + "vpand ymm1, ymm1, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpaddd ymm0, ymm0, ymm8", + "vpsrld ymm0, ymm0, 7", + "vmovdqa ymm1, ymm0", + "vpslld ymm2, ymm1, 1", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 3", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 10", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 11", + "vpaddd ymm0, ymm0, ymm2", + "vpslld ymm2, ymm1, 13", + "vpaddd ymm0, ymm0, ymm2", + "vpaddd ymm0, ymm0, ymm9", + "vpsrld ymm0, ymm0, 24", + "vpsubd ymm1, ymm0, ymm10", + "vpsrad ymm1, ymm1, 31", + "vpand ymm0, ymm0, ymm1", + "vpxor ymm0, ymm0, ymm3", + "vpaddd ymm0, ymm0, ymm11", + "vpsrld ymm0, ymm0, 6", + "vpslld ymm1, ymm0, 7", + "vmovdqu YMMWORD PTR [r10], ymm0", + "vpmovmskb eax, ymm1", + "mov edx, eax", + "shr edx, 4", + "add eax, edx", + "mov edx, eax", + "shr edx, 8", + "add eax, edx", + "mov edx, eax", + "shr edx, 16", + "add eax, edx", + "and eax, 15", + "add r9, rax", + "add rdi, 32", + "add rsi, 32", + "add r10, 32", + "sub rcx, 1", + "jne 22b", + "jmp 21f", + "20:", + "mov eax, 127", + "vmovq xmm8, rax", + "vpbroadcastd ymm8, xmm8", + "mov eax, 2097152", + "vmovq xmm9, rax", + "vpbroadcastd ymm9, xmm9", + "mov eax, 16", + "vmovq xmm10, rax", + "vpbroadcastd ymm10, xmm10", + "mov eax, 8380417", + "vmovq xmm15, rax", + "vpbroadcastd ymm15, xmm15", + "mov eax, 63", + "vmovq xmm11, rax", + "vpbroadcastd ymm11, xmm11", + "mov ecx, 32", + "23:", + "vmovdqu ymm0, YMMWORD PTR [rsi]", + "vmovdqu ymm5, YMMWORD PTR [rdi]", + "vmovdqa ymm4, ymm0", + "vpaddd ymm0, ymm0, ymm8", + "vpsrld ymm0, ymm0, 7", + "vmovdqa ymm1, ymm0", + "vpslld ymm2, ymm1, 10", + "vpaddd ymm0, ymm0, ymm2", + "vpaddd ymm0, ymm0, ymm9", + "vpsrld ymm0, ymm0, 22", + "vpsubd ymm1, ymm0, ymm10", + "vpsrad ymm1, ymm1, 31", + "vpand ymm0, ymm0, ymm1", + "vmovdqa ymm3, ymm0", + "vpaddd ymm0, ymm4, ymm5", + "vpsubd ymm0, ymm0, ymm15", + "vpsrad ymm1, ymm0, 31", + "vpand ymm1, ymm1, ymm15", + "vpaddd ymm0, ymm0, ymm1", + "vpaddd ymm0, ymm0, ymm8", + "vpsrld ymm0, ymm0, 7", + "vmovdqa ymm1, ymm0", + "vpslld ymm2, ymm1, 10", + "vpaddd ymm0, ymm0, ymm2", + "vpaddd ymm0, ymm0, ymm9", + "vpsrld ymm0, ymm0, 22", + "vpsubd ymm1, ymm0, ymm10", + "vpsrad ymm1, ymm1, 31", + "vpand ymm0, ymm0, ymm1", + "vpxor ymm0, ymm0, ymm3", + "vpaddd ymm0, ymm0, ymm11", + "vpsrld ymm0, ymm0, 6", + "vpslld ymm1, ymm0, 7", + "vmovdqu YMMWORD PTR [r10], ymm0", + "vpmovmskb eax, ymm1", + "mov edx, eax", + "shr edx, 4", + "add eax, edx", + "mov edx, eax", + "shr edx, 8", + "add eax, edx", + "mov edx, eax", + "shr edx, 16", + "add eax, edx", + "and eax, 15", + "add r9, rax", + "add rdi, 32", + "add rsi, 32", + "add r10, 32", + "sub rcx, 1", + "jne 23b", + "21:", + "mov rax, r9", + "vzeroupper", + "ret", + ) +} + /// `UseHint` (FIPS 204 Algorithm 40) of each pair of coefficients of `*h` (a hint bit: true if it is not 0) and `*r`, with `gamma2` = `γ₂`: writes the results to `*out`. /// /// Contract: `VG.Spec.MlDsa.useHintContract`. Constant time: only the pointers and `gamma2` may affect timing, not the data. diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index 2148553ea..e8be81af8 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -2032,7 +2032,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 14336", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2054,7 +2054,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 15360", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2076,7 +2076,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 16384", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2098,7 +2098,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 17408", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2126,7 +2126,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 7168", "mov esi, 95154", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2154,7 +2154,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 7168", "mov esi, 95154", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2182,7 +2182,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 7168", "mov esi, 95154", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -2210,7 +2210,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 7168", "mov esi, 95154", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 8192", @@ -2227,7 +2227,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 8192", "mov esi, 95232", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -2258,7 +2258,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov edx, 95232", "mov rcx, rbx", "add rcx, 10240", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -2277,7 +2277,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 8192", "mov esi, 95232", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -2308,7 +2308,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov edx, 95232", "mov rcx, rbx", "add rcx, 11264", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -2327,7 +2327,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 8192", "mov esi, 95232", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -2358,7 +2358,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov edx, 95232", "mov rcx, rbx", "add rcx, 12288", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -2377,7 +2377,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov rdi, rbx", "add rdi, 8192", "mov esi, 95232", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -2408,7 +2408,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], "mov edx, 95232", "mov rcx, rbx", "add rcx, 13312", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -2511,10 +2511,10 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign_avx2(sk: *const [u8; 2560], vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, vg_mldsa_low_bits_avx2 = sym super::mldsa::vg_mldsa_low_bits_avx2, - vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_make_hint_avx2 = sym super::mldsa::vg_mldsa_make_hint_avx2, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, ) @@ -2577,7 +2577,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 16384", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 608", @@ -2590,7 +2590,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 17408", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1184", @@ -2603,7 +2603,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 18432", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1760", @@ -2616,7 +2616,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 19456", "mov esi, 130994", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "test r15d, r15d", "jne 22f", @@ -3439,7 +3439,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "ret", vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 831178509..820874863 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -3025,7 +3025,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 16384", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3047,7 +3047,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 17408", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3069,7 +3069,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 18432", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3091,7 +3091,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 19456", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3113,7 +3113,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 20480", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3141,7 +3141,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3169,7 +3169,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3197,7 +3197,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3225,7 +3225,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3253,7 +3253,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -3281,7 +3281,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261692", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 8192", @@ -3298,7 +3298,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3329,7 +3329,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 10240", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3348,7 +3348,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3379,7 +3379,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 11264", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3398,7 +3398,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3429,7 +3429,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 12288", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3448,7 +3448,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3479,7 +3479,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 13312", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3498,7 +3498,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3529,7 +3529,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 14336", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3548,7 +3548,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -3579,7 +3579,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], "mov edx, 261888", "mov rcx, rbx", "add rcx, 15360", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -3690,10 +3690,10 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign_avx2(sk: *const [u8; 4032], vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, vg_mldsa_low_bits_avx2 = sym super::mldsa::vg_mldsa_low_bits_avx2, - vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_make_hint_avx2 = sym super::mldsa::vg_mldsa_make_hint_avx2, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, ) @@ -3756,7 +3756,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 16384", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 688", @@ -3769,7 +3769,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 17408", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1328", @@ -3782,7 +3782,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 18432", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1968", @@ -3795,7 +3795,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 19456", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 2608", @@ -3808,7 +3808,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov rdi, rbx", "add rdi, 20480", "mov esi, 524092", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "test r15d, r15d", "jne 22f", @@ -5154,7 +5154,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "ret", vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index 78ca0d68e..ca8b58813 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -4697,7 +4697,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 18432", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4719,7 +4719,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 19456", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4741,7 +4741,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 20480", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4763,7 +4763,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 21504", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4785,7 +4785,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 22528", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4807,7 +4807,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 23552", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4829,7 +4829,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 24576", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4857,7 +4857,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4885,7 +4885,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4913,7 +4913,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4941,7 +4941,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4969,7 +4969,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -4997,7 +4997,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -5025,7 +5025,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 6144", @@ -5053,7 +5053,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 7168", "mov esi, 261768", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 8192", @@ -5070,7 +5070,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5101,7 +5101,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 10240", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5120,7 +5120,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5151,7 +5151,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 11264", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5170,7 +5170,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5201,7 +5201,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 12288", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5220,7 +5220,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5251,7 +5251,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 13312", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5270,7 +5270,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5301,7 +5301,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 14336", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5320,7 +5320,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5351,7 +5351,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 15360", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5370,7 +5370,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5401,7 +5401,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 16384", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5420,7 +5420,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov rdi, rbx", "add rdi, 8192", "mov esi, 261888", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, rbx", "add rdi, 9216", @@ -5451,7 +5451,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], "mov edx, 261888", "mov rcx, rbx", "add rcx, 17408", - "call {vg_mldsa_make_hint}", + "call {vg_mldsa_make_hint_avx2}", "mov ecx, DWORD PTR [rbx+904]", "add ecx, eax", "mov QWORD PTR [rbx+904], rcx", @@ -5578,10 +5578,10 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign_avx2(sk: *const [u8; 4896], vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_add_avx2 = sym super::mldsa::vg_mldsa_add_avx2, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, vg_mldsa_low_bits_avx2 = sym super::mldsa::vg_mldsa_low_bits_avx2, - vg_mldsa_make_hint = sym super::mldsa::vg_mldsa_make_hint, + vg_mldsa_make_hint_avx2 = sym super::mldsa::vg_mldsa_make_hint_avx2, vg_mldsa_bit_pack = sym super::mldsa::vg_mldsa_bit_pack, vg_mldsa_hint_bit_pack = sym super::mldsa::vg_mldsa_hint_bit_pack, ) @@ -5644,7 +5644,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 16384", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 704", @@ -5657,7 +5657,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 17408", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1344", @@ -5670,7 +5670,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 18432", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 1984", @@ -5683,7 +5683,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 19456", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 2624", @@ -5696,7 +5696,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 20480", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 3264", @@ -5709,7 +5709,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 21504", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "mov rdi, r13", "add rdi, 3904", @@ -5722,7 +5722,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "mov rdi, rbx", "add rdi, 22528", "mov esi, 524168", - "call {vg_mldsa_norm_lt}", + "call {vg_mldsa_norm_lt_avx2}", "and r15d, eax", "test r15d, r15d", "jne 22f", @@ -7968,7 +7968,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "ret", vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, From 474dfa3acb87133c027f515fcf32b028602ed609 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 02:14:05 +0000 Subject: [PATCH 2/2] Regenerate src/asm after merging claude/fervent-einstein-ukl7t7-ybits2 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ddof3szoTi7HB8iCsM2MCr --- src/asm/x86_64/mldsa44.rs | 816 ++--- src/asm/x86_64/mldsa65.rs | 1508 +++----- src/asm/x86_64/mldsa87.rs | 6838 +++++++++++++++---------------------- 3 files changed, 3358 insertions(+), 5804 deletions(-) diff --git a/src/asm/x86_64/mldsa44.rs b/src/asm/x86_64/mldsa44.rs index e8be81af8..53ac13157 100644 --- a/src/asm/x86_64/mldsa44.rs +++ b/src/asm/x86_64/mldsa44.rs @@ -2537,7 +2537,7 @@ pub(crate) const VG_MLDSA44_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { @@ -2634,119 +2634,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -2754,270 +2718,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -3033,13 +2841,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -3415,7 +3223,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -3423,7 +3231,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -3440,7 +3248,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify_avx2(pk: *const [u8; 1312 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, @@ -5980,7 +5788,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_sign(sk: *const [u8; 2560], mu: /// * `scratch` must be valid for reads and writes of 77824 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu: *const [u8; 64], sig: *const [u8; 2420], scratch: *mut [u64; 9728]) -> u32 { core::arch::naked_asm!( @@ -6076,119 +5884,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,270 +5968,114 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "212:", + "add rdi, 36864", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "216:", + "add rdi, 45056", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", + "jne 211b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "220:", + "add rdi, 53248", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", + "jne 212b", "mov rdi, r13", "add rdi, 0", "mov esi, 32", @@ -6475,13 +6091,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "221:", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 213b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -6857,7 +6473,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rdi, 0", "mov ecx, 32", "mov edx, 0", - "222:", + "214:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -6865,7 +6481,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 222b", + "jne 214b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -6882,7 +6498,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa44_verify(pk: *const [u8; 1312], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt, diff --git a/src/asm/x86_64/mldsa65.rs b/src/asm/x86_64/mldsa65.rs index 820874863..1708a829d 100644 --- a/src/asm/x86_64/mldsa65.rs +++ b/src/asm/x86_64/mldsa65.rs @@ -3716,7 +3716,7 @@ pub(crate) const VG_MLDSA65_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { @@ -3826,119 +3826,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3946,14 +3910,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 29b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -3961,7 +3925,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 32768", "mov ecx, 256", "210:", "mov eax, DWORD PTR [rdi]", @@ -3970,23 +3934,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -3994,14 +3970,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4009,7 +3985,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 40960", "mov ecx, 256", "212:", "mov eax, DWORD PTR [rdi]", @@ -4018,23 +3994,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4044,12 +4032,12 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "jne 213b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 1", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4057,7 +4045,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 49152", "mov ecx, 256", "214:", "mov eax, DWORD PTR [rdi]", @@ -4067,22 +4055,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "sub rcx, 1", "jne 214b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4090,14 +4090,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 215b", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4105,7 +4105,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 57344", "mov ecx, 256", "216:", "mov eax, DWORD PTR [rdi]", @@ -4114,23 +4114,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4138,14 +4150,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 217b", - "mov eax, 3", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 2", + "mov eax, 4", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 65536", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4153,7 +4165,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 65536", "mov ecx, 256", "218:", "mov eax, DWORD PTR [rdi]", @@ -4162,23 +4174,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -4186,14 +4210,14 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 5", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -4201,7 +4225,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", + "add rdi, 73728", "mov ecx, 256", "220:", "mov eax, DWORD PTR [rdi]", @@ -4210,22 +4234,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 220b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 48", + "mov edx, 49", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 23552", "mov ecx, 256", "221:", "mov eax, DWORD PTR [rdi]", @@ -4234,422 +4256,88 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 4", "sub rcx, 1", "jne 221b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 29696", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 48", - "mov edx, 49", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", @@ -5130,7 +4818,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -5138,7 +4826,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -5155,6 +4843,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify_avx2(pk: *const [u8; 1952 vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, @@ -8874,7 +8563,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_sign(sk: *const [u8; 4032], mu: /// * `scratch` must be valid for reads and writes of 103424 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu: *const [u8; 64], sig: *const [u8; 3309], scratch: *mut [u64; 12928]) -> u32 { core::arch::naked_asm!( @@ -8983,398 +8672,98 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "mov rdi, rbx", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 219b", + "jne 28b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "220:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 1", + "jne 29b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 0", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9382,47 +8771,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 54272", + "add rdi, 32768", "mov ecx, 256", - "221:", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", + "jne 210b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "222:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 3", + "jne 211b", + "mov eax, 4", "mov BYTE PTR [rbx+928], al", - "mov eax, 3", + "mov eax, 1", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9430,47 +8831,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 56320", + "add rdi, 40960", "mov ecx, 256", - "223:", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "224:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "jne 213b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 2", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9478,47 +8891,59 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 61440", + "add rdi, 49152", "mov ecx, 256", - "225:", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "226:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "jne 215b", "mov eax, 4", + "mov BYTE PTR [rbx+928], al", + "mov eax, 3", "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", "add rdi, 896", "mov rsi, rbx", - "add rsi, 63488", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 4096", "call {vg_mldsa_rej_ntt_poly}", @@ -9526,39 +8951,51 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 63488", + "add rdi, 57344", "mov ecx, 256", - "227:", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "228:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", + "jne 217b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 4", @@ -9576,109 +9013,49 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 65536", "mov ecx, 256", - "229:", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", + "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2626], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 77824", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "233:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", + "jne 219b", "mov eax, 4", "mov BYTE PTR [rbx+928], al", "mov eax, 5", @@ -9696,13 +9073,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 73728", "mov ecx, 256", - "234:", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", + "jne 220b", "mov rdi, r13", "add rdi, 0", "mov esi, 48", @@ -9718,13 +9095,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "235:", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", + "jne 221b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -10287,7 +9664,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rdi, 0", "mov ecx, 48", "mov edx, 0", - "236:", + "222:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -10295,7 +9672,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 236b", + "jne 222b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -10312,6 +9689,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa65_verify(pk: *const [u8; 1952], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, diff --git a/src/asm/x86_64/mldsa87.rs b/src/asm/x86_64/mldsa87.rs index ca8b58813..6562cfbf6 100644 --- a/src/asm/x86_64/mldsa87.rs +++ b/src/asm/x86_64/mldsa87.rs @@ -5604,7 +5604,7 @@ pub(crate) const VG_MLDSA87_VERIFY_AVX2_FEATURES: &[&str] = &["avx", "avx2"]; /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). /// * The CPU must support the `avx` and `avx2` target features. #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { @@ -5740,119 +5740,83 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 1", "sub rcx, 1", "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", + "add rdi, 28672", + "mov ecx, 1024", "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5860,23 +5824,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", + "add rdi, 31744", + "mov ecx, 1024", "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5884,23 +5860,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", "mov eax, 0", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", + "add rdi, 36864", + "mov ecx, 1024", "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5908,23 +5896,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", + "add rdi, 39936", + "mov ecx, 1024", "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5932,23 +5932,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", + "add rdi, 45056", + "mov ecx, 1024", "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5956,23 +5968,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 213b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", + "add rdi, 48128", + "mov ecx, 1024", "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -5980,23 +6004,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", + "add rdi, 53248", + "mov ecx, 1024", "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6004,23 +6040,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", + "add rdi, 56320", + "mov ecx, 1024", "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6028,23 +6076,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", + "add rdi, 61440", + "mov ecx, 1024", "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6052,23 +6112,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", + "add rdi, 64512", + "mov ecx, 1024", "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6077,22 +6149,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 218b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", + "mov rdi, rbx", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", + "add rdi, 69632", + "mov ecx, 1024", "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6100,23 +6184,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", + "add rdi, 72704", + "mov ecx, 1024", "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6124,23 +6220,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", + "add rdi, 77824", + "mov ecx, 1024", "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6149,22 +6257,34 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "sub rcx, 1", "jne 221b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", + "add rdi, 80896", + "mov ecx, 1024", "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6172,23 +6292,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", + "add rdi, 86016", + "mov ecx, 1024", "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6196,23 +6328,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4_avx2}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", + "add rdi, 89088", + "mov ecx, 1024", "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", @@ -6220,22 +6364,20 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "mov rdi, r13", + "add rdi, 0", + "mov esi, 64", + "mov edx, 60", + "mov rcx, rbx", + "add rcx, 23552", + "mov r8, rbx", + "add r8, 4096", + "call {vg_mldsa_sample_in_ball}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 51200", + "add rdi, 23552", "mov ecx, 256", "225:", "mov eax, DWORD PTR [rdi]", @@ -6244,971 +6386,473 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 4", "sub rcx, 1", "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 16384", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 17408", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 18432", "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 20480", "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 22528", "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 23552", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 29696", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 30720", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 31744", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 32768", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 33792", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 61440", + "add rsi, 34816", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 32", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 62464", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 8192", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1024", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 64512", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 37888", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 65536", + "add rsi, 38912", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 39936", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 66560", + "add rsi, 40960", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 41984", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 67584", + "add rsi, 43008", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 352", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 9216", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1152", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 46080", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 47104", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 48128", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 49152", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 50176", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 51200", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 672", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 246b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 247b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 10240", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1280", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 248b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 54272", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 55296", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 249b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 57344", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 58368", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 59392", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 992", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 24576", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 25600", + "call {vg_mldsa_sub_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rsi, 4096", + "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 11264", + "mov rsi, rbx", + "add rsi, 26624", + "mov edx, 261888", + "mov rcx, rbx", + "add rcx, 27648", + "call {vg_mldsa_use_hint}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 27648", + "mov esi, 15", + "mov rdx, rbx", + "add rdx, 1408", + "mov ecx, 128", + "call {vg_mldsa_simple_bit_pack}", + "mov rdi, rbx", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 16384", + "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 62464", + "mov rdx, rbx", + "add rdx, 17408", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 63488", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 18432", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 64512", + "mov rdx, rbx", + "add rdx, 19456", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 65536", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 20480", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 26624", + "mov rsi, rbx", + "add rsi, 66560", + "mov rdx, rbx", + "add rdx, 21504", + "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 26624", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 67584", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdx, 22528", + "call {vg_mldsa_multiply_add_ntt_avx2}", + "mov rdi, rbp", + "add rdi, 1312", + "mov rsi, rbx", + "add rsi, 24576", + "call {vg_mldsa_unpack_t1}", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "add rdi, 24576", + "mov rsi, rbx", + "add rsi, 4096", + "call {vg_mldsa_ntt_avx2}", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 25600", "mov rsi, rbx", - "add rsi, 91136", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 92160", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov rdi, r13", - "add rdi, 0", - "mov esi, 64", - "mov edx, 60", - "mov rcx, rbx", - "add rcx, 23552", - "mov r8, rbx", - "add r8, 4096", - "call {vg_mldsa_sample_in_ball}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 23552", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov rdi, rbx", - "add rdi, 16384", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 17408", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 18432", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 19456", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 20480", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 21504", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 22528", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 23552", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 32", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", - "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 25600", - "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 24576", "call {vg_mldsa_multiply_ntt_avx2}", @@ -7223,7 +6867,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 12288", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7234,60 +6878,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1024", + "add rdx, 1536", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 69632", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 70656", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 71680", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 72704", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 73728", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 74752", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 75776", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 352", + "add rdi, 1632", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7314,7 +6958,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 13312", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7325,60 +6969,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1152", + "add rdx, 1664", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 77824", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 78848", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 47104", + "add rsi, 79872", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 48128", + "add rsi, 80896", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 49152", + "add rsi, 81920", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 50176", + "add rsi, 82944", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 51200", + "add rsi, 83968", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 672", + "add rdi, 1952", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7405,7 +7049,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 14336", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7416,60 +7060,60 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1280", + "add rdx, 1792", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 53248", + "add rsi, 86016", "mov rdx, rbx", "add rdx, 16384", "call {vg_mldsa_multiply_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 54272", + "add rsi, 87040", "mov rdx, rbx", "add rdx, 17408", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 55296", + "add rsi, 88064", "mov rdx, rbx", "add rdx, 18432", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 56320", + "add rsi, 89088", "mov rdx, rbx", "add rdx, 19456", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 57344", + "add rsi, 90112", "mov rdx, rbx", "add rdx, 20480", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 58368", + "add rsi, 91136", "mov rdx, rbx", "add rdx, 21504", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbx", "add rdi, 26624", "mov rsi, rbx", - "add rsi, 59392", + "add rsi, 92160", "mov rdx, rbx", "add rdx, 22528", "call {vg_mldsa_multiply_add_ntt_avx2}", "mov rdi, rbp", - "add rdi, 992", + "add rdi, 2272", "mov rsi, rbx", "add rsi, 24576", "call {vg_mldsa_unpack_t1}", @@ -7496,7 +7140,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rsi, 4096", "call {vg_mldsa_inv_ntt_avx2}", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 15360", "mov rsi, rbx", "add rsi, 26624", "mov edx, 261888", @@ -7507,618 +7151,854 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify_avx2(pk: *const [u8; 2592 "add rdi, 27648", "mov esi, 15", "mov rdx, rbx", - "add rdx, 1408", + "add rdx, 1920", "mov ecx, 128", "call {vg_mldsa_simple_bit_pack}", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, r12", + "add rcx, 0", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 64", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 1024", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 2048", + "mov r8d, 64", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rsi, 2048", + "mov rdi, r13", + "add rdi, 0", + "mov ecx, 64", + "mov edx, 0", + "226:", + "movzx eax, BYTE PTR [rsi]", + "movzx r8d, BYTE PTR [rdi]", + "xor rax, r8", + "or rdx, rax", + "add rsi, 1", + "add rdi, 1", + "sub rcx, 1", + "jne 226b", + "sub rdx, 1", + "sbb rax, rax", + "and r15d, eax", + "23:", + "21:", + "mov eax, r15d", + "mov r15, QWORD PTR [rbx+880]", + "mov r14, QWORD PTR [rbx+872]", + "mov r13, QWORD PTR [rbx+864]", + "mov r12, QWORD PTR [rbx+856]", + "mov rbp, QWORD PTR [rbx+848]", + "mov rbx, QWORD PTR [rbx+840]", + "ret", + vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, + vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, + vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, + vg_mldsa_rej_ntt_poly4_avx2 = sym super::mldsa::vg_mldsa_rej_ntt_poly4_avx2, + vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, + vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, + vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, + vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, + vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, + vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, + vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, + vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, + vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, + vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, + vg_keccak_pad = sym super::sha3::vg_keccak_pad, + vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, + ) +} + +/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. +/// +/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. +/// +/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. +/// +/// # Safety +/// +/// * `seed` must be valid for reads of 32 bytes. +/// * `pk` must be valid for reads and writes of 2592 bytes. +/// * `sk` must be valid for reads and writes of 4896 bytes. +/// * `scratch` must be valid for reads and writes of 144384 bytes. +/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). +/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). +/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +#[unsafe(naked)] +pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { + core::arch::naked_asm!( + "mov QWORD PTR [rcx+840], rbx", + "mov QWORD PTR [rcx+848], rbp", + "mov QWORD PTR [rcx+856], r12", + "mov QWORD PTR [rcx+864], r13", + "mov QWORD PTR [rcx+872], r14", + "mov QWORD PTR [rcx+880], r15", + "mov rbx, rcx", + "mov rbp, rdi", + "mov r12, rsi", + "mov r13, rdx", + "mov r15d, 1", + "mov eax, 8", + "mov BYTE PTR [rbx+896], al", + "mov eax, 7", + "mov BYTE PTR [rbx+897], al", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbp", + "add rcx, 0", + "mov r8d, 32", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1312", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 32", + "mov rcx, rbx", + "add rcx, 896", + "mov r8d, 2", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_absorb}", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 0", + "mov esi, 136", + "mov edx, 34", + "mov ecx, 31", + "mov r8, rbx", + "add r8, 200", + "call {vg_keccak_pad}", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 0", + "mov esi, 136", + "mov edx, 0", + "mov rcx, rbx", + "add rcx, 1024", + "mov r8d, 128", + "mov r9, rbx", + "add r9, 200", + "call {vg_keccak_squeeze}", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", - "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rsi, 1024", + "mov ecx, 32", + "20:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 20b", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1216", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 1056", + "mov ecx, 64", + "21:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 21b", + "mov eax, 0", + "mov BYTE PTR [rbx+1281], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 12288", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", "mov rdx, rbx", - "add rdx, 1536", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 4096", + "mov ecx, 256", + "22:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 22b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 5120", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 5120", + "mov ecx, 256", + "23:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 23b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 6144", + "mov ecx, 256", + "24:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 24b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 7168", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 7168", + "mov ecx, 256", + "25:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 25b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 8192", + "mov ecx, 256", + "26:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 26b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 9216", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 9216", + "mov ecx, 256", + "27:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 27b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 0", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 10240", + "mov ecx, 256", + "28:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 11264", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1632", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 11264", + "mov ecx, 256", + "29:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 29b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdi, 12288", + "mov ecx, 256", + "210:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 210b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 13312", - "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1664", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "mov ecx, 256", + "211:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 78848", - "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 14336", + "mov ecx, 256", + "212:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 212b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 15360", "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 80896", - "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 15360", + "mov ecx, 256", + "213:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 213b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 16384", "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 82944", - "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 16384", + "mov ecx, 256", + "214:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 214b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 1", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 17408", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 1952", - "mov rsi, rbx", - "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "add rdi, 17408", + "mov ecx, 256", + "215:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 215b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 18432", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "add rdi, 18432", + "mov ecx, 256", + "216:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 216b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", - "mov rdi, rbx", - "add rdi, 27648", - "mov esi, 15", + "add rsi, 19456", "mov rdx, rbx", - "add rdx, 1792", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 86016", - "mov rdx, rbx", - "add rdx, 16384", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdi, 19456", + "mov ecx, 256", + "217:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 217b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 20480", "mov rdx, rbx", - "add rdx, 17408", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 88064", - "mov rdx, rbx", - "add rdx, 18432", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdi, 20480", + "mov ecx, 256", + "218:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 218b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 21504", "mov rdx, rbx", - "add rdx, 19456", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbx", - "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 90112", - "mov rdx, rbx", - "add rdx, 20480", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 21504", + "mov ecx, 256", + "219:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 219b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 22528", "mov rdx, rbx", - "add rdx, 21504", - "call {vg_mldsa_multiply_add_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 22528", + "mov ecx, 256", + "220:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 220b", + "mov eax, 5", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 23552", "mov rdx, rbx", - "add rdx, 22528", - "call {vg_mldsa_multiply_add_ntt_avx2}", - "mov rdi, rbp", - "add rdi, 2272", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", + "mov rdi, rbx", + "add rdi, 23552", + "mov ecx, 256", + "221:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 221b", + "mov eax, 6", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 2", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", "add rsi, 24576", - "call {vg_mldsa_unpack_t1}", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 24576", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_ntt_avx2}", + "mov ecx, 256", + "222:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 25600", "mov rdx, rbx", - "add rdx, 24576", - "call {vg_mldsa_multiply_ntt_avx2}", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 25600", + "mov ecx, 256", + "223:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 223b", + "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", + "mov rdi, rbx", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", - "call {vg_mldsa_sub_avx2}", + "add rsi, 26624", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 26624", - "mov rsi, rbx", - "add rsi, 4096", - "call {vg_mldsa_inv_ntt_avx2}", + "mov ecx, 256", + "224:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 224b", + "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", - "mov edx, 261888", - "mov rcx, rbx", - "add rcx, 27648", - "call {vg_mldsa_use_hint}", + "add rsi, 27648", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", "add rdi, 27648", - "mov esi, 15", - "mov rdx, rbx", - "add rdx, 1920", - "mov ecx, 128", - "call {vg_mldsa_simple_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov ecx, 256", + "225:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 225b", + "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, r12", - "add rcx, 0", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", + "add rdi, 1152", + "mov rsi, rbx", + "add rsi, 28672", + "mov rdx, rbx", + "add rdx, 2048", + "call {vg_mldsa_rej_ntt_poly}", + "and r15d, eax", + "mov r8d, 0", + "sub r8d, eax", "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 64", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 1024", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 2048", - "mov r8d, 64", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rsi, rbx", - "add rsi, 2048", - "mov rdi, r13", - "add rdi, 0", - "mov ecx, 64", - "mov edx, 0", - "262:", - "movzx eax, BYTE PTR [rsi]", - "movzx r8d, BYTE PTR [rdi]", - "xor rax, r8", - "or rdx, rax", - "add rsi, 1", - "add rdi, 1", - "sub rcx, 1", - "jne 262b", - "sub rdx, 1", - "sbb rax, rax", - "and r15d, eax", - "23:", - "21:", - "mov eax, r15d", - "mov r15, QWORD PTR [rbx+880]", - "mov r14, QWORD PTR [rbx+872]", - "mov r13, QWORD PTR [rbx+864]", - "mov r12, QWORD PTR [rbx+856]", - "mov rbp, QWORD PTR [rbx+848]", - "mov rbx, QWORD PTR [rbx+840]", - "ret", - vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, - vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, - vg_mldsa_norm_lt_avx2 = sym super::mldsa::vg_mldsa_norm_lt_avx2, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, - vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, - vg_mldsa_ntt_avx2 = sym super::mldsa::vg_mldsa_ntt_avx2, - vg_mldsa_multiply_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_ntt_avx2, - vg_mldsa_multiply_add_ntt_avx2 = sym super::mldsa::vg_mldsa_multiply_add_ntt_avx2, - vg_mldsa_unpack_t1 = sym super::mldsa::vg_mldsa_unpack_t1, - vg_mldsa_sub_avx2 = sym super::mldsa::vg_mldsa_sub_avx2, - vg_mldsa_inv_ntt_avx2 = sym super::mldsa::vg_mldsa_inv_ntt_avx2, - vg_mldsa_use_hint = sym super::mldsa::vg_mldsa_use_hint, - vg_mldsa_simple_bit_pack = sym super::mldsa::vg_mldsa_simple_bit_pack, - vg_keccak_absorb = sym super::sha3::vg_keccak_absorb, - vg_keccak_pad = sym super::sha3::vg_keccak_pad, - vg_keccak_squeeze = sym super::sha3::vg_keccak_squeeze, - ) -} - -/// ML-DSA-87 key generation from a seed, `ML-DSA.KeyGen_internal(ξ)` (FIPS 204 Algorithm 6): with the 32-byte seed `ξ` at `seed`, writes the public key to `*pk` and the private key to `*sk`. Returns 1 on success. Returns 0 if a loop reaches its bound, which is at least the limit of FIPS 204 Appendix C, Table 3 (this happens with probability about 2^-256 or less): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. -/// -/// Contract: `VG.Spec.MlDsa.keyGenContract`. Constant time but for `ρ` and rejections: timing may depend on the pointers, on `ρ` (the first 32 bytes of the public key), and on which half-bytes of the SHAKE256 outputs `RejBoundedPoly` rejects (`rejBoundedLeak`, which is independent of the coefficients it samples), but not on anything else of the seed or the keys. -/// -/// The function saves its caller's callee-saved registers in `scratch`; its calls use the 32 bytes of stack below its return address. -/// -/// It samples every polynomial of `A` and of `s1` and `s2` whatever the samplers return, and zeroes the polynomial of a sampler that fails rather than branching on it: its timing does not depend on whether key generation fails. -/// -/// # Safety -/// -/// * `seed` must be valid for reads of 32 bytes. -/// * `pk` must be valid for reads and writes of 2592 bytes. -/// * `sk` must be valid for reads and writes of 4896 bytes. -/// * `scratch` must be valid for reads and writes of 144384 bytes. -/// * `seed` must be random bytes from an approved RBG (FIPS 204 §3.6.1), or a seed so generated before. -/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). -/// * `pk`, `sk` and `scratch` must not overlap each other or `seed` (distinct Rust objects never do). -/// * None of `seed`, `pk`, `sk` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). -#[unsafe(naked)] -pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk: *mut [u8; 2592], sk: *mut [u8; 4896], scratch: *mut [u64; 18048]) -> u32 { - core::arch::naked_asm!( - "mov QWORD PTR [rcx+840], rbx", - "mov QWORD PTR [rcx+848], rbp", - "mov QWORD PTR [rcx+856], r12", - "mov QWORD PTR [rcx+864], r13", - "mov QWORD PTR [rcx+872], r14", - "mov QWORD PTR [rcx+880], r15", - "mov rbx, rcx", - "mov rbp, rdi", - "mov r12, rsi", - "mov r13, rdx", - "mov r15d, 1", - "mov eax, 8", - "mov BYTE PTR [rbx+896], al", - "mov eax, 7", - "mov BYTE PTR [rbx+897], al", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbp", - "add rcx, 0", - "mov r8d, 32", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 32", - "mov rcx, rbx", - "add rcx, 896", - "mov r8d, 2", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_absorb}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 34", - "mov ecx, 31", - "mov r8, rbx", - "add r8, 200", - "call {vg_keccak_pad}", - "mov rdi, rbx", - "add rdi, 0", - "mov esi, 136", - "mov edx, 0", - "mov rcx, rbx", - "add rcx, 1024", - "mov r8d, 128", - "mov r9, rbx", - "add r9, 200", - "call {vg_keccak_squeeze}", - "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "20:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 20b", - "mov rdi, rbx", - "add rdi, 1216", - "mov rsi, rbx", - "add rsi, 1056", - "mov ecx, 64", - "21:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 21b", - "mov eax, 0", - "mov BYTE PTR [rbx+1281], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 28672", + "mov ecx, 256", + "226:", + "mov eax, DWORD PTR [rdi]", + "and eax, r8d", + "mov DWORD PTR [rdi], eax", + "add rdi, 4", + "sub rcx, 1", + "jne 226b", + "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 3", + "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 4096", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8126,23 +8006,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 4096", + "add rdi, 29696", "mov ecx, 256", - "22:", + "227:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 22b", - "mov eax, 1", + "jne 227b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8150,23 +8030,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 5120", + "add rdi, 30720", "mov ecx, 256", - "23:", + "228:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 23b", - "mov eax, 2", + "jne 228b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 3", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8174,23 +8054,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 6144", + "add rdi, 31744", "mov ecx, 256", - "24:", + "229:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 24b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", + "jne 229b", "mov eax, 0", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8198,23 +8078,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 7168", + "add rdi, 32768", "mov ecx, 256", - "25:", + "230:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 25b", - "mov eax, 4", + "jne 230b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8222,23 +8102,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 8192", + "add rdi, 33792", "mov ecx, 256", - "26:", + "231:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 26b", - "mov eax, 5", + "jne 231b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8246,23 +8126,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 9216", + "add rdi, 34816", "mov ecx, 256", - "27:", + "232:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 27b", - "mov eax, 6", + "jne 232b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 0", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8270,23 +8150,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 10240", + "add rdi, 35840", "mov ecx, 256", - "28:", + "233:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 28b", - "mov eax, 0", + "jne 233b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8294,23 +8174,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 11264", + "add rdi, 36864", "mov ecx, 256", - "29:", + "234:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 29b", - "mov eax, 1", + "jne 234b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8318,23 +8198,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 12288", + "add rdi, 37888", "mov ecx, 256", - "210:", + "235:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 210b", - "mov eax, 2", + "jne 235b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 4", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8342,23 +8222,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 13312", + "add rdi, 38912", "mov ecx, 256", - "211:", + "236:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 211b", - "mov eax, 3", + "jne 236b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8366,23 +8246,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 14336", + "add rdi, 39936", "mov ecx, 256", - "212:", + "237:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 212b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", + "jne 237b", "mov eax, 1", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8390,23 +8270,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 15360", + "add rdi, 40960", "mov ecx, 256", - "213:", + "238:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 213b", - "mov eax, 5", + "jne 238b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8414,23 +8294,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 16384", + "add rdi, 41984", "mov ecx, 256", - "214:", + "239:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 214b", - "mov eax, 6", + "jne 239b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 1", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8438,23 +8318,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 17408", + "add rdi, 43008", "mov ecx, 256", - "215:", + "240:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 215b", - "mov eax, 0", + "jne 240b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8462,23 +8342,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 18432", + "add rdi, 44032", "mov ecx, 256", - "216:", + "241:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 216b", - "mov eax, 1", + "jne 241b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8486,23 +8366,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 19456", + "add rdi, 45056", "mov ecx, 256", - "217:", + "242:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 217b", - "mov eax, 2", + "jne 242b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 5", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8510,23 +8390,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 20480", + "add rdi, 46080", "mov ecx, 256", - "218:", + "243:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 218b", - "mov eax, 3", + "jne 243b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8534,23 +8414,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 21504", + "add rdi, 47104", "mov ecx, 256", - "219:", + "244:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 219b", - "mov eax, 4", + "jne 244b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8558,23 +8438,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 22528", + "add rdi, 48128", "mov ecx, 256", - "220:", + "245:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 220b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", + "jne 245b", "mov eax, 2", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8582,23 +8462,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 23552", + "add rdi, 49152", "mov ecx, 256", - "221:", + "246:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 221b", - "mov eax, 6", + "jne 246b", + "mov eax, 3", "mov BYTE PTR [rbx+1184], al", - "mov eax, 2", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8606,23 +8486,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 24576", + "add rdi, 50176", "mov ecx, 256", - "222:", + "247:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 222b", - "mov eax, 0", + "jne 247b", + "mov eax, 4", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8630,23 +8510,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 25600", + "add rdi, 51200", "mov ecx, 256", - "223:", + "248:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 223b", - "mov eax, 1", + "jne 248b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8654,23 +8534,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 26624", + "add rdi, 52224", "mov ecx, 256", - "224:", + "249:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 224b", - "mov eax, 2", + "jne 249b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 6", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8678,23 +8558,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 27648", + "add rdi, 53248", "mov ecx, 256", - "225:", + "250:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 225b", - "mov eax, 3", + "jne 250b", + "mov eax, 0", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8702,23 +8582,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 28672", + "add rdi, 54272", "mov ecx, 256", - "226:", + "251:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 226b", - "mov eax, 4", + "jne 251b", + "mov eax, 1", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8726,23 +8606,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 29696", + "add rdi, 55296", "mov ecx, 256", - "227:", + "252:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 227b", - "mov eax, 5", + "jne 252b", + "mov eax, 2", "mov BYTE PTR [rbx+1184], al", - "mov eax, 3", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8750,23 +8630,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 30720", + "add rdi, 56320", "mov ecx, 256", - "228:", + "253:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 228b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", + "jne 253b", "mov eax, 3", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8774,23 +8654,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 31744", + "add rdi, 57344", "mov ecx, 256", - "229:", + "254:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 229b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", + "jne 254b", "mov eax, 4", + "mov BYTE PTR [rbx+1184], al", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8798,23 +8678,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 32768", + "add rdi, 58368", "mov ecx, 256", - "230:", + "255:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 230b", - "mov eax, 1", + "jne 255b", + "mov eax, 5", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8822,23 +8702,23 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 33792", + "add rdi, 59392", "mov ecx, 256", - "231:", + "256:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 231b", - "mov eax, 2", + "jne 256b", + "mov eax, 6", "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", + "mov eax, 7", "mov BYTE PTR [rbx+1185], al", "mov rdi, rbx", "add rdi, 1152", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 2048", "call {vg_mldsa_rej_ntt_poly}", @@ -8846,1197 +8726,759 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 34816", + "add rdi, 60416", "mov ecx, 256", - "232:", + "257:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 232b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 257b", + "mov eax, 0", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 35840", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 61440", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 35840", + "add rdi, 61440", "mov ecx, 256", - "233:", + "258:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 233b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 258b", + "mov eax, 1", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 36864", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 62464", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 36864", + "add rdi, 62464", "mov ecx, 256", - "234:", + "259:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 234b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 259b", + "mov eax, 2", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 37888", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 63488", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 37888", + "add rdi, 63488", "mov ecx, 256", - "235:", + "260:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 235b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 4", - "mov BYTE PTR [rbx+1185], al", + "jne 260b", + "mov eax, 3", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 38912", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 64512", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 38912", + "add rdi, 64512", "mov ecx, 256", - "236:", + "261:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 236b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 261b", + "mov eax, 4", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 39936", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 65536", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 39936", + "add rdi, 65536", "mov ecx, 256", - "237:", + "262:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 237b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", + "jne 262b", "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 40960", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 66560", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 40960", + "add rdi, 66560", "mov ecx, 256", - "238:", + "263:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 238b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 263b", + "mov eax, 6", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 41984", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 67584", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 41984", + "add rdi, 67584", "mov ecx, 256", - "239:", + "264:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 239b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 264b", + "mov eax, 7", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 43008", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 68608", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 43008", + "add rdi, 68608", "mov ecx, 256", - "240:", + "265:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 240b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 265b", + "mov eax, 8", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 44032", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 69632", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 44032", + "add rdi, 69632", "mov ecx, 256", - "241:", + "266:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 241b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 266b", + "mov eax, 9", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 45056", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 70656", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 45056", + "add rdi, 70656", "mov ecx, 256", - "242:", + "267:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 242b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 5", - "mov BYTE PTR [rbx+1185], al", + "jne 267b", + "mov eax, 10", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 46080", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 71680", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 46080", + "add rdi, 71680", "mov ecx, 256", - "243:", + "268:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 243b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 268b", + "mov eax, 11", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 47104", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 72704", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 47104", + "add rdi, 72704", "mov ecx, 256", - "244:", + "269:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 244b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 269b", + "mov eax, 12", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 48128", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 73728", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 48128", + "add rdi, 73728", "mov ecx, 256", - "245:", + "270:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 270b", + "mov eax, 13", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 49152", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 74752", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 49152", + "add rdi, 74752", "mov ecx, 256", - "246:", + "271:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", + "jne 271b", + "mov eax, 14", + "mov BYTE PTR [rbx+1280], al", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 50176", + "add rdi, 1216", + "mov esi, 2", "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 75776", + "mov rcx, rbx", + "add rcx, 2048", + "call {vg_mldsa_rej_bounded_poly}", "and r15d, eax", "mov r8d, 0", "sub r8d, eax", "mov rdi, rbx", - "add rdi, 50176", + "add rdi, 75776", "mov ecx, 256", - "247:", + "272:", "mov eax, DWORD PTR [rdi]", "and eax, r8d", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 272b", + "mov rdi, r12", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "248:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "273:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 248b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 273b", + "mov rdi, r13", + "add rdi, 0", "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 52224", - "mov ecx, 256", - "249:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1024", + "mov ecx, 32", + "274:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 249b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 6", - "mov BYTE PTR [rbx+1185], al", - "mov rdi, rbx", - "add rdi, 1152", + "jne 274b", + "mov rdi, r13", + "add rdi, 32", "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "250:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rsi, 1120", + "mov ecx, 32", + "275:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 250b", - "mov eax, 0", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "jne 275b", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 61440", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 128", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "251:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 251b", - "mov eax, 1", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 62464", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 224", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 63488", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 320", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "252:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 252b", - "mov eax, 2", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 64512", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 416", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 65536", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 512", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "253:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 253b", - "mov eax, 3", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 66560", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 608", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 67584", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 704", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "254:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 254b", - "mov eax, 4", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 68608", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 800", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 69632", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 896", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "255:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 255b", - "mov eax, 5", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 70656", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 992", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 71680", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1088", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "256:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 256b", - "mov eax, 6", - "mov BYTE PTR [rbx+1184], al", - "mov eax, 7", - "mov BYTE PTR [rbx+1185], al", + "add rdi, 72704", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1184", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1152", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 2048", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 73728", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1280", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 60416", - "mov ecx, 256", - "257:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 257b", - "mov eax, 0", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 74752", + "mov esi, 2", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1376", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", + "add rdi, 75776", "mov esi, 2", - "mov rdx, rbx", - "add rdx, 61440", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov edx, 2", + "mov rcx, r13", + "add rcx, 1472", + "mov r8d, 96", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 61440", - "mov ecx, 256", - "258:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 258b", - "mov eax, 1", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 62464", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 62464", - "mov ecx, 256", - "259:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 259b", - "mov eax, 2", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 63488", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 63488", - "mov ecx, 256", - "260:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 260b", - "mov eax, 3", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 64512", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 64512", - "mov ecx, 256", - "261:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 261b", - "mov eax, 4", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 65536", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 65536", - "mov ecx, 256", - "262:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 262b", - "mov eax, 5", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 66560", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 66560", - "mov ecx, 256", - "263:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 263b", - "mov eax, 6", - "mov BYTE PTR [rbx+1280], al", - "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 67584", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", "add rdi, 67584", - "mov ecx, 256", - "264:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 264b", - "mov eax, 7", - "mov BYTE PTR [rbx+1280], al", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 4096", "mov rdx, rbx", - "add rdx, 68608", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 68608", - "mov ecx, 256", - "265:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 265b", - "mov eax, 8", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 5120", + "mov rdx, rbx", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 6144", "mov rdx, rbx", - "add rdx, 69632", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "266:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 266b", - "mov eax, 9", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 7168", + "mov rdx, rbx", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 8192", "mov rdx, rbx", - "add rdx, 70656", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "267:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 267b", - "mov eax, 10", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 9216", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 10240", "mov rdx, rbx", - "add rdx, 71680", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "268:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 268b", - "mov eax, 11", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 68608", + "call {vg_mldsa_add}", + "mov rdi, rbx", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 72704", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "269:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 269b", - "mov eax, 12", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 32", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", - "mov rdx, rbx", - "add rdx, 73728", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", + "mov rcx, r13", + "add rcx, 1568", + "mov r8d, 416", + "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "270:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 270b", - "mov eax, 13", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 11264", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 12288", "mov rdx, rbx", - "add rdx, 74752", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 62464", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "271:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 271b", - "mov eax, 14", - "mov BYTE PTR [rbx+1280], al", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 13312", + "mov rdx, rbx", + "add rdx, 63488", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 1216", - "mov esi, 2", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 14336", "mov rdx, rbx", - "add rdx, 75776", - "mov rcx, rbx", - "add rcx, 2048", - "call {vg_mldsa_rej_bounded_poly}", - "and r15d, eax", - "mov r8d, 0", - "sub r8d, eax", + "add rdx, 64512", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "272:", - "mov eax, DWORD PTR [rdi]", - "and eax, r8d", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 272b", - "mov rdi, r12", - "add rdi, 0", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "273:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 273b", - "mov rdi, r13", - "add rdi, 0", + "add rsi, 15360", + "mov rdx, rbx", + "add rdx, 65536", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1024", - "mov ecx, 32", - "274:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 274b", - "mov rdi, r13", - "add rdi, 32", + "add rsi, 16384", + "mov rdx, rbx", + "add rdx, 66560", + "call {vg_mldsa_multiply_add_ntt}", + "mov rdi, rbx", + "add rdi, 76800", "mov rsi, rbx", - "add rsi, 1120", - "mov ecx, 32", - "275:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 275b", + "add rsi, 17408", + "mov rdx, rbx", + "add rdx, 67584", + "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", - "add rdi, 61440", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 128", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 2048", + "call {vg_mldsa_inv_ntt}", "mov rdi, rbx", - "add rdi, 62464", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 224", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 69632", + "call {vg_mldsa_add}", "mov rdi, rbx", - "add rdi, 63488", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 320", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 77824", + "mov rdx, rbx", + "add rdx, 78848", + "call {vg_mldsa_power2round}", "mov rdi, rbx", - "add rdi, 64512", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 416", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", + "add rdi, 77824", + "mov esi, 1023", + "mov rdx, r12", + "add rdx, 352", + "mov ecx, 320", + "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", - "add rdi, 65536", - "mov esi, 2", - "mov edx, 2", + "add rdi, 78848", + "mov esi, 4095", + "mov edx, 4096", "mov rcx, r13", - "add rcx, 512", - "mov r8d, 96", + "add rcx, 1984", + "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", - "add rdi, 66560", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 608", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 67584", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 704", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 68608", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 800", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 69632", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 896", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 70656", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 992", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 71680", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1088", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 72704", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1184", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 73728", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1280", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 74752", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1376", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 75776", - "mov esi, 2", - "mov edx, 2", - "mov rcx, r13", - "add rcx, 1472", - "mov r8d, 96", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 61440", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 62464", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 63488", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 64512", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 65536", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 66560", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 67584", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 4096", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", + "add rdi, 76800", + "mov rsi, rbx", + "add rsi, 18432", + "mov rdx, rbx", + "add rdx, 61440", + "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 5120", + "add rsi, 19456", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 6144", + "add rsi, 20480", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 7168", + "add rsi, 21504", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 8192", + "add rsi, 22528", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 9216", + "add rsi, 23552", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 10240", + "add rsi, 24576", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10048,7 +9490,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 68608", + "add rsi, 70656", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10061,7 +9503,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 32", + "add rdx, 672", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10069,55 +9511,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1568", + "add rcx, 2400", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 11264", + "add rsi, 25600", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 12288", + "add rsi, 26624", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 13312", + "add rsi, 27648", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 14336", + "add rsi, 28672", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 15360", + "add rsi, 29696", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 16384", + "add rsi, 30720", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 17408", + "add rsi, 31744", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10129,7 +9571,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 69632", + "add rsi, 71680", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10142,7 +9584,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 352", + "add rdx, 992", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10150,55 +9592,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 1984", + "add rcx, 2816", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 18432", + "add rsi, 32768", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 19456", + "add rsi, 33792", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 20480", + "add rsi, 34816", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 21504", + "add rsi, 35840", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 22528", + "add rsi, 36864", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 23552", + "add rsi, 37888", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 24576", + "add rsi, 38912", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10210,7 +9652,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 70656", + "add rsi, 72704", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10223,7 +9665,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 672", + "add rdx, 1312", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10231,55 +9673,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2400", + "add rcx, 3232", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 25600", + "add rsi, 39936", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 26624", + "add rsi, 40960", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 27648", + "add rsi, 41984", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 28672", + "add rsi, 43008", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 29696", + "add rsi, 44032", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 30720", + "add rsi, 45056", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 31744", + "add rsi, 46080", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10291,7 +9733,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 71680", + "add rsi, 73728", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10304,7 +9746,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 992", + "add rdx, 1632", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10312,55 +9754,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 2816", + "add rcx, 3648", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 32768", + "add rsi, 47104", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 33792", + "add rsi, 48128", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 34816", + "add rsi, 49152", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 35840", + "add rsi, 50176", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 36864", + "add rsi, 51200", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 37888", + "add rsi, 52224", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 38912", + "add rsi, 53248", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10372,7 +9814,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 72704", + "add rsi, 74752", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10385,7 +9827,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1312", + "add rdx, 1952", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10393,55 +9835,55 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3232", + "add rcx, 4064", "mov r8d, 416", "call {vg_mldsa_bit_pack}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 39936", + "add rsi, 54272", "mov rdx, rbx", "add rdx, 61440", "call {vg_mldsa_multiply_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 40960", + "add rsi, 55296", "mov rdx, rbx", "add rdx, 62464", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 41984", + "add rsi, 56320", "mov rdx, rbx", "add rdx, 63488", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 43008", + "add rsi, 57344", "mov rdx, rbx", "add rdx, 64512", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 44032", + "add rsi, 58368", "mov rdx, rbx", "add rdx, 65536", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 45056", + "add rsi, 59392", "mov rdx, rbx", "add rdx, 66560", "call {vg_mldsa_multiply_add_ntt}", "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 46080", + "add rsi, 60416", "mov rdx, rbx", "add rdx, 67584", "call {vg_mldsa_multiply_add_ntt}", @@ -10453,7 +9895,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov rdi, rbx", "add rdi, 76800", "mov rsi, rbx", - "add rsi, 73728", + "add rsi, 75776", "call {vg_mldsa_add}", "mov rdi, rbx", "add rdi, 76800", @@ -10466,7 +9908,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "add rdi, 77824", "mov esi, 1023", "mov rdx, r12", - "add rdx, 1632", + "add rdx, 2272", "mov ecx, 320", "call {vg_mldsa_simple_bit_pack}", "mov rdi, rbx", @@ -10474,197 +9916,35 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_keygen(seed: *const [u8; 32], pk "mov esi, 4095", "mov edx, 4096", "mov rcx, r13", - "add rcx, 3648", + "add rcx, 4480", "mov r8d, 416", "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 52224", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 74752", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 1952", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4064", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 61440", - "call {vg_mldsa_multiply_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 62464", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 63488", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 64512", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 65536", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 66560", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 60416", - "mov rdx, rbx", - "add rdx, 67584", - "call {vg_mldsa_multiply_add_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 2048", - "call {vg_mldsa_inv_ntt}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 75776", - "call {vg_mldsa_add}", - "mov rdi, rbx", - "add rdi, 76800", - "mov rsi, rbx", - "add rsi, 77824", - "mov rdx, rbx", - "add rdx, 78848", - "call {vg_mldsa_power2round}", - "mov rdi, rbx", - "add rdi, 77824", - "mov esi, 1023", - "mov rdx, r12", - "add rdx, 2272", - "mov ecx, 320", - "call {vg_mldsa_simple_bit_pack}", - "mov rdi, rbx", - "add rdi, 78848", - "mov esi, 4095", - "mov edx, 4096", - "mov rcx, r13", - "add rcx, 4480", - "mov r8d, 416", - "call {vg_mldsa_bit_pack}", - "mov eax, 0", - "mov QWORD PTR [rbx], rax", - "mov QWORD PTR [rbx+8], rax", - "mov QWORD PTR [rbx+16], rax", - "mov QWORD PTR [rbx+24], rax", - "mov QWORD PTR [rbx+32], rax", - "mov QWORD PTR [rbx+40], rax", - "mov QWORD PTR [rbx+48], rax", - "mov QWORD PTR [rbx+56], rax", - "mov QWORD PTR [rbx+64], rax", - "mov QWORD PTR [rbx+72], rax", - "mov QWORD PTR [rbx+80], rax", - "mov QWORD PTR [rbx+88], rax", - "mov QWORD PTR [rbx+96], rax", - "mov QWORD PTR [rbx+104], rax", - "mov QWORD PTR [rbx+112], rax", - "mov QWORD PTR [rbx+120], rax", - "mov QWORD PTR [rbx+128], rax", - "mov QWORD PTR [rbx+136], rax", - "mov QWORD PTR [rbx+144], rax", - "mov QWORD PTR [rbx+152], rax", - "mov QWORD PTR [rbx+160], rax", - "mov QWORD PTR [rbx+168], rax", - "mov QWORD PTR [rbx+176], rax", - "mov QWORD PTR [rbx+184], rax", - "mov QWORD PTR [rbx+192], rax", + "mov eax, 0", + "mov QWORD PTR [rbx], rax", + "mov QWORD PTR [rbx+8], rax", + "mov QWORD PTR [rbx+16], rax", + "mov QWORD PTR [rbx+24], rax", + "mov QWORD PTR [rbx+32], rax", + "mov QWORD PTR [rbx+40], rax", + "mov QWORD PTR [rbx+48], rax", + "mov QWORD PTR [rbx+56], rax", + "mov QWORD PTR [rbx+64], rax", + "mov QWORD PTR [rbx+72], rax", + "mov QWORD PTR [rbx+80], rax", + "mov QWORD PTR [rbx+88], rax", + "mov QWORD PTR [rbx+96], rax", + "mov QWORD PTR [rbx+104], rax", + "mov QWORD PTR [rbx+112], rax", + "mov QWORD PTR [rbx+120], rax", + "mov QWORD PTR [rbx+128], rax", + "mov QWORD PTR [rbx+136], rax", + "mov QWORD PTR [rbx+144], rax", + "mov QWORD PTR [rbx+152], rax", + "mov QWORD PTR [rbx+160], rax", + "mov QWORD PTR [rbx+168], rax", + "mov QWORD PTR [rbx+176], rax", + "mov QWORD PTR [rbx+184], rax", + "mov QWORD PTR [rbx+192], rax", "mov rdi, rbx", "add rdi, 0", "mov esi, 136", @@ -13576,7 +12856,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_sign(sk: *const [u8; 4896], mu: /// * `scratch` must be valid for reads and writes of 144384 bytes. /// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 204 §3.6.3). /// * `scratch` must not overlap `pk`, `mu` or `sig` (distinct Rust objects never do). -/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 24 bytes of stack below it, or wrap around the end of the address space (no Rust object does). +/// * None of `pk`, `mu`, `sig` and `scratch` may overlap the return address on the stack or the 32 bytes of stack below it, or wrap around the end of the address space (no Rust object does). #[unsafe(naked)] pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu: *const [u8; 64], sig: *const [u8; 4627], scratch: *mut [u64; 18048]) -> u32 { core::arch::naked_asm!( @@ -13636,1425 +12916,705 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov edx, 524287", "mov ecx, 524288", "mov r8, rbx", - "add r8, 18432", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 18432", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 1984", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 19456", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 19456", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 2624", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 20480", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 20480", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3264", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 21504", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 21504", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "mov rdi, r13", - "add rdi, 3904", - "mov esi, 640", - "mov edx, 524287", - "mov ecx, 524288", - "mov r8, rbx", - "add r8, 22528", - "call {vg_mldsa_bit_unpack}", - "mov rdi, rbx", - "add rdi, 22528", - "mov esi, 524168", - "call {vg_mldsa_norm_lt}", - "and r15d, eax", - "test r15d, r15d", - "jne 22f", - "jmp 23f", - "22:", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbp", - "add rsi, 0", - "mov ecx, 32", - "24:", - "movzx eax, BYTE PTR [rsi]", - "mov BYTE PTR [rdi], al", - "add rdi, 1", - "add rsi, 1", - "sub rcx, 1", - "jne 24b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 28672", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 28672", - "mov ecx, 256", - "25:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 25b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 29696", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 29696", - "mov ecx, 256", - "26:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 26b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 30720", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 30720", - "mov ecx, 256", - "27:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 27b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 31744", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 31744", - "mov ecx, 256", - "28:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 28b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 32768", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 32768", - "mov ecx, 256", - "29:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 29b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 33792", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 33792", - "mov ecx, 256", - "210:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 210b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 0", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 34816", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 34816", - "mov ecx, 256", - "211:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 211b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 36864", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 36864", - "mov ecx, 256", - "212:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 212b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 37888", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 37888", - "mov ecx, 256", - "213:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 213b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 38912", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 38912", - "mov ecx, 256", - "214:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 214b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 39936", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 39936", - "mov ecx, 256", - "215:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 215b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 40960", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 40960", - "mov ecx, 256", - "216:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 216b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 41984", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 41984", - "mov ecx, 256", - "217:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 217b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 1", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 43008", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 43008", - "mov ecx, 256", - "218:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 218b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 45056", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 45056", - "mov ecx, 256", - "219:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 219b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 46080", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 46080", - "mov ecx, 256", - "220:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 220b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 47104", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 47104", - "mov ecx, 256", - "221:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 221b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 48128", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 48128", - "mov ecx, 256", - "222:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 222b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 49152", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 49152", - "mov ecx, 256", - "223:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 223b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 50176", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 50176", - "mov ecx, 256", - "224:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 224b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 2", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 51200", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 51200", - "mov ecx, 256", - "225:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 225b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 53248", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 53248", - "mov ecx, 256", - "226:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 226b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 54272", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 54272", - "mov ecx, 256", - "227:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 227b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 55296", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 55296", - "mov ecx, 256", - "228:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 228b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 56320", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 56320", - "mov ecx, 256", - "229:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 229b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 57344", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 57344", - "mov ecx, 256", - "230:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 230b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 58368", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 58368", - "mov ecx, 256", - "231:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 231b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 3", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 59392", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 59392", - "mov ecx, 256", - "232:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 232b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 61440", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 61440", - "mov ecx, 256", - "233:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 233b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 62464", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 62464", - "mov ecx, 256", - "234:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 234b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 63488", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 63488", - "mov ecx, 256", - "235:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 235b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 64512", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 64512", - "mov ecx, 256", - "236:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 236b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 65536", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 65536", - "mov ecx, 256", - "237:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 237b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 66560", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 66560", - "mov ecx, 256", - "238:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 238b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 4", - "mov BYTE PTR [rbx+929], al", - "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 67584", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", - "mov rdi, rbx", - "add rdi, 67584", - "mov ecx, 256", - "239:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 239b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add r8, 18432", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 69632", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 18432", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 1984", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 19456", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 69632", - "mov ecx, 256", - "240:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 240b", - "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 19456", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 2624", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 20480", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 70656", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 20480", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "mov rdi, r13", + "add rdi, 3264", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 21504", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 70656", - "mov ecx, 256", - "241:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", - "sub rcx, 1", - "jne 241b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "add rdi, 21504", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", + "and r15d, eax", + "mov rdi, r13", + "add rdi, 3904", + "mov esi, 640", + "mov edx, 524287", + "mov ecx, 524288", + "mov r8, rbx", + "add r8, 22528", + "call {vg_mldsa_bit_unpack}", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 71680", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdi, 22528", + "mov esi, 524168", + "call {vg_mldsa_norm_lt}", "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "test r15d, r15d", + "jne 22f", + "jmp 23f", + "22:", "mov rdi, rbx", - "add rdi, 71680", - "mov ecx, 256", - "242:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 896", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "24:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 242b", - "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 24b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 72704", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2560", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "25:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 25b", "mov rdi, rbx", - "add rdi, 72704", - "mov ecx, 256", - "243:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2594", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "26:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 243b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 26b", "mov rdi, rbx", - "add rdi, 896", - "mov rsi, rbx", - "add rsi, 73728", - "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", - "and r15d, eax", - "mov edx, 0", - "sub edx, eax", + "add rdi, 2628", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "27:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", + "sub rcx, 1", + "jne 27b", "mov rdi, rbx", - "add rdi, 73728", - "mov ecx, 256", - "244:", - "mov eax, DWORD PTR [rdi]", - "and eax, edx", - "mov DWORD PTR [rdi], eax", - "add rdi, 4", + "add rdi, 2662", + "mov rsi, rbp", + "add rsi, 0", + "mov ecx, 32", + "28:", + "movzx eax, BYTE PTR [rsi]", + "mov BYTE PTR [rdi], al", + "add rdi, 1", + "add rsi, 1", "sub rcx, 1", - "jne 244b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", - "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "jne 28b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 74752", + "add rsi, 28672", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 74752", - "mov ecx, 256", - "245:", + "add rdi, 28672", + "mov ecx, 1024", + "29:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 245b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "jne 29b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 0", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 75776", + "add rsi, 31744", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 75776", - "mov ecx, 256", - "246:", + "add rdi, 31744", + "mov ecx, 1024", + "210:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 246b", + "jne 210b", "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 77824", + "add rsi, 36864", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 77824", - "mov ecx, 256", - "247:", + "add rdi, 36864", + "mov ecx, 1024", + "211:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 247b", + "jne 211b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 78848", + "add rsi, 39936", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 78848", - "mov ecx, 256", - "248:", + "add rdi, 39936", + "mov ecx, 1024", + "212:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 248b", + "jne 212b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 79872", + "add rsi, 45056", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 79872", - "mov ecx, 256", - "249:", + "add rdi, 45056", + "mov ecx, 1024", + "213:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 249b", + "jne 213b", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 80896", + "add rsi, 48128", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 80896", - "mov ecx, 256", - "250:", + "add rdi, 48128", + "mov ecx, 1024", + "214:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 250b", - "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 214b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 81920", + "add rsi, 53248", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 81920", - "mov ecx, 256", - "251:", + "add rdi, 53248", + "mov ecx, 1024", + "215:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 251b", + "jne 215b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2627], al", "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 82944", + "add rsi, 56320", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 82944", - "mov ecx, 256", - "252:", + "add rdi, 56320", + "mov ecx, 1024", + "216:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 252b", - "mov eax, 6", - "mov BYTE PTR [rbx+928], al", - "mov eax, 6", - "mov BYTE PTR [rbx+929], al", + "jne 216b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 83968", + "add rsi, 61440", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 83968", - "mov ecx, 256", - "253:", + "add rdi, 61440", + "mov ecx, 1024", + "217:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 253b", - "mov eax, 0", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 217b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 86016", + "add rsi, 64512", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 86016", - "mov ecx, 256", - "254:", + "add rdi, 64512", + "mov ecx, 1024", + "218:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 254b", + "jne 218b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", "mov eax, 1", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 87040", + "add rsi, 69632", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 87040", - "mov ecx, 256", - "255:", + "add rdi, 69632", + "mov ecx, 1024", + "219:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 255b", - "mov eax, 2", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "jne 219b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 88064", + "add rsi, 72704", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 88064", - "mov ecx, 256", - "256:", + "add rdi, 72704", + "mov ecx, 1024", + "220:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 256b", + "jne 220b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", "mov eax, 3", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 89088", + "add rsi, 77824", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 89088", - "mov ecx, 256", - "257:", + "add rdi, 77824", + "mov ecx, 1024", + "221:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 257b", + "jne 221b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2593], al", "mov eax, 4", - "mov BYTE PTR [rbx+928], al", - "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2694], al", + "mov eax, 6", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 90112", + "add rsi, 80896", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 90112", - "mov ecx, 256", - "258:", + "add rdi, 80896", + "mov ecx, 1024", + "222:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 258b", - "mov eax, 5", - "mov BYTE PTR [rbx+928], al", + "jne 222b", + "mov eax, 0", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 1", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 2", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", + "mov eax, 3", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 91136", + "add rsi, 86016", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 91136", - "mov ecx, 256", - "259:", + "add rdi, 86016", + "mov ecx, 1024", + "223:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 259b", + "jne 223b", + "mov eax, 3", + "mov BYTE PTR [rbx+2592], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2593], al", + "mov eax, 4", + "mov BYTE PTR [rbx+2626], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2627], al", + "mov eax, 5", + "mov BYTE PTR [rbx+2660], al", + "mov eax, 7", + "mov BYTE PTR [rbx+2661], al", "mov eax, 6", - "mov BYTE PTR [rbx+928], al", + "mov BYTE PTR [rbx+2694], al", "mov eax, 7", - "mov BYTE PTR [rbx+929], al", + "mov BYTE PTR [rbx+2695], al", "mov rdi, rbx", - "add rdi, 896", + "add rdi, 2560", "mov rsi, rbx", - "add rsi, 92160", + "add rsi, 89088", "mov rdx, rbx", - "add rdx, 4096", - "call {vg_mldsa_rej_ntt_poly}", + "add rdx, 94208", + "call {vg_mldsa_rej_ntt_poly4}", "and r15d, eax", "mov edx, 0", "sub edx, eax", "mov rdi, rbx", - "add rdi, 92160", - "mov ecx, 256", - "260:", + "add rdi, 89088", + "mov ecx, 1024", + "224:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 260b", + "jne 224b", "mov rdi, r13", "add rdi, 0", "mov esi, 64", @@ -15070,13 +13630,13 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "mov rdi, rbx", "add rdi, 23552", "mov ecx, 256", - "261:", + "225:", "mov eax, DWORD PTR [rdi]", "and eax, edx", "mov DWORD PTR [rdi], eax", "add rdi, 4", "sub rcx, 1", - "jne 261b", + "jne 225b", "mov rdi, rbx", "add rdi, 16384", "mov rsi, rbx", @@ -15915,7 +14475,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rdi, 0", "mov ecx, 64", "mov edx, 0", - "262:", + "226:", "movzx eax, BYTE PTR [rsi]", "movzx r8d, BYTE PTR [rdi]", "xor rax, r8", @@ -15923,7 +14483,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu "add rsi, 1", "add rdi, 1", "sub rcx, 1", - "jne 262b", + "jne 226b", "sub rdx, 1", "sbb rax, rax", "and r15d, eax", @@ -15940,7 +14500,7 @@ pub(crate) unsafe extern "sysv64" fn vg_mldsa87_verify(pk: *const [u8; 2592], mu vg_mldsa_hint_bit_unpack = sym super::mldsa::vg_mldsa_hint_bit_unpack, vg_mldsa_bit_unpack = sym super::mldsa::vg_mldsa_bit_unpack, vg_mldsa_norm_lt = sym super::mldsa::vg_mldsa_norm_lt, - vg_mldsa_rej_ntt_poly = sym super::mldsa::vg_mldsa_rej_ntt_poly, + vg_mldsa_rej_ntt_poly4 = sym super::mldsa::vg_mldsa_rej_ntt_poly4, vg_mldsa_sample_in_ball = sym super::mldsa::vg_mldsa_sample_in_ball, vg_mldsa_ntt = sym super::mldsa::vg_mldsa_ntt, vg_mldsa_multiply_ntt = sym super::mldsa::vg_mldsa_multiply_ntt,